Add local MarianMT translation service

Reviewed-on: http://192.168.68.119:3000/adolforeyna/EMI-Backend/pulls/5

.gitignore

@@ -118,6 +118,10 @@ dist
 .yarn/build-state.yml
 .yarn/install-state.gz
 .pnp.*
+
+# Local Python environments
+translation-service/.venv/
+translation-service/__pycache__/
 # Logs
 logs
 *.log

README.md

@@ -31,6 +31,27 @@ A step by step series of examples that tell you how to get a development env run
     npm start
     ```
 
+### Local MarianMT translation
+
+The translation service is independent from the Node backend and provides `GET /health` and `POST /translate`.
+
+1. Start it locally:
+   ```
+   cd translation-service
+   python3 -m venv .venv
+   .venv/bin/pip install -r requirements.txt
+   .venv/bin/python server.py
+   ```
+2. Configure the Node backend:
+   ```
+   TRANSLATION_PROVIDER=marian
+   MARIAN_TRANSLATION_URL=http://127.0.0.1:8000
+   ```
+
+With Docker Compose, the service runs as the internal `translation` service. Set `TRANSLATION_PROVIDER=marian` before running `docker compose up`. MarianMT models download only when first needed and are stored in the `translation-models` Docker volume.
+
+Supported languages are English (`en`), Spanish (`es`), French (`fr`), Danish (`da`), and Arabic (`ar`). Non-English pairs translate through English. Keep the service on the internal Docker network; it has no public port mapping.
+
 ### API Documentation
 
 Once the server is running, you can access the interactive API documentation powered by Swagger UI at:
@@ -130,6 +151,14 @@ The API is divided into several sections based on functionality. Most routes und
 -   `GET /chapters/:chapterId`: Get the content of a chapter.
 -   `GET /chapters/:chapterId/verses`: Get the verses of a chapter.
 -   `GET /search`: Search the Bible.
+-   `GET /mine`: Get Bible highlights and notes for the active profile.
+-   `GET /verses/:verseKey/counters`: Get highlight and note counters for a verse key like `GEN.1.1`.
+-   `GET /verses/:verseKey/activity`: Get profiles that highlighted/noted a verse and recent notes, limited to 10 by default.
+-   `POST /verses/:verseKey/highlight`: Highlight a verse for the active profile.
+-   `DELETE /verses/:verseKey/highlight`: Remove a verse highlight for the active profile.
+-   `POST /verses/:verseKey/notes`: Add a note for a verse.
+-   `PUT /verses/:verseKey/notes/:noteId`: Update one of the active profile's notes.
+-   `DELETE /verses/:verseKey/notes/:noteId`: Delete one of the active profile's notes.
 
 ### Subsplash (`/subsplash`)
 

auth/authEmail.js

@@ -3,7 +3,7 @@ const { client_logger } = require('../utils/analyticsLogger');
 const bcrypt = require('bcrypt');
 const crypto = require('crypto');
 const { getSessionId, getUserId, getProfileId } = require('../utils/sessionUtils.js');
-const { cookiesOptions } = require('../config/cookiesOptions');
+const { getCookiesOptions } = require('../config/cookiesOptions');
 const Notifications = require("../notifications");
 
 // Object Definitions
@@ -19,6 +19,7 @@ const createPasswordTokenHash = (rawToken) =>
 
 const createSessionFromUser = async ({ DB, user, req, res }) => {
     const sessionObj = await DB.newSession(user._id);
+    const cookiesOptions = getCookiesOptions(req);
     res.cookie('user_sid', user._id, cookiesOptions);
     res.cookie('session_id', sessionObj.insertedId, cookiesOptions);
     const latestUpdatedProfile = await DB.latestProfile(user._id);
@@ -143,8 +144,9 @@ const logout = async function (req, res) {
     const session_id = getSessionId(req);
     const user_sid = getUserId(req);
     if (session_id && user_sid) {
-        res.clearCookie('session_id');
-        res.clearCookie('user_sid');
+        const cookiesOptions = getCookiesOptions(req);
+        res.clearCookie('session_id', cookiesOptions);
+        res.clearCookie('user_sid', cookiesOptions);
         //remove from DB
         const DB = await MongoDB.getDB;
         DB.removeSession(session_id);

config/cookiesOptions.js

@@ -1,12 +1,49 @@
 const isProduction = process.env.NODE_ENV === "production";
 const forceSecureCookie = process.env.COOKIE_SECURE === "true";
-const secure = forceSecureCookie || isProduction;
 
-const cookiesOptions = {
-    maxAge: 1000 * 60 * 60 * 24 * 90, // would expire after 90 days
-    httpOnly: true, // The cookie only accessible by the web server
-    sameSite: secure ? 'none' : 'lax',
-    secure,
+const COOKIE_MAX_AGE_MS = 1000 * 60 * 60 * 24 * 90; // 90 days
+const LOCAL_ORIGIN_REGEX = /^http:\/\/(localhost|127\.0\.0\.1|aeropi\.local)(:\d+)?$/i;
+const LOCAL_HOST_REGEX = /^(localhost|127\.0\.0\.1|aeropi\.local)(:\d+)?$/i;
+
+const getHeaderValue = (req, key) => {
+    if (!req || !req.headers) return "";
+    const raw = req.headers[key];
+    if (Array.isArray(raw)) return raw[0] || "";
+    return raw || "";
 };
 
-module.exports = { cookiesOptions };
+const isLocalRequest = (req) => {
+    const origin = getHeaderValue(req, "origin");
+    const host = getHeaderValue(req, "host");
+    return LOCAL_ORIGIN_REGEX.test(origin) || LOCAL_HOST_REGEX.test(host);
+};
+
+const isHttpsRequest = (req) => {
+    if (!req) return false;
+    const forwardedProto = String(getHeaderValue(req, "x-forwarded-proto")).split(",")[0].trim().toLowerCase();
+    const reqProtocol = String(req.protocol || "").toLowerCase();
+    const origin = String(getHeaderValue(req, "origin") || "").toLowerCase();
+    if (forwardedProto === "https" || reqProtocol === "https") return true;
+    return origin.startsWith("https://");
+};
+
+const shouldUseSecureCookie = (req) => {
+    if (forceSecureCookie) return true;
+    if (isLocalRequest(req)) return false;
+    if (isHttpsRequest(req)) return true;
+    return isProduction;
+};
+
+const getCookiesOptions = (req) => {
+    const secure = shouldUseSecureCookie(req);
+    return {
+        maxAge: COOKIE_MAX_AGE_MS,
+        httpOnly: true,
+        sameSite: secure ? "none" : "lax",
+        secure,
+    };
+};
+
+const cookiesOptions = getCookiesOptions();
+
+module.exports = { cookiesOptions, getCookiesOptions };

config/corsOptions.js

@@ -7,6 +7,7 @@ var corsOptions = {
         'http://127.0.0.1:8081',
         'http://localhost:3000', 
         "https://social.emmint.com",  
+        "https://www.social.emmint.com",
         "https://fellowship.emmint.com",
         "https://aeropi.local",
     ],

dbTools/bible.js

@@ -0,0 +1,287 @@
+const DBName = "EMI_SOCIAL";
+
+const toProfileIds = (DB, profileIds = []) => {
+    return profileIds
+        .filter((profileId) => DB.ObjectID.isValid(profileId))
+        .map((profileId) => DB.ObjectID(profileId));
+};
+
+const publicProfile = (profile) => {
+    if (!profile) return null;
+    return {
+        _id: profile._id,
+        profile: profile.profile,
+        username: profile.username,
+        isGroup: profile.isGroup,
+        isCourse: profile.isCourse,
+    };
+};
+
+const bibleDB = (DB) => {
+    DB.bibleVerseCols = DB.db.db(DBName).collection("bible_verses");
+    DB.bibleVerseCols.createIndex({ highlightedBy: 1 }).catch(console.error);
+    DB.bibleVerseCols.createIndex({ notedBy: 1 }).catch(console.error);
+
+    const getVerseDoc = async (verseKey) => {
+        return DB.bibleVerseCols.findOne({ _id: verseKey }).catch((err) => {
+            console.log(err);
+            return false;
+        });
+    };
+
+    DB.getBibleProfileData = async (profileid) => {
+        if (!DB.ObjectID.isValid(profileid)) return false;
+        const profile = await DB.profileCols.findOne(
+            { _id: DB.ObjectID(profileid) },
+            { projection: { bibleHighlights: 1, bibleNotes: 1 } }
+        ).catch((err) => {
+            console.log(err);
+            return false;
+        });
+
+        return {
+            highlights: profile?.bibleHighlights || [],
+            notes: profile?.bibleNotes || [],
+        };
+    };
+
+    DB.getBibleVerseCounters = async (verseKey) => {
+        const verse = await getVerseDoc(verseKey);
+        const highlightedBy = Array.isArray(verse?.highlightedBy) ? verse.highlightedBy : [];
+        const notedBy = Array.isArray(verse?.notedBy) ? verse.notedBy : [];
+        return {
+            verseKey,
+            highlightCount: highlightedBy.length,
+            noteProfileCount: notedBy.length,
+            notesCount: verse?.notesCount || 0,
+        };
+    };
+
+    DB.getBibleVerseActivity = async (verseKey, limit = 10) => {
+        const verse = await getVerseDoc(verseKey);
+        const highlightedBy = Array.isArray(verse?.highlightedBy) ? verse.highlightedBy : [];
+        const notedBy = Array.isArray(verse?.notedBy) ? verse.notedBy : [];
+        const highlightProfiles = toProfileIds(DB, highlightedBy);
+        const noteProfiles = toProfileIds(DB, notedBy);
+
+        const [highlights, noteUsers, notes] = await Promise.all([
+            highlightProfiles.length
+                ? DB.profileCols.find({ _id: { $in: highlightProfiles } }).project({
+                    profile: 1,
+                    username: 1,
+                    isGroup: 1,
+                    isCourse: 1,
+                }).toArray()
+                : [],
+            noteProfiles.length
+                ? DB.profileCols.find({ _id: { $in: noteProfiles } }).project({
+                    profile: 1,
+                    username: 1,
+                    isGroup: 1,
+                    isCourse: 1,
+                }).toArray()
+                : [],
+            DB.profileCols.aggregate([
+                { $match: { "bibleNotes.verseKey": verseKey } },
+                { $unwind: "$bibleNotes" },
+                { $match: { "bibleNotes.verseKey": verseKey } },
+                { $sort: { "bibleNotes.updatedAt": -1, "bibleNotes.createdAt": -1 } },
+                { $limit: limit },
+                {
+                    $project: {
+                        _id: 0,
+                        note: "$bibleNotes",
+                        profile: {
+                            _id: "$_id",
+                            profile: "$profile",
+                            username: "$username",
+                            isGroup: "$isGroup",
+                            isCourse: "$isCourse",
+                        }
+                    }
+                }
+            ]).toArray(),
+        ]);
+
+        return {
+            counters: await DB.getBibleVerseCounters(verseKey),
+            highlightedBy: highlights.map(publicProfile),
+            notedBy: noteUsers.map(publicProfile),
+            notes,
+        };
+    };
+
+    DB.addBibleHighlight = async (profileid, verseKey) => {
+        if (!DB.ObjectID.isValid(profileid)) return false;
+        const now = new Date();
+        const _id = DB.ObjectID(profileid);
+        const profileUpdate = await DB.profileCols.updateOne(
+            { _id, "bibleHighlights.verseKey": { $ne: verseKey } },
+            {
+                $push: {
+                    bibleHighlights: {
+                        verseKey,
+                        createdAt: now,
+                    }
+                },
+                $set: { lastUpdate: now }
+            }
+        ).catch((err) => {
+            console.log(err);
+            return false;
+        });
+
+        await DB.bibleVerseCols.updateOne(
+            { _id: verseKey },
+            {
+                $setOnInsert: { createdAt: now },
+                $set: { updatedAt: now },
+                $addToSet: { highlightedBy: profileid + "" }
+            },
+            { upsert: true }
+        ).catch((err) => {
+            console.log(err);
+            return false;
+        });
+
+        if (DB.clearProfileCache) DB.clearProfileCache(profileid);
+        return profileUpdate;
+    };
+
+    DB.removeBibleHighlight = async (profileid, verseKey) => {
+        if (!DB.ObjectID.isValid(profileid)) return false;
+        const now = new Date();
+        const _id = DB.ObjectID(profileid);
+        const profileUpdate = await DB.profileCols.updateOne(
+            { _id },
+            {
+                $pull: { bibleHighlights: { verseKey } },
+                $set: { lastUpdate: now }
+            }
+        ).catch((err) => {
+            console.log(err);
+            return false;
+        });
+
+        await DB.bibleVerseCols.updateOne(
+            { _id: verseKey },
+            {
+                $pull: { highlightedBy: profileid + "" },
+                $set: { updatedAt: now }
+            }
+        ).catch((err) => {
+            console.log(err);
+            return false;
+        });
+
+        if (DB.clearProfileCache) DB.clearProfileCache(profileid);
+        return profileUpdate;
+    };
+
+    DB.addBibleNote = async (profileid, verseKey, text) => {
+        if (!DB.ObjectID.isValid(profileid)) return false;
+        const now = new Date();
+        const note = {
+            _id: DB.ObjectID().toString(),
+            verseKey,
+            text,
+            createdAt: now,
+            updatedAt: now,
+        };
+        const _id = DB.ObjectID(profileid);
+        const profileUpdate = await DB.profileCols.updateOne(
+            { _id },
+            {
+                $push: { bibleNotes: note },
+                $set: { lastUpdate: now }
+            }
+        ).catch((err) => {
+            console.log(err);
+            return false;
+        });
+
+        await DB.bibleVerseCols.updateOne(
+            { _id: verseKey },
+            {
+                $setOnInsert: { createdAt: now },
+                $set: { updatedAt: now },
+                $addToSet: { notedBy: profileid + "" },
+                $inc: { notesCount: 1 }
+            },
+            { upsert: true }
+        ).catch((err) => {
+            console.log(err);
+            return false;
+        });
+
+        if (DB.clearProfileCache) DB.clearProfileCache(profileid);
+        return profileUpdate ? note : false;
+    };
+
+    DB.updateBibleNote = async (profileid, verseKey, noteId, text) => {
+        if (!DB.ObjectID.isValid(profileid)) return false;
+        const now = new Date();
+        const _id = DB.ObjectID(profileid);
+        const result = await DB.profileCols.findOneAndUpdate(
+            { _id, bibleNotes: { $elemMatch: { _id: noteId, verseKey } } },
+            {
+                $set: {
+                    "bibleNotes.$.text": text,
+                    "bibleNotes.$.updatedAt": now,
+                    lastUpdate: now,
+                }
+            },
+            {
+                returnOriginal: false,
+                projection: { bibleNotes: 1 }
+            }
+        ).catch((err) => {
+            console.log(err);
+            return false;
+        });
+
+        if (DB.clearProfileCache) DB.clearProfileCache(profileid);
+        return result?.value?.bibleNotes?.find((note) => note._id === noteId) || null;
+    };
+
+    DB.removeBibleNote = async (profileid, verseKey, noteId) => {
+        if (!DB.ObjectID.isValid(profileid)) return false;
+        const now = new Date();
+        const _id = DB.ObjectID(profileid);
+        const result = await DB.profileCols.updateOne(
+            { _id, bibleNotes: { $elemMatch: { _id: noteId, verseKey } } },
+            {
+                $pull: { bibleNotes: { _id: noteId, verseKey } },
+                $set: { lastUpdate: now }
+            }
+        ).catch((err) => {
+            console.log(err);
+            return false;
+        });
+
+        if (result?.modifiedCount) {
+            const stillHasNotes = await DB.profileCols.findOne(
+                { _id, "bibleNotes.verseKey": verseKey },
+                { projection: { _id: 1 } }
+            );
+
+            const verseUpdate = {
+                $inc: { notesCount: -1 },
+                $set: { updatedAt: now }
+            };
+            if (!stillHasNotes) {
+                verseUpdate.$pull = { notedBy: profileid + "" };
+            }
+
+            await DB.bibleVerseCols.updateOne({ _id: verseKey }, verseUpdate).catch((err) => {
+                console.log(err);
+                return false;
+            });
+        }
+
+        if (DB.clearProfileCache) DB.clearProfileCache(profileid);
+        return result;
+    };
+}
+
+module.exports = bibleDB;

dbTools/chat.js

@@ -27,6 +27,13 @@ const chatDB = (DB) => {
         };
     };
 
+    DB.getChatParticipants = async () => {
+        return DB.chatMessagesCol.distinct("senderProfileId").catch((err) => {
+            console.log(err);
+            return [];
+        });
+    };
+
     DB.getRecentChatMessages = async (limit = 100) => {
         const safeLimit = Math.min(Math.max(parseInt(limit, 10) || 100, 1), 200);
         const messages = await DB.chatMessagesCol.find({})

dbTools/post.js

@@ -38,6 +38,20 @@ postDB = (DB)=>{
         });
     }
 
+    DB.addTranslation = (postid, lang, translatedText) => {
+        if(!DB.ObjectID.isValid(postid)) return false;
+        const id = DB.ObjectID(postid);
+        let update = {
+            $set:{
+                ["translations." + lang]: translatedText
+            }
+        }
+        return DB.postCols.updateOne({_id: id}, update).catch((err)=>{
+            console.log(err);
+            return false;
+        });
+    }
+
     DB.newReaction = (postid, profileid, reaction) => {
         if(!DB.ObjectID.isValid(postid)) return false;
         const id = DB.ObjectID(postid);

dbTools/profile.js

@@ -5,6 +5,10 @@ let userProfileCache = {};
 userDB = (DB) => {
     DB.profileCols = DB.db.db(DBName).collection("profiles");
 
+    DB.clearProfileCache = (profileid) => {
+        if (userProfileCache[profileid]) delete userProfileCache[profileid];
+    };
+
     DB.newProfile = (profileObj) => {
         return DB.profileCols.insertOne(profileObj.toObj()).catch((err) => {
             console.log(err);

def/profile.js

@@ -17,6 +17,8 @@ class User {
         this.lastUpdate = info.lastUpdate || new Date();
         this.newsFeedCache = info.newsFeedCache || [];
         this.notifications = info.notifications || [];
+        this.bibleHighlights = info.bibleHighlights || [];
+        this.bibleNotes = info.bibleNotes || [];
 
         //groupRelated
         this.isGroup = info.isGroup || false;

docker-compose.yml

@@ -18,15 +18,32 @@ services:
       - WEB_PUSH_EMAIL=${WEB_PUSH_EMAIL}
       - EMAILPASS=${EMAILPASS}
       - PORT=3001
+      - TRANSLATION_PROVIDER=${TRANSLATION_PROVIDER:-openai}
+      - MARIAN_TRANSLATION_URL=http://translation:8000
     volumes:
       - .:/app
       - '/app/node_modules'
     #depends_on:
     #  - mongo
     command: node index.js
+    depends_on:
+      - translation
+    networks:
+      - emi-network
   #  networks:
   #    - emi-network
 
+  translation:
+    build:
+      context: ./translation-service
+    restart: unless-stopped
+    environment:
+      - MARIAN_HOST=0.0.0.0
+    volumes:
+      - translation-models:/models
+    networks:
+      - emi-network
+
   #mongo:
   #  image: mongo:latest
   #  ports:
@@ -38,6 +55,8 @@ services:
   #    - ./dump:/dump
     #entrypoint: mongodump ${MONGO_URL} && mongorestore --db EMI_SOCIAL dump/EMI_SOCIAL/ && mongod
 
-#volumes:
+volumes:
+  translation-models:
+    driver: local
   #mongodbdata:
   #  driver: local  # This ensures the volume is created

index.js

@@ -29,6 +29,7 @@ const limiter = rateLimit({
     limit: 500, // Limit each IP to 100 requests per `window` (here, per 15 minutes).
     standardHeaders: 'draft-8', // draft-6: `RateLimit-*` headers; draft-7 & draft-8: combined `RateLimit` header
     legacyHeaders: false, // Disable the `X-RateLimit-*` headers.
+    skip: (req) => req.path.startsWith("/live-captions"),
     keyGenerator: (req) => {
         const forwarded = req.headers["x-forwarded-for"]?.split(",")[0]; // Take the first IP in the list
         const ip = forwarded || req.ip; // Fallback to req.ip
@@ -184,6 +185,7 @@ const songsRoute = require('./routes/songs.js');
 const paymentsRoute = require('./routes/payments.js');
 const bibleRoute = require('./routes/bible.js');
 const chatRoute = require('./routes/chat.js');
+const liveCaptionsRoute = require('./routes/liveCaptions.js');
 const sessionChecker = require('./middleware/sessionChecker');
 // -- Private Routes
 app.use('/user', sessionChecker, profileRoute);
@@ -192,6 +194,7 @@ app.use('/payments', paymentsRoute);
 app.use('/bible', sessionChecker, bibleRoute);
 app.use('/songs', sessionChecker, songsRoute);
 app.use('/chat', sessionChecker, chatRoute);
+app.use('/live-captions', liveCaptionsRoute);
 // -- Public Routes
 const subsplashRoute = require('./routes/subsplash.js');
 app.use('/subsplash', subsplashRoute);
@@ -240,7 +243,7 @@ const webPushEmail = process.env.WEB_PUSH_EMAIL;
 webPush.setVapidDetails('mailto:' + webPushEmail, publicVapidKey, privateVapidKey);
 
 
-const { cookiesOptions } = require('./config/cookiesOptions');
+const { getCookiesOptions } = require('./config/cookiesOptions');
 const { client_logger } = require('./utils/analyticsLogger.js');
 const { getSessionId, getUserId, getProfileId } = require('./utils/sessionUtils.js');
 
@@ -410,7 +413,7 @@ DB.getDB.then((DB) => {
                 return res.status(403).json({ status: "Profile does not belong to the logged-in user" });
             }
             // Update active profile cookie
-            res.cookie('profile_id', profile._id, cookiesOptions);
+            res.cookie('profile_id', profile._id, getCookiesOptions(req));
             return res.json({ status: "ok", profile });
         } catch (error) {
             console.error("Error changing profile:", error);

middleware/sessionChecker.js

@@ -1,9 +1,22 @@
 const { getSessionId, getUserId, getProfileId } = require('../utils/sessionUtils');
 const { client_logger } = require('../utils/analyticsLogger');
-const { cookiesOptions } = require('../config/cookiesOptions');
+const { getCookiesOptions } = require('../config/cookiesOptions');
 const MongoDB = require("../mongoDB.js");
 const { ObjectId } = require("mongodb");
 
+const shouldReturnJson = (req) => {
+    const accept = String(req?.headers?.accept || "").toLowerCase();
+    const contentType = String(req?.headers?.["content-type"] || "").toLowerCase();
+    return !!req?.headers?.origin || accept.includes("application/json") || contentType.includes("application/json");
+};
+
+const rejectUnauthorized = (req, res) => {
+    if (shouldReturnJson(req)) {
+        return res.status(401).json({ status: "Unauthorized" });
+    }
+    return res.redirect('/login');
+};
+
 const sessionChecker = async (req, res, next) => {
     try {
         const session_id = getSessionId(req);
@@ -11,10 +24,10 @@ const sessionChecker = async (req, res, next) => {
         let profile_id = getProfileId(req);
 
         if (!session_id || !user_sid) {
-            return res.redirect('/login');
+            return rejectUnauthorized(req, res);
         }
         if (!ObjectId.isValid(session_id) || !ObjectId.isValid(user_sid)) {
-            return res.redirect('/login');
+            return rejectUnauthorized(req, res);
         }
 
         const DB = await MongoDB.getDB;
@@ -24,15 +37,15 @@ const sessionChecker = async (req, res, next) => {
         if (!await DB.getProfileCache(profile_id)) {
             const latestProfile = await DB.latestProfile(user_sid);
             if (!latestProfile || !latestProfile._id) {
-                return res.redirect('/login');
+                return rejectUnauthorized(req, res);
             }
-            res.cookie('profile_id', latestProfile._id, cookiesOptions);
+            res.cookie('profile_id', latestProfile._id, getCookiesOptions(req));
             profile_id = latestProfile._id;
         }
 
         req.profileInfo = { _id: profile_id };
 
-        if (!userInfo) return res.redirect('/login');
+        if (!userInfo) return rejectUnauthorized(req, res);
 
         client_logger.capture({
             distinctId: user_sid,
@@ -42,7 +55,7 @@ const sessionChecker = async (req, res, next) => {
         next();
     } catch (error) {
         console.error("Session checker error", error);
-        return res.redirect('/login');
+        return rejectUnauthorized(req, res);
     }
 };
 

mongoDB.js

@@ -10,6 +10,7 @@ const profileDB = require("./dbTools/profile.js");
 const paymentDB = require("./dbTools/payments.js");
 const songsDB = require("./dbTools/songs.js");
 const chatDB = require("./dbTools/chat.js");
+const bibleDB = require("./dbTools/bible.js");
 
 console.log("Connecting to MongoDB...");
 const nodeMajorVersion = parseInt((process.versions.node || "0").split(".")[0], 10);
@@ -179,6 +180,7 @@ const getDB = new Promise((resolve, reject) => {
         paymentDB(DB);
         songsDB(DB);
         chatDB(DB);
+        bibleDB(DB);
     
         resolve(DB);
     });

notifications.js

@@ -489,6 +489,26 @@ const Notifications = {
     // sendWebNotification(requesterProfile.webSubscription, notifBody);
     DB.addNotification(requesterProfile, notifBody, null, null, groupProfile._id);
   },
+  async youGotANewChatMessage(senderProfileId, messageText) {
+    const DB = await DBGetter.getDB;
+    const participants = await DB.getChatParticipants();
+    const senderProfile = await DB.getProfileCache(senderProfileId);
+    
+    const tokens = [];
+    for (const participantProfileId of participants) {
+      if (participantProfileId.toString() === senderProfileId.toString()) continue;
+      
+      const participantProfile = await DB.getProfileCache(participantProfileId);
+      if (participantProfile && Array.isArray(participantProfile.token)) {
+        tokens.push(...participantProfile.token);
+      }
+    }
+
+    if (tokens.length > 0) {
+      const notifBody = `${senderProfile.profile.firstName}: ${messageText.substring(0, 100)}${messageText.length > 100 ? '...' : ''}`;
+      sendPushNotification(tokens, notifBody, { type: 'chat' });
+    }
+  },
   
 }
 

package.json

@@ -6,6 +6,8 @@
   "scripts": {
     "test": "npx mocha test/auth.test.js",
     "start": "node index.js",
+    "dev": "node --watch index.js",
+    "live-captions:test-sender": "node scripts/liveCaptionsTestSender.js",
     "docker": "docker compose up -d",
     "docker_restore": "docker-compose exec mongo mongorestore --db EMI_SOCIAL /dump/EMI_SOCIAL/",
     "docker_dump": "docker-compose exec mongo mongodump --uri ${MONGO_URL} --out /dump"

routes/bible.js

@@ -2,6 +2,7 @@ const axios = require('axios');
 var express = require('express')
 var router = express.Router()
 const DB = require("./../mongoDB.js");
+const { getProfileId } = require("./../utils/sessionUtils.js");
 
 const fetchAPI = async (path) => {
     baseUrl = "https://api.scripture.api.bible/v1/"
@@ -11,6 +12,20 @@ const fetchAPI = async (path) => {
 
 const defaultBibleId = "592420522e16049f-01";
 
+const normalizeVerseKey = (verseKey) => {
+    if (!verseKey || typeof verseKey !== "string") return "";
+    return verseKey.trim().toUpperCase();
+};
+
+const isValidVerseKey = (verseKey) => {
+    return /^[A-Z0-9]+(\.[A-Z0-9]+)+$/.test(verseKey);
+};
+
+const getNoteText = (req) => {
+    const text = req.body?.text || req.body?.note || "";
+    return typeof text === "string" ? text.trim() : "";
+};
+
 //getMedia('y42zyf3').then(console.log)
 DB.getDB.then((DB) => {
     /**
@@ -39,6 +54,302 @@ DB.getDB.then((DB) => {
 
     /**
  * @swagger
+ * /bible/mine:
+ *   get:
+ *     summary: Get Bible highlights and notes for the active profile
+ *     tags: [Bible]
+ *     security:
+ *       - cookieAuth: []
+ *     responses:
+ *       200:
+ *         description: OK
+ */
+    router.get("/mine", async (req, res) => {
+        const profileid = getProfileId(req);
+        const bibleData = await DB.getBibleProfileData(profileid);
+        if (!bibleData) return res.status(400).json({ status: "Invalid profile" });
+        return res.json({
+            status: "ok",
+            ...bibleData
+        });
+    });
+
+    /**
+ * @swagger
+ * /bible/verses/{verseKey}/counters:
+ *   get:
+ *     summary: Get highlight and note counters for a Bible verse
+ *     tags: [Bible]
+ *     security:
+ *       - cookieAuth: []
+ *     parameters:
+ *       - in: path
+ *         name: verseKey
+ *         required: true
+ *         schema:
+ *           type: string
+ *           example: GEN.1.1
+ *     responses:
+ *       200:
+ *         description: OK
+ */
+    router.get("/verses/:verseKey/counters", async (req, res) => {
+        const verseKey = normalizeVerseKey(req.params.verseKey);
+        if (!isValidVerseKey(verseKey)) {
+            return res.status(400).json({ status: "Invalid verse key" });
+        }
+
+        const counters = await DB.getBibleVerseCounters(verseKey);
+        return res.json({
+            status: "ok",
+            ...counters
+        });
+    });
+
+    /**
+ * @swagger
+ * /bible/verses/{verseKey}/activity:
+ *   get:
+ *     summary: Get profiles and recent notes for a Bible verse
+ *     tags: [Bible]
+ *     security:
+ *       - cookieAuth: []
+ *     parameters:
+ *       - in: path
+ *         name: verseKey
+ *         required: true
+ *         schema:
+ *           type: string
+ *           example: GEN.1.1
+ *       - in: query
+ *         name: limit
+ *         schema:
+ *           type: integer
+ *           default: 10
+ *     responses:
+ *       200:
+ *         description: OK
+ */
+    router.get("/verses/:verseKey/activity", async (req, res) => {
+        const verseKey = normalizeVerseKey(req.params.verseKey);
+        if (!isValidVerseKey(verseKey)) {
+            return res.status(400).json({ status: "Invalid verse key" });
+        }
+
+        const limit = Math.min(Math.max(parseInt(req.query.limit, 10) || 10, 1), 10);
+        const activity = await DB.getBibleVerseActivity(verseKey, limit);
+        return res.json({
+            status: "ok",
+            verseKey,
+            ...activity
+        });
+    });
+
+    /**
+ * @swagger
+ * /bible/verses/{verseKey}/highlight:
+ *   post:
+ *     summary: Highlight a Bible verse for the active profile
+ *     tags: [Bible]
+ *     security:
+ *       - cookieAuth: []
+ *     parameters:
+ *       - in: path
+ *         name: verseKey
+ *         required: true
+ *         schema:
+ *           type: string
+ *           example: GEN.1.1
+ *     responses:
+ *       200:
+ *         description: OK
+ */
+    router.post("/verses/:verseKey/highlight", async (req, res) => {
+        const profileid = getProfileId(req);
+        const verseKey = normalizeVerseKey(req.params.verseKey);
+        if (!isValidVerseKey(verseKey)) {
+            return res.status(400).json({ status: "Invalid verse key" });
+        }
+
+        const result = await DB.addBibleHighlight(profileid, verseKey);
+        if (!result) return res.status(400).json({ status: "Could not add highlight" });
+
+        const counters = await DB.getBibleVerseCounters(verseKey);
+        return res.json({
+            status: "ok",
+            highlighted: true,
+            ...counters
+        });
+    });
+
+    /**
+ * @swagger
+ * /bible/verses/{verseKey}/highlight:
+ *   delete:
+ *     summary: Remove a Bible verse highlight for the active profile
+ *     tags: [Bible]
+ *     security:
+ *       - cookieAuth: []
+ *     parameters:
+ *       - in: path
+ *         name: verseKey
+ *         required: true
+ *         schema:
+ *           type: string
+ *           example: GEN.1.1
+ *     responses:
+ *       200:
+ *         description: OK
+ */
+    router.delete("/verses/:verseKey/highlight", async (req, res) => {
+        const profileid = getProfileId(req);
+        const verseKey = normalizeVerseKey(req.params.verseKey);
+        if (!isValidVerseKey(verseKey)) {
+            return res.status(400).json({ status: "Invalid verse key" });
+        }
+
+        const result = await DB.removeBibleHighlight(profileid, verseKey);
+        if (!result) return res.status(400).json({ status: "Could not remove highlight" });
+
+        const counters = await DB.getBibleVerseCounters(verseKey);
+        return res.json({
+            status: "ok",
+            highlighted: false,
+            ...counters
+        });
+    });
+
+    /**
+ * @swagger
+ * /bible/verses/{verseKey}/notes:
+ *   post:
+ *     summary: Add a Bible verse note for the active profile
+ *     tags: [Bible]
+ *     security:
+ *       - cookieAuth: []
+ *     parameters:
+ *       - in: path
+ *         name: verseKey
+ *         required: true
+ *         schema:
+ *           type: string
+ *           example: GEN.1.1
+ *     responses:
+ *       200:
+ *         description: OK
+ */
+    router.post("/verses/:verseKey/notes", async (req, res) => {
+        const profileid = getProfileId(req);
+        const verseKey = normalizeVerseKey(req.params.verseKey);
+        const text = getNoteText(req);
+        if (!isValidVerseKey(verseKey)) {
+            return res.status(400).json({ status: "Invalid verse key" });
+        }
+        if (!text) return res.status(400).json({ status: "Note text is required" });
+
+        const note = await DB.addBibleNote(profileid, verseKey, text);
+        if (!note) return res.status(400).json({ status: "Could not add note" });
+
+        const counters = await DB.getBibleVerseCounters(verseKey);
+        return res.json({
+            status: "ok",
+            verseKey,
+            note,
+            counters
+        });
+    });
+
+    /**
+ * @swagger
+ * /bible/verses/{verseKey}/notes/{noteId}:
+ *   put:
+ *     summary: Update a Bible verse note for the active profile
+ *     tags: [Bible]
+ *     security:
+ *       - cookieAuth: []
+ *     parameters:
+ *       - in: path
+ *         name: verseKey
+ *         required: true
+ *         schema:
+ *           type: string
+ *           example: GEN.1.1
+ *       - in: path
+ *         name: noteId
+ *         required: true
+ *         schema:
+ *           type: string
+ *     responses:
+ *       200:
+ *         description: OK
+ */
+    router.put("/verses/:verseKey/notes/:noteId", async (req, res) => {
+        const profileid = getProfileId(req);
+        const verseKey = normalizeVerseKey(req.params.verseKey);
+        const noteId = req.params.noteId;
+        const text = getNoteText(req);
+        if (!isValidVerseKey(verseKey)) {
+            return res.status(400).json({ status: "Invalid verse key" });
+        }
+        if (!text) return res.status(400).json({ status: "Note text is required" });
+
+        const note = await DB.updateBibleNote(profileid, verseKey, noteId, text);
+        if (!note) return res.status(404).json({ status: "Note not found" });
+
+        return res.json({
+            status: "ok",
+            verseKey,
+            note
+        });
+    });
+
+    /**
+ * @swagger
+ * /bible/verses/{verseKey}/notes/{noteId}:
+ *   delete:
+ *     summary: Delete a Bible verse note for the active profile
+ *     tags: [Bible]
+ *     security:
+ *       - cookieAuth: []
+ *     parameters:
+ *       - in: path
+ *         name: verseKey
+ *         required: true
+ *         schema:
+ *           type: string
+ *           example: GEN.1.1
+ *       - in: path
+ *         name: noteId
+ *         required: true
+ *         schema:
+ *           type: string
+ *     responses:
+ *       200:
+ *         description: OK
+ */
+    router.delete("/verses/:verseKey/notes/:noteId", async (req, res) => {
+        const profileid = getProfileId(req);
+        const verseKey = normalizeVerseKey(req.params.verseKey);
+        const noteId = req.params.noteId;
+        if (!isValidVerseKey(verseKey)) {
+            return res.status(400).json({ status: "Invalid verse key" });
+        }
+
+        const result = await DB.removeBibleNote(profileid, verseKey, noteId);
+        if (!result) return res.status(400).json({ status: "Could not delete note" });
+        if (!result.modifiedCount) return res.status(404).json({ status: "Note not found" });
+
+        const counters = await DB.getBibleVerseCounters(verseKey);
+        return res.json({
+            status: "ok",
+            verseKey,
+            deleted: true,
+            counters
+        });
+    });
+
+/**
+ * @swagger
  * /bible/books:
  *   get:
  *     summary: Get the books of a Bible
@@ -148,7 +459,8 @@ DB.getDB.then((DB) => {
     router.get("/chapters/:chapterId", async (req, res) => {
         const chapterId = req.params.chapterId;
         const bibleId = req.query.bibleId || defaultBibleId;
-        const bibles = await fetchAPI('bibles/' + bibleId + "/chapters/" + chapterId);
+        const contentType = req.query['content-type'] ? `?content-type=${req.query['content-type']}` : '';
+        const bibles = await fetchAPI('bibles/' + bibleId + "/chapters/" + chapterId + contentType);
         return res.json(bibles);
     });
 

routes/chat.js

@@ -2,6 +2,7 @@ var express = require('express');
 var router = express.Router();
 
 const DB = require("../mongoDB.js");
+const Notifications = require("../notifications.js");
 const { getUserId, getProfileId } = require("../utils/sessionUtils.js");
 const { normalizeLanguageCode, translateText } = require("../utils/chatTranslation.js");
 
@@ -190,6 +191,8 @@ DB.getDB.then((DB) => {
                 return res.status(500).json({ status: "Could not save message" });
             }
 
+            Notifications.youGotANewChatMessage(profileId, text);
+
             activeUsers.set(profileId + "", {
                 profileId: profileId + "",
                 userId: userId + "",

routes/liveCaptions.js

@@ -0,0 +1,247 @@
+var express = require('express');
+var router = express.Router();
+const { rateLimit } = require("express-rate-limit");
+
+const sessionChecker = require("../middleware/sessionChecker.js");
+
+const MAX_BUFFER_SIZE = 300;
+const DEFAULT_INITIAL_LIMIT = 40;
+const MAX_INITIAL_LIMIT = 120;
+const INACTIVITY_RESET_MS = 10 * 60 * 1000;
+const CAPTION_META_KEYS = new Set(["sequence", "createdAt", "original", "draft", "sourceLang", "lang", "isDraft", "status", "translations"]);
+
+const liveCaptionState = {
+    startedAt: Date.now(),
+    lastIngestAt: 0,
+    latestSequence: 0,
+    captions: [],
+};
+
+const liveCaptionsLimiter = rateLimit({
+    windowMs: 10 * 60 * 1000,
+    limit: 6000,
+    standardHeaders: "draft-8",
+    legacyHeaders: false,
+    keyGenerator: (req) => {
+        const forwarded = req.headers["x-forwarded-for"]?.split(",")[0];
+        const ip = forwarded || req.ip || "";
+        return ip.includes(":") ? ip.split(":")[0] : ip;
+    },
+});
+
+router.use(liveCaptionsLimiter);
+
+const normalizeLang = (lang = "") => {
+    const value = String(lang || "").trim().toLowerCase();
+    if (!value) return "";
+    const base = value.split(",")[0].split("-")[0].trim();
+    return base || value;
+};
+
+const normalizeTranslations = (translations) => {
+    if (!translations || typeof translations !== "object" || Array.isArray(translations)) return {};
+    const normalized = {};
+    for (const [langKey, translatedText] of Object.entries(translations)) {
+        const lang = normalizeLang(langKey);
+        const text = typeof translatedText === "string" ? translatedText.trim() : "";
+        if (!lang || !text) continue;
+        normalized[lang] = text;
+    }
+    return normalized;
+};
+
+const readText = (value) => {
+    if (typeof value === "string") return value.trim();
+    return "";
+};
+
+const extractDraftText = (body = {}) => {
+    const directDraft = readText(body?.draft);
+    if (directDraft) return directDraft;
+    const nestedDraft = readText(body?.draft?.text);
+    if (nestedDraft) return nestedDraft;
+    const fallbackText = readText(body?.text);
+    if (fallbackText) return fallbackText;
+    return "";
+};
+
+const buildTranslationsFromFlatPayload = (payload) => {
+    const ignoredKeys = new Set(["original", "draft", "sourceLang", "lang", "isDraft", "status", "translations"]);
+    const normalized = {};
+    for (const [key, value] of Object.entries(payload || {})) {
+        if (ignoredKeys.has(key)) continue;
+        const lang = normalizeLang(key);
+        const text = typeof value === "string" ? value.trim() : "";
+        if (!lang || !text) continue;
+        normalized[lang] = text;
+    }
+    return normalized;
+};
+
+const inferSourceLangFromTranslations = (original, translations) => {
+    const normalizedOriginal = String(original || "").trim();
+    if (!normalizedOriginal) return "original";
+    for (const [lang, text] of Object.entries(translations || {})) {
+        if (String(text || "").trim() === normalizedOriginal) return normalizeLang(lang);
+    }
+    return "original";
+};
+
+const getAvailableLanguages = () => {
+    const langs = new Set();
+    for (const caption of liveCaptionState.captions) {
+        Object.keys(caption || {}).forEach((key) => {
+            if (CAPTION_META_KEYS.has(key)) return;
+            const lang = normalizeLang(key);
+            if (lang) langs.add(lang);
+        });
+    }
+    return Array.from(langs).filter(Boolean).sort();
+};
+
+const resetLiveCaptionState = () => {
+    liveCaptionState.startedAt = Date.now();
+    liveCaptionState.lastIngestAt = 0;
+    liveCaptionState.latestSequence = 0;
+    liveCaptionState.captions = [];
+};
+
+const maybeResetForInactivity = () => {
+    if (!liveCaptionState.lastIngestAt) return;
+    if ((Date.now() - liveCaptionState.lastIngestAt) < INACTIVITY_RESET_MS) return;
+    resetLiveCaptionState();
+};
+
+router.get("/stream", async (req, res) => {
+    try {
+        maybeResetForInactivity();
+        const sinceSequence = Number.parseInt(req.query?.sinceSequence, 10);
+        const requestedLimit = Number.parseInt(req.query?.limit, 10);
+        const initialLimit = Number.isFinite(requestedLimit)
+            ? Math.max(1, Math.min(requestedLimit, MAX_INITIAL_LIMIT))
+            : DEFAULT_INITIAL_LIMIT;
+
+        let captions = [];
+        if (Number.isFinite(sinceSequence) && sinceSequence >= 0) {
+            captions = liveCaptionState.captions.filter((item) => item.sequence > sinceSequence);
+        } else {
+            captions = liveCaptionState.captions.slice(-initialLimit);
+        }
+
+        return res.json({
+            status: "ok",
+            latestSequence: liveCaptionState.latestSequence,
+            startedAt: new Date(liveCaptionState.startedAt).toISOString(),
+            availableLanguages: getAvailableLanguages(),
+            captions,
+        });
+    } catch (error) {
+        console.error("Error getting live captions stream", error);
+        return res.status(500).json({
+            status: "Internal server error",
+            latestSequence: liveCaptionState.latestSequence,
+            captions: [],
+            availableLanguages: [],
+        });
+    }
+});
+
+router.post("/ingest", async (req, res) => {
+    try {
+        // TODO: Add basic auth/API key validation before production roll-out.
+        const draft = extractDraftText(req.body || {});
+        const originalFromPayload = readText(req.body?.original);
+        const original = originalFromPayload || draft;
+        const requestedLang = normalizeLang(req.body?.lang);
+        const sourceLangFromRequest = normalizeLang(req.body?.sourceLang || (requestedLang && requestedLang !== "draft" ? requestedLang : ""));
+        const isDraft = !!draft || requestedLang === "draft" || sourceLangFromRequest === "draft" || req.body?.isDraft === true || req.body?.status === "draft";
+        const mapFromNested = normalizeTranslations(req.body?.translations);
+        const mapFromFlat = buildTranslationsFromFlatPayload(req.body);
+        const translations = isDraft ? {} : { ...mapFromNested, ...mapFromFlat };
+        const inferredSource = inferSourceLangFromTranslations(original, translations);
+        const sourceLang = isDraft ? "" : (sourceLangFromRequest || inferredSource);
+
+        if (!original) {
+            return res.status(400).json({ status: "Original text is required" });
+        }
+        if (sourceLang && sourceLang !== "original" && sourceLang !== "draft" && !translations[sourceLang]) {
+            translations[sourceLang] = original;
+        }
+
+        const sequence = liveCaptionState.latestSequence + 1;
+        const caption = {
+            sequence,
+            createdAt: new Date().toISOString(),
+            original,
+            sourceLang: sourceLang || undefined,
+            lang: isDraft ? "draft" : (sourceLang || undefined),
+            isDraft,
+            status: isDraft ? "draft" : "final",
+            ...translations,
+        };
+
+        liveCaptionState.latestSequence = sequence;
+        liveCaptionState.lastIngestAt = Date.now();
+        liveCaptionState.captions.push(caption);
+        if (liveCaptionState.captions.length > MAX_BUFFER_SIZE) {
+            liveCaptionState.captions.splice(0, liveCaptionState.captions.length - MAX_BUFFER_SIZE);
+        }
+
+        return res.json({
+            status: "ok",
+            caption,
+            latestSequence: liveCaptionState.latestSequence,
+            availableLanguages: getAvailableLanguages(),
+        });
+    } catch (error) {
+        console.error("Error ingesting live captions", error);
+        return res.status(500).json({ status: "Internal server error" });
+    }
+});
+
+router.post("/reset", async (_, res) => {
+    try {
+        // TODO: Add admin authorization before exposing this endpoint.
+        resetLiveCaptionState();
+        return res.json({ status: "ok" });
+    } catch (error) {
+        console.error("Error resetting live captions state", error);
+        return res.status(500).json({ status: "Internal server error" });
+    }
+});
+
+router.get("/public/stream", async (req, res) => {
+    try {
+        maybeResetForInactivity();
+        const sinceSequence = Number.parseInt(req.query?.sinceSequence, 10);
+        const requestedLimit = Number.parseInt(req.query?.limit, 10);
+        const initialLimit = Number.isFinite(requestedLimit)
+            ? Math.max(1, Math.min(requestedLimit, MAX_INITIAL_LIMIT))
+            : DEFAULT_INITIAL_LIMIT;
+
+        let captions = [];
+        if (Number.isFinite(sinceSequence) && sinceSequence >= 0) {
+            captions = liveCaptionState.captions.filter((item) => item.sequence > sinceSequence);
+        } else {
+            captions = liveCaptionState.captions.slice(-initialLimit);
+        }
+
+        return res.json({
+            status: "ok",
+            latestSequence: liveCaptionState.latestSequence,
+            startedAt: new Date(liveCaptionState.startedAt).toISOString(),
+            availableLanguages: getAvailableLanguages(),
+            captions,
+        });
+    } catch (error) {
+        console.error("Error getting public live captions stream", error);
+        return res.status(500).json({
+            status: "Internal server error",
+            latestSequence: liveCaptionState.latestSequence,
+            captions: [],
+            availableLanguages: [],
+        });
+    }
+});
+
+module.exports = router;

routes/post.js

@@ -4,6 +4,7 @@ var router = express.Router();
 const DB = require("./../mongoDB.js");
 const Post = require("./../def/post.js");
 const Notifications = require("./../notifications.js");
+const { translateText, normalizeLanguageCode } = require("../utils/chatTranslation.js");
 
 DB.getDB.then((DB) => {
 
@@ -481,6 +482,47 @@ DB.getDB.then((DB) => {
         })
     });
 
+    router.post("/translate", async (req, res) => {
+        let postid = req.body.postid;
+        let targetLang = normalizeLanguageCode(req.body.targetLang);
+        
+        // Return ack immediately
+        res.json({ status: "ok", message: "Translation queued" });
+
+        if (!postid || !targetLang) return;
+
+        try {
+            // Get post
+            const posts = await DB.getPostsByTag('', null); // No good way to get one post by ID directly exposed? 
+            // Let's use dbCols directly if needed or find it. Wait, how do we get a single post?
+            // I'll assume DB.getPost exists, let me check that later. Actually I will use DB.postCols directly.
+            const post = await DB.postCols.findOne({ _id: DB.ObjectID(postid) });
+            if (!post || !post.content) return;
+
+            // Strip inline tags and bible tags before translating to reduce token usage and confusion,
+            // or just translate the raw content and let the AI handle it? The chat translator prompt says:
+            // "You translate chat messages. Keep meaning, tone, emojis, names, and references. Return only the translated text."
+            // So it can handle tags.
+
+            // To avoid huge translations or mostly-media posts
+            if (post.content.length > 1000) return;
+
+            if (post.translations && post.translations[targetLang]) return;
+
+            const translation = await translateText({
+                text: post.content,
+                sourceLang: "auto",
+                targetLang: targetLang
+            });
+
+            if (translation && translation.translatedText) {
+                await DB.addTranslation(postid, targetLang, translation.translatedText);
+            }
+        } catch (error) {
+            console.error("Error in background post translation", error);
+        }
+    });
+
     /**
  * @swagger
  * /post/react:

scripts/liveCaptionsTestSender.js

@@ -0,0 +1,89 @@
+#!/usr/bin/env node
+
+require("dotenv").config();
+const axios = require("axios");
+
+const baseUrl = (process.env.CAPTION_TEST_BASE_URL || process.env.BASE_URL || "http://localhost:3000").replace(/\/+$/, "");
+const ingestUrl = `${baseUrl}/live-captions/ingest`;
+const intervalMs = 6000;
+
+const samples = [
+    {
+        original: "Bienvenidos a nuestro servicio de adoracion.",
+        es: "Bienvenidos a nuestro servicio de adoracion.",
+        en: "Welcome to our worship service.",
+        fr: "Bienvenue a notre service de louange.",
+    },
+    {
+        original: "Leamos juntos en el Salmo 23.",
+        es: "Leamos juntos en el Salmo 23.",
+        en: "Let us read together in Psalm 23.",
+        fr: "Lisons ensemble le Psaume 23.",
+    },
+    {
+        original: "Dios es fiel en todo tiempo.",
+        es: "Dios es fiel en todo tiempo.",
+        en: "God is faithful at all times.",
+        fr: "Dieu est fidele en tout temps.",
+    },
+    {
+        original: "Tomemos un momento para orar.",
+        es: "Tomemos un momento para orar.",
+        en: "Let us take a moment to pray.",
+        fr: "Prenons un moment pour prier.",
+    },
+];
+
+let sampleIndex = 0;
+let timer = null;
+
+const postPayload = async (payload) => {
+    const kind = payload?.draft ? "draft" : "final";
+
+    try {
+        const response = await axios.post(ingestUrl, payload, {
+            headers: { "Content-Type": "application/json" },
+            timeout: 10000,
+        });
+        const seq = response?.data?.caption?.sequence || response?.data?.latestSequence || "?";
+        const text = payload?.draft || payload?.original || "";
+        console.log(`[live-captions:test-sender] sent ${kind} sequence=${seq} text="${text}"`);
+    } catch (error) {
+        const status = error?.response?.status;
+        const body = error?.response?.data;
+        const message = error?.message || "request failed";
+        console.error("[live-captions:test-sender] send failed", { status, body, message });
+    }
+};
+
+const sendNextSample = async () => {
+    const payload = samples[sampleIndex];
+
+    const draftWords = String(payload?.original || "").split(" ").filter(Boolean);
+    if (draftWords.length > 2) {
+        await postPayload({ draft: draftWords.slice(0, 2).join(" ") });
+        await new Promise((resolve) => setTimeout(resolve, 550));
+        await postPayload({ draft: draftWords.slice(0, 4).join(" ") });
+        await new Promise((resolve) => setTimeout(resolve, 550));
+    }
+
+    await postPayload(payload);
+    sampleIndex = (sampleIndex + 1) % samples.length;
+};
+
+const start = async () => {
+    console.log(`[live-captions:test-sender] posting to ${ingestUrl} every ${intervalMs / 1000}s`);
+    await sendNextSample();
+    timer = setInterval(sendNextSample, intervalMs);
+};
+
+const shutdown = () => {
+    if (timer) clearInterval(timer);
+    console.log("[live-captions:test-sender] stopped");
+    process.exit(0);
+};
+
+process.on("SIGINT", shutdown);
+process.on("SIGTERM", shutdown);
+
+start();

translation-service/.dockerignore

@@ -0,0 +1,3 @@
+.venv/
+__pycache__/
+*.pyc

translation-service/Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+COPY server.py ./
+
+ENV MARIAN_HOST=0.0.0.0
+ENV TRANSFORMERS_CACHE=/models
+VOLUME ["/models"]
+EXPOSE 8000
+
+CMD ["python", "server.py"]

translation-service/requirements.txt

@@ -0,0 +1,4 @@
+torch>=2.2,<3
+transformers>=4.40,<5
+sentencepiece>=0.2,<1
+langdetect>=1.0.9,<2

translation-service/server.py

@@ -0,0 +1,118 @@
+import json
+import os
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+from langdetect import DetectorFactory, LangDetectException, detect
+from transformers import MarianMTModel, MarianTokenizer
+
+DetectorFactory.seed = 0
+
+HOST = os.getenv("MARIAN_HOST", "127.0.0.1")
+PORT = int(os.getenv("MARIAN_PORT", "8000"))
+MAX_INPUT_LENGTH = int(os.getenv("MARIAN_MAX_INPUT_LENGTH", "1000"))
+DEFAULT_SOURCE_LANGUAGE = os.getenv("MARIAN_DEFAULT_SOURCE_LANGUAGE", "en")
+SUPPORTED_LANGUAGES = {"en", "es", "fr", "da", "ar"}
+MODEL_CACHE = {}
+
+
+def normalize_language(value):
+    language = str(value or "").strip().lower().split(",")[0].split("-")[0]
+    return language
+
+
+def detect_source_language(text):
+    try:
+        detected = normalize_language(detect(text))
+        if detected in SUPPORTED_LANGUAGES:
+            return detected
+    except LangDetectException:
+        pass
+    return DEFAULT_SOURCE_LANGUAGE
+
+
+def get_model(source, target):
+    model_name = f"Helsinki-NLP/opus-mt-{source}-{target}"
+    if model_name not in MODEL_CACHE:
+        MODEL_CACHE[model_name] = (
+            MarianTokenizer.from_pretrained(model_name),
+            MarianMTModel.from_pretrained(model_name),
+        )
+    return model_name, MODEL_CACHE[model_name]
+
+
+def translate_once(text, source, target):
+    model_name, (tokenizer, model) = get_model(source, target)
+    encoded = tokenizer([text], return_tensors="pt", truncation=True)
+    generated = model.generate(**encoded)
+    return tokenizer.batch_decode(generated, skip_special_tokens=True)[0], model_name
+
+
+def translate(text, source, target):
+    if source == "auto":
+        source = detect_source_language(text)
+    if source not in SUPPORTED_LANGUAGES or target not in SUPPORTED_LANGUAGES:
+        raise ValueError("Only en, es, fr, da, and ar are supported")
+    if source == target:
+        return text, source, "none"
+    if source == "en" or target == "en":
+        translated, model_name = translate_once(text, source, target)
+        return translated, source, model_name
+
+    english, first_model = translate_once(text, source, "en")
+    translated, second_model = translate_once(english, "en", target)
+    return translated, source, f"{first_model},{second_model}"
+
+
+class TranslationHandler(BaseHTTPRequestHandler):
+    def send_json(self, status, body):
+        payload = json.dumps(body).encode("utf-8")
+        self.send_response(status)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(payload)))
+        self.end_headers()
+        self.wfile.write(payload)
+
+    def do_GET(self):
+        if self.path != "/health":
+            self.send_json(404, {"status": "not found"})
+            return
+        self.send_json(200, {"status": "ok", "provider": "marianmt", "loadedModels": list(MODEL_CACHE)})
+
+    def do_POST(self):
+        if self.path != "/translate":
+            self.send_json(404, {"status": "not found"})
+            return
+        try:
+            content_length = int(self.headers.get("Content-Length", "0"))
+            body = json.loads(self.rfile.read(content_length).decode("utf-8"))
+            text = str(body.get("text") or "").strip()
+            source = normalize_language(body.get("sourceLang")) or "auto"
+            target = normalize_language(body.get("targetLang"))
+            if not text or not target:
+                self.send_json(400, {"status": "text and targetLang are required"})
+                return
+            if len(text) > MAX_INPUT_LENGTH:
+                self.send_json(400, {"status": f"text exceeds {MAX_INPUT_LENGTH} characters"})
+                return
+            translated, detected_source, model_name = translate(text, source, target)
+            self.send_json(200, {
+                "status": "ok",
+                "translatedText": translated,
+                "sourceLang": detected_source,
+                "targetLang": target,
+                "provider": "marianmt",
+                "model": model_name,
+            })
+        except (ValueError, json.JSONDecodeError) as error:
+            self.send_json(400, {"status": str(error)})
+        except Exception as error:
+            print(f"Translation failed: {error}", flush=True)
+            self.send_json(502, {"status": "Translation failed"})
+
+    def log_message(self, format_string, *args):
+        print(f"[marianmt] {self.address_string()} {format_string % args}", flush=True)
+
+
+if __name__ == "__main__":
+    print(f"MarianMT translation service listening on {HOST}:{PORT}", flush=True)
+    ThreadingHTTPServer((HOST, PORT), TranslationHandler).serve_forever()

utils/chatTranslation.js

@@ -1,6 +1,8 @@
 const axios = require("axios");
 
 const DEFAULT_MODEL = process.env.OPENAI_TRANSLATION_MODEL || process.env.OPENAI_MODEL || "gpt-4o-mini";
+const TRANSLATION_PROVIDER = (process.env.TRANSLATION_PROVIDER || "openai").trim().toLowerCase();
+const MARIAN_TRANSLATION_URL = (process.env.MARIAN_TRANSLATION_URL || "http://127.0.0.1:8000").replace(/\/$/, "");
 
 const normalizeLanguageCode = (rawLanguage) => {
     if (!rawLanguage || typeof rawLanguage !== "string") return "en";
@@ -40,6 +42,31 @@ const translateText = async ({ text, sourceLang, targetLang }) => {
         };
     }
 
+    if (TRANSLATION_PROVIDER === "marian") {
+        try {
+            const response = await axios.post(
+                `${MARIAN_TRANSLATION_URL}/translate`,
+                { text, sourceLang: sourceLang || "auto", targetLang: normalizedTarget },
+                { timeout: 30000, headers: { "Content-Type": "application/json" } }
+            );
+            const translatedText = response?.data?.translatedText?.trim();
+            if (!translatedText) return null;
+            return {
+                translatedText,
+                provider: response.data.provider || "marianmt",
+                model: response.data.model || "unknown",
+            };
+        } catch (error) {
+            console.error("Error translating with MarianMT", error?.response?.data || error?.message || error);
+            return null;
+        }
+    }
+
+    if (TRANSLATION_PROVIDER !== "openai") {
+        console.error(`Unsupported translation provider: ${TRANSLATION_PROVIDER}`);
+        return null;
+    }
+
     const apiKey = process.env.OPENAI_API_KEY;
     if (!apiKey) return null;
 
@@ -54,7 +81,7 @@ const translateText = async ({ text, sourceLang, targetLang }) => {
                         content: [
                             {
                                 type: "input_text",
-                                text: "You translate chat messages. Keep meaning, tone, emojis, names, and references. Return only the translated text.",
+                                text: "You translate chat messages and posts. Keep meaning, tone, emojis, names, and references. Do not translate structural tags starting with @ (e.g. @image:..., @youtube:..., @bible:...). Leave them exactly as they are or omit them if they do not fit the text flow. Return only the translated text.",
                             },
                         ],
                     },