chore(prod): preserve production compose settings

.gitignore

@@ -8,6 +8,7 @@ lerna-debug.log*
 
 # Dumps
 dump
+backups/
 
 
 # Diagnostic reports (https://nodejs.org/api/report.html)
@@ -118,6 +119,10 @@ dist
 .yarn/build-state.yml
 .yarn/install-state.gz
 .pnp.*
+
+# Local Python environments
+translation-service/.venv/
+translation-service/__pycache__/
 # Logs
 logs
 *.log

README.md

@@ -31,6 +31,27 @@ A step by step series of examples that tell you how to get a development env run
     npm start
     ```
 
+### Local MarianMT translation
+
+The translation service is independent from the Node backend and provides `GET /health` and `POST /translate`.
+
+1. Start it locally:
+   ```
+   cd translation-service
+   python3 -m venv .venv
+   .venv/bin/pip install -r requirements.txt
+   .venv/bin/python server.py
+   ```
+2. Configure the Node backend:
+   ```
+   TRANSLATION_PROVIDER=marian
+   MARIAN_TRANSLATION_URL=http://127.0.0.1:8000
+   ```
+
+With Docker Compose, the service runs as the internal `translation` service. Set `TRANSLATION_PROVIDER=marian` before running `docker compose up`. MarianMT models download only when first needed and are stored in the `translation-models` Docker volume.
+
+Supported languages are English (`en`), Spanish (`es`), French (`fr`), Danish (`da`), and Arabic (`ar`). Non-English pairs translate through English. Keep the service on the internal Docker network; it has no public port mapping.
+
 ### API Documentation
 
 Once the server is running, you can access the interactive API documentation powered by Swagger UI at:

docker-compose.yml

@@ -18,15 +18,34 @@ services:
       - WEB_PUSH_EMAIL=${WEB_PUSH_EMAIL}
       - EMAILPASS=${EMAILPASS}
       - PORT=3001
+      - COOKIE_SECURE=true
+      - NODE_ENV=production
+      - TRANSLATION_PROVIDER=${TRANSLATION_PROVIDER:-openai}
+      - MARIAN_TRANSLATION_URL=http://translation:8000
     volumes:
       - .:/app
       - '/app/node_modules'
     #depends_on:
     #  - mongo
     command: node index.js
+    depends_on:
+      - translation
+    networks:
+      - emi-network
   #  networks:
   #    - emi-network
 
+  translation:
+    build:
+      context: ./translation-service
+    restart: unless-stopped
+    environment:
+      - MARIAN_HOST=0.0.0.0
+    volumes:
+      - translation-models:/models
+    networks:
+      - emi-network
+
   #mongo:
   #  image: mongo:latest
   #  ports:
@@ -38,6 +57,8 @@ services:
   #    - ./dump:/dump
     #entrypoint: mongodump ${MONGO_URL} && mongorestore --db EMI_SOCIAL dump/EMI_SOCIAL/ && mongod
 
-#volumes:
+volumes:
+  translation-models:
+    driver: local
   #mongodbdata:
   #  driver: local  # This ensures the volume is created

scripts/monthlyMongoBackup.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+APP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+BACKUP_ROOT="${BACKUP_ROOT:-$APP_DIR/backups}"
+DB_NAME="${DB_NAME:-EMI_SOCIAL}"
+MONGO_IMAGE="${MONGO_BACKUP_IMAGE:-mongo:7}"
+KEEP_ARCHIVES="${KEEP_ARCHIVES:-12}"
+
+timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
+dump_name="emi_social_${timestamp}"
+archive_path="$BACKUP_ROOT/${dump_name}.tar.gz"
+
+mkdir -p "$BACKUP_ROOT"
+
+docker run --rm \
+  --user "$(id -u):$(id -g)" \
+  --env-file "$APP_DIR/.env" \
+  -e DB_NAME="$DB_NAME" \
+  -e DUMP_NAME="$dump_name" \
+  -v "$BACKUP_ROOT:/backup" \
+  "$MONGO_IMAGE" \
+  bash -lc '
+    set -euo pipefail
+    : "${MONGO_URL:?MONGO_URL is missing}"
+
+    uri="$MONGO_URL"
+    uri="${uri/\/myFirstDatabase?/\/${DB_NAME}?}"
+    uri="${uri/\/myFirstDatabase$/\/${DB_NAME}}"
+
+    mongodump --uri "$uri" --db "$DB_NAME" --out "/backup/$DUMP_NAME"
+  '
+
+tar -czf "$archive_path" -C "$BACKUP_ROOT" "$dump_name"
+rm -rf "$BACKUP_ROOT/$dump_name"
+
+find "$BACKUP_ROOT" -maxdepth 1 -name 'emi_social_*.tar.gz' -type f \
+  -printf '%T@ %p\n' \
+  | sort -rn \
+  | awk -v keep="$KEEP_ARCHIVES" 'NR > keep { print $2 }' \
+  | xargs -r rm -f
+
+echo "Created MongoDB backup: $archive_path"

translation-service/.dockerignore

@@ -0,0 +1,3 @@
+.venv/
+__pycache__/
+*.pyc

translation-service/Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+COPY server.py ./
+
+ENV MARIAN_HOST=0.0.0.0
+ENV TRANSFORMERS_CACHE=/models
+VOLUME ["/models"]
+EXPOSE 8000
+
+CMD ["python", "server.py"]

translation-service/requirements.txt

@@ -0,0 +1,4 @@
+torch>=2.2,<3
+transformers>=4.40,<5
+sentencepiece>=0.2,<1
+langdetect>=1.0.9,<2

translation-service/server.py

@@ -0,0 +1,118 @@
+import json
+import os
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+from langdetect import DetectorFactory, LangDetectException, detect
+from transformers import MarianMTModel, MarianTokenizer
+
+DetectorFactory.seed = 0
+
+HOST = os.getenv("MARIAN_HOST", "127.0.0.1")
+PORT = int(os.getenv("MARIAN_PORT", "8000"))
+MAX_INPUT_LENGTH = int(os.getenv("MARIAN_MAX_INPUT_LENGTH", "1000"))
+DEFAULT_SOURCE_LANGUAGE = os.getenv("MARIAN_DEFAULT_SOURCE_LANGUAGE", "en")
+SUPPORTED_LANGUAGES = {"en", "es", "fr", "da", "ar"}
+MODEL_CACHE = {}
+
+
+def normalize_language(value):
+    language = str(value or "").strip().lower().split(",")[0].split("-")[0]
+    return language
+
+
+def detect_source_language(text):
+    try:
+        detected = normalize_language(detect(text))
+        if detected in SUPPORTED_LANGUAGES:
+            return detected
+    except LangDetectException:
+        pass
+    return DEFAULT_SOURCE_LANGUAGE
+
+
+def get_model(source, target):
+    model_name = f"Helsinki-NLP/opus-mt-{source}-{target}"
+    if model_name not in MODEL_CACHE:
+        MODEL_CACHE[model_name] = (
+            MarianTokenizer.from_pretrained(model_name),
+            MarianMTModel.from_pretrained(model_name),
+        )
+    return model_name, MODEL_CACHE[model_name]
+
+
+def translate_once(text, source, target):
+    model_name, (tokenizer, model) = get_model(source, target)
+    encoded = tokenizer([text], return_tensors="pt", truncation=True)
+    generated = model.generate(**encoded)
+    return tokenizer.batch_decode(generated, skip_special_tokens=True)[0], model_name
+
+
+def translate(text, source, target):
+    if source == "auto":
+        source = detect_source_language(text)
+    if source not in SUPPORTED_LANGUAGES or target not in SUPPORTED_LANGUAGES:
+        raise ValueError("Only en, es, fr, da, and ar are supported")
+    if source == target:
+        return text, source, "none"
+    if source == "en" or target == "en":
+        translated, model_name = translate_once(text, source, target)
+        return translated, source, model_name
+
+    english, first_model = translate_once(text, source, "en")
+    translated, second_model = translate_once(english, "en", target)
+    return translated, source, f"{first_model},{second_model}"
+
+
+class TranslationHandler(BaseHTTPRequestHandler):
+    def send_json(self, status, body):
+        payload = json.dumps(body).encode("utf-8")
+        self.send_response(status)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(payload)))
+        self.end_headers()
+        self.wfile.write(payload)
+
+    def do_GET(self):
+        if self.path != "/health":
+            self.send_json(404, {"status": "not found"})
+            return
+        self.send_json(200, {"status": "ok", "provider": "marianmt", "loadedModels": list(MODEL_CACHE)})
+
+    def do_POST(self):
+        if self.path != "/translate":
+            self.send_json(404, {"status": "not found"})
+            return
+        try:
+            content_length = int(self.headers.get("Content-Length", "0"))
+            body = json.loads(self.rfile.read(content_length).decode("utf-8"))
+            text = str(body.get("text") or "").strip()
+            source = normalize_language(body.get("sourceLang")) or "auto"
+            target = normalize_language(body.get("targetLang"))
+            if not text or not target:
+                self.send_json(400, {"status": "text and targetLang are required"})
+                return
+            if len(text) > MAX_INPUT_LENGTH:
+                self.send_json(400, {"status": f"text exceeds {MAX_INPUT_LENGTH} characters"})
+                return
+            translated, detected_source, model_name = translate(text, source, target)
+            self.send_json(200, {
+                "status": "ok",
+                "translatedText": translated,
+                "sourceLang": detected_source,
+                "targetLang": target,
+                "provider": "marianmt",
+                "model": model_name,
+            })
+        except (ValueError, json.JSONDecodeError) as error:
+            self.send_json(400, {"status": str(error)})
+        except Exception as error:
+            print(f"Translation failed: {error}", flush=True)
+            self.send_json(502, {"status": "Translation failed"})
+
+    def log_message(self, format_string, *args):
+        print(f"[marianmt] {self.address_string()} {format_string % args}", flush=True)
+
+
+if __name__ == "__main__":
+    print(f"MarianMT translation service listening on {HOST}:{PORT}", flush=True)
+    ThreadingHTTPServer((HOST, PORT), TranslationHandler).serve_forever()

update.sh

@@ -0,0 +1,3 @@
+git pull
+docker compose down
+docker compose up -d

utils/chatTranslation.js

@@ -1,6 +1,8 @@
 const axios = require("axios");
 
 const DEFAULT_MODEL = process.env.OPENAI_TRANSLATION_MODEL || process.env.OPENAI_MODEL || "gpt-4o-mini";
+const TRANSLATION_PROVIDER = (process.env.TRANSLATION_PROVIDER || "openai").trim().toLowerCase();
+const MARIAN_TRANSLATION_URL = (process.env.MARIAN_TRANSLATION_URL || "http://127.0.0.1:8000").replace(/\/$/, "");
 
 const normalizeLanguageCode = (rawLanguage) => {
     if (!rawLanguage || typeof rawLanguage !== "string") return "en";
@@ -40,6 +42,31 @@ const translateText = async ({ text, sourceLang, targetLang }) => {
         };
     }
 
+    if (TRANSLATION_PROVIDER === "marian") {
+        try {
+            const response = await axios.post(
+                `${MARIAN_TRANSLATION_URL}/translate`,
+                { text, sourceLang: sourceLang || "auto", targetLang: normalizedTarget },
+                { timeout: 30000, headers: { "Content-Type": "application/json" } }
+            );
+            const translatedText = response?.data?.translatedText?.trim();
+            if (!translatedText) return null;
+            return {
+                translatedText,
+                provider: response.data.provider || "marianmt",
+                model: response.data.model || "unknown",
+            };
+        } catch (error) {
+            console.error("Error translating with MarianMT", error?.response?.data || error?.message || error);
+            return null;
+        }
+    }
+
+    if (TRANSLATION_PROVIDER !== "openai") {
+        console.error(`Unsupported translation provider: ${TRANSLATION_PROVIDER}`);
+        return null;
+    }
+
     const apiKey = process.env.OPENAI_API_KEY;
     if (!apiKey) return null;