Complete history.md with Phase 9: Server Ingest

.gitignore

@@ -1 +1,2 @@
 __pycache__/\n*.pyc\n.DS_Store
+build/\ndist/\n*.spec

build.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Build Script for whisper-transcribe
+# This script bundles the project into a single executable for Apple Silicon.
+
+echo "--- Starting Build Process ---"
+
+# Ensure PyInstaller is installed
+if ! command -v pyinstaller &> /dev/null
+then
+    echo "PyInstaller not found. Installing..."
+    pip install pyinstaller
+fi
+
+# Run PyInstaller
+# --onefile: Bundle into a single executable
+# --collect-all: Ensure all sub-dependencies of heavy libraries are included
+# --hidden-import: Explicitly include libraries that might be missed by static analysis
+# Added --collect-all mlx and --hidden-import mlx._reprlib_fix to solve the runtime error
+pyinstaller --onefile \
+    --name whisper-transcribe \
+    --collect-all mlx \
+    --collect-all mlx_whisper \
+    --collect-all transformers \
+    --collect-all silero_vad \
+    --collect-all torch \
+    --hidden-import=mlx._reprlib_fix \
+    --hidden-import=sacremoses \
+    --hidden-import=joblib \
+    transcribe.py
+
+echo "--- Build Complete ---"
+echo "Binary location: dist/whisper-transcribe"

history.md

@@ -52,3 +52,31 @@ The project now features a high-performance, Apple Silicon-optimized pipeline th
 - **Solution:** 
     - Artificially truncated input transcription to a maximum of 250 characters.
     - Added `max_new_tokens=150` to the translation generation call to ensure the model terminates even if it gets stuck in a loop.
+
+## Phase 7: Multilingual Detection & Bridge Translation
+- **Goal:** Support input in any language, detect it, and translate to English + others.
+- **Approach:** 
+    - Switched to `whisper-small-mlx` (multilingual).
+    - **Hub-and-Spoke Model:** If a non-English language is detected, Whisper's `task="translate"` is used to create an English "bridge" text, which is then fed into the specialized MarianMT models.
+- **Outcome:** Full support for multilingual input with centralized translation.
+
+## Phase 8: Compilation to Binary
+- **Goal:** Distribute the script as a single, standalone executable for macOS terminal.
+- **Tool:** `PyInstaller`.
+- **Process:** 
+    - Used `--onefile` to bundle the entire Python runtime and its heavy dependencies (Torch, MLX, Transformers).
+    - Excluded build artifacts (`build/`, `dist/`, `.spec`) from the repository.
+- **Build Script:** 
+    ```bash
+    chmod +x build.sh
+    ./build.sh
+    ```
+- **Troubleshooting:** Fixed a runtime `ModuleNotFoundError: No module named 'mlx._reprlib_fix'` by explicitly adding `--collect-all mlx` and `--hidden-import=mlx._reprlib_fix` to the PyInstaller configuration. Also added `multiprocessing.freeze_support()` to fix infinite loops in the compiled binary.
+
+## Phase 9: Real-Time Server Ingest
+- **Goal:** Send live captions and translations to a central ingest server.
+- **Backend:** `https://emiapi.reynafamily.com/live-captions/ingest`.
+- **Approach:** 
+    - Implemented a background `ingest_worker` thread to handle HTTP POST requests without stalling the audio processing.
+    - **Flat JSON Schema:** Used a key-value format as requested (e.g., `original`, `es`, `en`, `fr`).
+    - **Reliability:** Integrated exponential backoff retries (1s to 15s) to handle network or server failures.

transcribe.py

@@ -1,4 +1,8 @@
 import sys
+import time
+import requests
+import threading
+import json
 from unittest.mock import MagicMock
 
 # Comprehensive workaround for missing _lzma in some Python builds
@@ -25,12 +29,15 @@ from silero_vad import load_silero_vad, get_speech_timestamps
 from transformers import MarianMTModel, MarianTokenizer
 
 # Parameters
-WHISPER_MODEL = "mlx-community/whisper-small.en-mlx"
+WHISPER_MODEL = "mlx-community/whisper-small-mlx" 
-# List of language pairs (English to ...)
+INGEST_URL = "https://emiapi.reynafamily.com/live-captions/ingest"
+
+# Translation models (English -> Target)
+# Map to the specific keys requested by the backend
 TARGET_LANGS = {
-    "Spanish": "Helsinki-NLP/opus-mt-en-es",
+    "es": "Helsinki-NLP/opus-mt-en-es",
-    "French":  "Helsinki-NLP/opus-mt-en-fr",
+    "fr": "Helsinki-NLP/opus-mt-en-fr",
-    "Arabic":  "Helsinki-NLP/opus-mt-en-ar"
+    "ar": "Helsinki-NLP/opus-mt-en-ar" # Added Arabic as discussed before
 }
 
 CHANNELS = 1
@@ -41,30 +48,63 @@ BUFFER_LIMIT = SAMPLERATE * 30
 MIN_SILENCE_DURATION_MS = 500 
 
 audio_queue = queue.Queue()
+ingest_queue = queue.Queue()
 
 def callback(indata, frames, time, status):
     if status:
         print(status, file=sys.stderr)
     audio_queue.put(indata.copy())
 
+def ingest_worker():
+    """Background thread to handle server ingestion with retries."""
+    while True:
+        payload = ingest_queue.get()
+        if payload is None: break
+        
+        delay = 1
+        max_delay = 15
+        success = False
+        
+        while not success:
+            try:
+                response = requests.post(INGEST_URL, json=payload, timeout=5)
+                if response.status_code == 200:
+                    success = True
+                else:
+                    print(f"\n[Ingest Error] Server returned {response.status_code}. Retrying in {delay}s...")
+            except Exception as e:
+                print(f"\n[Ingest Error] {e}. Retrying in {delay}s...")
+            
+            if not success:
+                time.sleep(delay)
+                delay = min(delay * 2, max_delay)
+        
+        ingest_queue.task_done()
+
 def main():
     device = "mps" if torch.backends.mps.is_available() else "cpu"
     print(f"Using device: {device}")
 
-    print(f"Loading Whisper model '{WHISPER_MODEL}'...")
+    # Start ingest thread
+    threading.Thread(target=ingest_worker, daemon=True).start()
+
+    # 1. Load models
+    print(f"Loading Multilingual Whisper model '{WHISPER_MODEL}'...")
     
-    # Dictionary to hold models and tokenizers
     translation_engines = {}
-    
+    for lang_key, model_id in TARGET_LANGS.items():
-    for lang_name, model_id in TARGET_LANGS.items():
+        print(f"Loading {lang_key} translation model ({model_id})...")
-        print(f"Loading {lang_name} translation model ({model_id})...")
         tokenizer = MarianTokenizer.from_pretrained(model_id)
         model = MarianMTModel.from_pretrained(model_id).to(device)
-        translation_engines[lang_name] = (model, tokenizer)
+        translation_engines[lang_key] = (model, tokenizer)
 
+    print("Loading Silero VAD model...")
+    vad_model = load_silero_vad()
+    print("Models loaded.")
+
+    # 2. Select Audio Device
     print("\nAvailable Audio Devices:")
-    devices = sd.query_devices()
+    print(sd.query_devices())
-    print(devices)
     
     try:
         device_input = input("\nSelect input device index (or press Enter for default): ")
@@ -73,7 +113,7 @@ def main():
         print("Invalid input, using default device.")
         device_index = None
 
-    print(f"\nStarting live transcription & multiple translations using device {device_index if device_index is not None else 'default'}... (Press Ctrl+C to stop)")
+    print(f"\nStarting live transcription & server ingest... (Press Ctrl+C to stop)")
     
     audio_buffer = []
     speech_started = False
@@ -104,25 +144,46 @@ def main():
                         
                         if (buffer_len_samples - last_end) > (SAMPLERATE * MIN_SILENCE_DURATION_MS / 1000) or buffer_len_samples > BUFFER_LIMIT:
                             
-                            # 1. Transcribe once
+                            # 1. Transcribe & Detect Language
-                            result = mlx_whisper.transcribe(current_audio, path_or_hf_repo=WHISPER_MODEL)
+                            transcription_result = mlx_whisper.transcribe(current_audio, path_or_hf_repo=WHISPER_MODEL)
-                            original_text = result['text'].strip()
+                            original_text = transcription_result['text'].strip()
+                            detected_lang = transcription_result.get('language', 'en')
                             
                             if original_text:
-                                # Limit the input length to avoid memory spikes or model glitches
+                                print(f"\n[{detected_lang.upper()}]: {original_text}")
-                                if len(original_text) > 250:
-                                    original_text = original_text[:247] + "..."
                                 
-                                print(f"\n[EN]: {original_text}")
+                                # Prepare payload
+                                payload = {"original": original_text}
+                                # Rule 3: include source language key
+                                if detected_lang in TARGET_LANGS or detected_lang == "en":
+                                    payload[detected_lang] = original_text
+
+                                # 2. Bridge to English if not already English
+                                if detected_lang != "en":
+                                    bridge_result = mlx_whisper.transcribe(current_audio, path_or_hf_repo=WHISPER_MODEL, task="translate")
+                                    english_text = bridge_result['text'].strip()
+                                    payload["en"] = english_text
+                                else:
+                                    english_text = original_text
+
+                                # 3. Translate from English to other languages
+                                if english_text:
+                                    # Limit input length
+                                    clean_en = english_text[:247] + "..." if len(english_text) > 250 else english_text
+                                    
+                                    for lang_key, (model, tokenizer) in translation_engines.items():
+                                        # Skip if we already filled this (e.g. detected lang was 'es')
+                                        if lang_key in payload: continue
+                                        
+                                        inputs = tokenizer(clean_en, return_tensors="pt", padding=True).to(device)
+                                        with torch.no_grad():
+                                            translated_tokens = model.generate(**inputs, max_new_tokens=150)
+                                        translated_text = tokenizer.decode(translated_tokens[0], skip_special_tokens=True)
+                                        payload[lang_key] = translated_text
                                 
-                                # 2. Translate to all targets
+                                # Queue for background ingestion
-                                for lang_name, (model, tokenizer) in translation_engines.items():
+                                ingest_queue.put(payload)
-                                    inputs = tokenizer(original_text, return_tensors="pt", padding=True).to(device)
+                                print(f"Sent to ingest: {list(payload.keys())}")
-                                    with torch.no_grad():
-                                        # Added max_new_tokens to prevent runaway generation
-                                        translated_tokens = model.generate(**inputs, max_new_tokens=150)
-                                    translated_text = tokenizer.decode(translated_tokens[0], skip_special_tokens=True)
-                                    print(f"[{lang_name[:2].upper()}]: {translated_text}")
                             
                             audio_buffer = []
                             speech_started = False
@@ -136,4 +197,6 @@ def main():
         print(f"\nError: {e}")
 
 if __name__ == "__main__":
+    import multiprocessing
+    multiprocessing.freeze_support()
     main()