From 3c874c113c8961679681eae0bd671c797bbea278 Mon Sep 17 00:00:00 2001 From: Adolfo Reyna Date: Sat, 28 Feb 2026 18:55:37 -0500 Subject: [PATCH] feat: add command-line arguments for lang selection, ingest toggle, and device management; add README and requirements --- .gitignore | 1 + README.md | 76 ++++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 10 +++++++ transcribe.py | 75 ++++++++++++++++++++++++++++++++++------------- 4 files changed, 141 insertions(+), 21 deletions(-) create mode 100644 README.md create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 18c946b..eb9be72 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ __pycache__/\n*.pyc\n.DS_Store build/\ndist/\n*.spec +venv/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..32c134d --- /dev/null +++ b/README.md @@ -0,0 +1,76 @@ +# Python Whisper Live Transcription & Translation + +A real-time, low-latency audio transcription and translation tool utilizing OpenAI's Whisper (via `mlx-whisper` for Apple Silicon optimization), Silero VAD for speech detection, and Helsinki-NLP's Opus-MT models for translation. + +## Features +- **Live Transcription:** Real-time speech-to-text with automatic language detection. +- **On-the-fly Translation:** Bridge translations to English and then to target languages (Spanish, Arabic, French, etc.). +- **Voice Activity Detection (VAD):** Intelligent audio buffering using Silero VAD to process only actual speech. +- **Apple Silicon Optimized:** Uses MLX for high performance on Mac (MPS). +- **Background Ingestion:** Optional background thread to send JSON payloads to a remote server. +- **Configurable:** Command-line parameters to select languages, devices, and ingestion. + +## Installation + +1. **Clone the repository:** + ```bash + git clone + cd pythonwhisper + ``` + +2. **Install dependencies:** + Ensure you have Python 3.9+ and the required libraries: + ```bash + pip install mlx-whisper numpy sounddevice torch requests transformers silero-vad + ``` + *Note: On Apple Silicon, ensure `mlx` and `torch` with MPS support are correctly installed.* + +## Usage + +Run the script using `python3 transcribe.py` with optional flags. + +### Common Commands +- **List available audio devices:** + ```bash + python3 transcribe.py -l + ``` +- **Transcribe and translate to Spanish (screen only):** + ```bash + python3 transcribe.py -es + ``` +- **Enable Spanish, Arabic, and French with English bridging, and send to server:** + ```bash + python3 transcribe.py -es -ar -fr -en -i + ``` +- **Use a specific input device (e.g., index 3) and translate to Spanish:** + ```bash + python3 transcribe.py -d 3 -es + ``` + +### Arguments +- `-es`: Enable Spanish translation. +- `-en`: Enable English (shows original if detected, or bridged if not). +- `-ar`: Enable Arabic translation. +- `-fr`: Enable French translation. +- `-i`, `--ingest`: Enable data transmission to the remote server. +- `-l`, `--list-devices`: Show available audio devices and exit. +- `-d`, `--device [ID]`: Input device index (bypasses selection prompt). + +--- + +## Technical Note: Universal Translation Models + +While the current implementation uses specialized, per-language models from the **Helsinki-NLP Opus-MT** project (e.g., `opus-mt-en-es`, `opus-mt-en-ar`), there is an alternative approach: **Universal Models**. + +### Universal Model Alternative (e.g., Meta's NLLB-200) +The current per-language model approach is highly accurate and memory-efficient if you only need 1 or 2 target languages. However, if you require support for many languages simultaneously, loading multiple specialized models can consume significant RAM/VRAM. + +We have the option to switch the translation engine to a single, universal model such as **NLLB-200 (No Language Left Behind)**: +- **Model ID:** `facebook/nllb-200-distilled-600M` +- **Benefits:** + - Supports over **200 languages** in a single model. + - Simplified code: no need to load/manage multiple model objects. + - More efficient for complex multilingual environments. +- **Trade-off:** Slightly higher memory footprint for the single model compared to a single specialized model, but more efficient than 3+ specialized models. + +If you wish to switch to a universal model, the `transcribe.py` logic can be updated to use a single `M2M100` or `NLLB` pipeline instead of the current `MarianMT` loop. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4d48798 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +mlx-whisper +numpy +sounddevice +torch +requests +transformers +silero-vad +pyinstaller +sacremoses +joblib diff --git a/transcribe.py b/transcribe.py index 7148b5d..78f6468 100644 --- a/transcribe.py +++ b/transcribe.py @@ -3,6 +3,7 @@ import time import requests import threading import json +import argparse from unittest.mock import MagicMock # Comprehensive workaround for missing _lzma in some Python builds @@ -82,17 +83,37 @@ def ingest_worker(): ingest_queue.task_done() def main(): + parser = argparse.ArgumentParser(description="Live transcription and translation with Whisper.") + parser.add_argument("-es", action="store_true", help="Enable Spanish translation") + parser.add_argument("-en", action="store_true", help="Enable English (detected or bridged)") + parser.add_argument("-ar", action="store_true", help="Enable Arabic translation") + parser.add_argument("-fr", action="store_true", help="Enable French translation") + parser.add_argument("-i", "--ingest", action="store_true", help="Enable data transmission to server") + parser.add_argument("-l", "--list-devices", action="store_true", help="Show available audio devices and exit") + parser.add_argument("-d", "--device", type=int, help="Input device index") + + args = parser.parse_args() + + if args.list_devices: + print("\nAvailable Audio Devices:") + print(sd.query_devices()) + return + device = "mps" if torch.backends.mps.is_available() else "cpu" print(f"Using device: {device}") - # Start ingest thread - threading.Thread(target=ingest_worker, daemon=True).start() + # Only start ingest thread if enabled + if args.ingest: + threading.Thread(target=ingest_worker, daemon=True).start() # 1. Load models print(f"Loading Multilingual Whisper model '{WHISPER_MODEL}'...") + # Filter translation models to only those enabled by args + active_target_langs = {k: v for k, v in TARGET_LANGS.items() if getattr(args, k, False)} + translation_engines = {} - for lang_key, model_id in TARGET_LANGS.items(): + for lang_key, model_id in active_target_langs.items(): print(f"Loading {lang_key} translation model ({model_id})...") tokenizer = MarianTokenizer.from_pretrained(model_id) model = MarianMTModel.from_pretrained(model_id).to(device) @@ -103,17 +124,19 @@ def main(): print("Models loaded.") # 2. Select Audio Device - print("\nAvailable Audio Devices:") - print(sd.query_devices()) - - try: - device_input = input("\nSelect input device index (or press Enter for default): ") - device_index = int(device_input) if device_input.strip() else None - except ValueError: - print("Invalid input, using default device.") - device_index = None + if args.device is not None: + device_index = args.device + else: + print("\nAvailable Audio Devices:") + print(sd.query_devices()) + try: + device_input = input("\nSelect input device index (or press Enter for default): ") + device_index = int(device_input) if device_input.strip() else None + except ValueError: + print("Invalid input, using default device.") + device_index = None - print(f"\nStarting live transcription & server ingest... (Press Ctrl+C to stop)") + print(f"\nStarting live transcription{' & server ingest' if args.ingest else ''}... (Press Ctrl+C to stop)") audio_buffer = [] speech_started = False @@ -154,36 +177,46 @@ def main(): # Prepare payload payload = {"original": original_text} - # Rule 3: include source language key - if detected_lang in TARGET_LANGS or detected_lang == "en": + + # Include detected language if requested or if it's the bridge + if (detected_lang in active_target_langs) or (detected_lang == "en" and args.en): payload[detected_lang] = original_text # 2. Bridge to English if not already English if detected_lang != "en": bridge_result = mlx_whisper.transcribe(current_audio, path_or_hf_repo=WHISPER_MODEL, task="translate") english_text = bridge_result['text'].strip() - payload["en"] = english_text + if args.en: + payload["en"] = english_text + print(f"[EN]: {english_text}") else: english_text = original_text + if args.en: + payload["en"] = english_text # 3. Translate from English to other languages - if english_text: + if english_text and translation_engines: # Limit input length clean_en = english_text[:247] + "..." if len(english_text) > 250 else english_text for lang_key, (model, tokenizer) in translation_engines.items(): # Skip if we already filled this (e.g. detected lang was 'es') - if lang_key in payload: continue + if lang_key in payload: + if lang_key != detected_lang: # Already printed original + print(f"[{lang_key.upper()}]: {payload[lang_key]}") + continue inputs = tokenizer(clean_en, return_tensors="pt", padding=True).to(device) with torch.no_grad(): translated_tokens = model.generate(**inputs, max_new_tokens=150) translated_text = tokenizer.decode(translated_tokens[0], skip_special_tokens=True) payload[lang_key] = translated_text + print(f"[{lang_key.upper()}]: {translated_text}") - # Queue for background ingestion - ingest_queue.put(payload) - print(f"Sent to ingest: {list(payload.keys())}") + # Queue for background ingestion if enabled + if args.ingest: + ingest_queue.put(payload) + # print(f"Sent to ingest: {list(payload.keys())}") audio_buffer = [] speech_started = False