feat: add command-line arguments for lang selection, ingest toggle, and device management; add README and requirements
This commit is contained in:
@@ -1,2 +1,3 @@
|
|||||||
__pycache__/\n*.pyc\n.DS_Store
|
__pycache__/\n*.pyc\n.DS_Store
|
||||||
build/\ndist/\n*.spec
|
build/\ndist/\n*.spec
|
||||||
|
venv/
|
||||||
|
|||||||
@@ -0,0 +1,76 @@
|
|||||||
|
# Python Whisper Live Transcription & Translation
|
||||||
|
|
||||||
|
A real-time, low-latency audio transcription and translation tool utilizing OpenAI's Whisper (via `mlx-whisper` for Apple Silicon optimization), Silero VAD for speech detection, and Helsinki-NLP's Opus-MT models for translation.
|
||||||
|
|
||||||
|
## Features
|
||||||
|
- **Live Transcription:** Real-time speech-to-text with automatic language detection.
|
||||||
|
- **On-the-fly Translation:** Bridge translations to English and then to target languages (Spanish, Arabic, French, etc.).
|
||||||
|
- **Voice Activity Detection (VAD):** Intelligent audio buffering using Silero VAD to process only actual speech.
|
||||||
|
- **Apple Silicon Optimized:** Uses MLX for high performance on Mac (MPS).
|
||||||
|
- **Background Ingestion:** Optional background thread to send JSON payloads to a remote server.
|
||||||
|
- **Configurable:** Command-line parameters to select languages, devices, and ingestion.
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
|
||||||
|
1. **Clone the repository:**
|
||||||
|
```bash
|
||||||
|
git clone <repository-url>
|
||||||
|
cd pythonwhisper
|
||||||
|
```
|
||||||
|
|
||||||
|
2. **Install dependencies:**
|
||||||
|
Ensure you have Python 3.9+ and the required libraries:
|
||||||
|
```bash
|
||||||
|
pip install mlx-whisper numpy sounddevice torch requests transformers silero-vad
|
||||||
|
```
|
||||||
|
*Note: On Apple Silicon, ensure `mlx` and `torch` with MPS support are correctly installed.*
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
Run the script using `python3 transcribe.py` with optional flags.
|
||||||
|
|
||||||
|
### Common Commands
|
||||||
|
- **List available audio devices:**
|
||||||
|
```bash
|
||||||
|
python3 transcribe.py -l
|
||||||
|
```
|
||||||
|
- **Transcribe and translate to Spanish (screen only):**
|
||||||
|
```bash
|
||||||
|
python3 transcribe.py -es
|
||||||
|
```
|
||||||
|
- **Enable Spanish, Arabic, and French with English bridging, and send to server:**
|
||||||
|
```bash
|
||||||
|
python3 transcribe.py -es -ar -fr -en -i
|
||||||
|
```
|
||||||
|
- **Use a specific input device (e.g., index 3) and translate to Spanish:**
|
||||||
|
```bash
|
||||||
|
python3 transcribe.py -d 3 -es
|
||||||
|
```
|
||||||
|
|
||||||
|
### Arguments
|
||||||
|
- `-es`: Enable Spanish translation.
|
||||||
|
- `-en`: Enable English (shows original if detected, or bridged if not).
|
||||||
|
- `-ar`: Enable Arabic translation.
|
||||||
|
- `-fr`: Enable French translation.
|
||||||
|
- `-i`, `--ingest`: Enable data transmission to the remote server.
|
||||||
|
- `-l`, `--list-devices`: Show available audio devices and exit.
|
||||||
|
- `-d`, `--device [ID]`: Input device index (bypasses selection prompt).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Technical Note: Universal Translation Models
|
||||||
|
|
||||||
|
While the current implementation uses specialized, per-language models from the **Helsinki-NLP Opus-MT** project (e.g., `opus-mt-en-es`, `opus-mt-en-ar`), there is an alternative approach: **Universal Models**.
|
||||||
|
|
||||||
|
### Universal Model Alternative (e.g., Meta's NLLB-200)
|
||||||
|
The current per-language model approach is highly accurate and memory-efficient if you only need 1 or 2 target languages. However, if you require support for many languages simultaneously, loading multiple specialized models can consume significant RAM/VRAM.
|
||||||
|
|
||||||
|
We have the option to switch the translation engine to a single, universal model such as **NLLB-200 (No Language Left Behind)**:
|
||||||
|
- **Model ID:** `facebook/nllb-200-distilled-600M`
|
||||||
|
- **Benefits:**
|
||||||
|
- Supports over **200 languages** in a single model.
|
||||||
|
- Simplified code: no need to load/manage multiple model objects.
|
||||||
|
- More efficient for complex multilingual environments.
|
||||||
|
- **Trade-off:** Slightly higher memory footprint for the single model compared to a single specialized model, but more efficient than 3+ specialized models.
|
||||||
|
|
||||||
|
If you wish to switch to a universal model, the `transcribe.py` logic can be updated to use a single `M2M100` or `NLLB` pipeline instead of the current `MarianMT` loop.
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
mlx-whisper
|
||||||
|
numpy
|
||||||
|
sounddevice
|
||||||
|
torch
|
||||||
|
requests
|
||||||
|
transformers
|
||||||
|
silero-vad
|
||||||
|
pyinstaller
|
||||||
|
sacremoses
|
||||||
|
joblib
|
||||||
+43
-10
@@ -3,6 +3,7 @@ import time
|
|||||||
import requests
|
import requests
|
||||||
import threading
|
import threading
|
||||||
import json
|
import json
|
||||||
|
import argparse
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
# Comprehensive workaround for missing _lzma in some Python builds
|
# Comprehensive workaround for missing _lzma in some Python builds
|
||||||
@@ -82,17 +83,37 @@ def ingest_worker():
|
|||||||
ingest_queue.task_done()
|
ingest_queue.task_done()
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="Live transcription and translation with Whisper.")
|
||||||
|
parser.add_argument("-es", action="store_true", help="Enable Spanish translation")
|
||||||
|
parser.add_argument("-en", action="store_true", help="Enable English (detected or bridged)")
|
||||||
|
parser.add_argument("-ar", action="store_true", help="Enable Arabic translation")
|
||||||
|
parser.add_argument("-fr", action="store_true", help="Enable French translation")
|
||||||
|
parser.add_argument("-i", "--ingest", action="store_true", help="Enable data transmission to server")
|
||||||
|
parser.add_argument("-l", "--list-devices", action="store_true", help="Show available audio devices and exit")
|
||||||
|
parser.add_argument("-d", "--device", type=int, help="Input device index")
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.list_devices:
|
||||||
|
print("\nAvailable Audio Devices:")
|
||||||
|
print(sd.query_devices())
|
||||||
|
return
|
||||||
|
|
||||||
device = "mps" if torch.backends.mps.is_available() else "cpu"
|
device = "mps" if torch.backends.mps.is_available() else "cpu"
|
||||||
print(f"Using device: {device}")
|
print(f"Using device: {device}")
|
||||||
|
|
||||||
# Start ingest thread
|
# Only start ingest thread if enabled
|
||||||
|
if args.ingest:
|
||||||
threading.Thread(target=ingest_worker, daemon=True).start()
|
threading.Thread(target=ingest_worker, daemon=True).start()
|
||||||
|
|
||||||
# 1. Load models
|
# 1. Load models
|
||||||
print(f"Loading Multilingual Whisper model '{WHISPER_MODEL}'...")
|
print(f"Loading Multilingual Whisper model '{WHISPER_MODEL}'...")
|
||||||
|
|
||||||
|
# Filter translation models to only those enabled by args
|
||||||
|
active_target_langs = {k: v for k, v in TARGET_LANGS.items() if getattr(args, k, False)}
|
||||||
|
|
||||||
translation_engines = {}
|
translation_engines = {}
|
||||||
for lang_key, model_id in TARGET_LANGS.items():
|
for lang_key, model_id in active_target_langs.items():
|
||||||
print(f"Loading {lang_key} translation model ({model_id})...")
|
print(f"Loading {lang_key} translation model ({model_id})...")
|
||||||
tokenizer = MarianTokenizer.from_pretrained(model_id)
|
tokenizer = MarianTokenizer.from_pretrained(model_id)
|
||||||
model = MarianMTModel.from_pretrained(model_id).to(device)
|
model = MarianMTModel.from_pretrained(model_id).to(device)
|
||||||
@@ -103,9 +124,11 @@ def main():
|
|||||||
print("Models loaded.")
|
print("Models loaded.")
|
||||||
|
|
||||||
# 2. Select Audio Device
|
# 2. Select Audio Device
|
||||||
|
if args.device is not None:
|
||||||
|
device_index = args.device
|
||||||
|
else:
|
||||||
print("\nAvailable Audio Devices:")
|
print("\nAvailable Audio Devices:")
|
||||||
print(sd.query_devices())
|
print(sd.query_devices())
|
||||||
|
|
||||||
try:
|
try:
|
||||||
device_input = input("\nSelect input device index (or press Enter for default): ")
|
device_input = input("\nSelect input device index (or press Enter for default): ")
|
||||||
device_index = int(device_input) if device_input.strip() else None
|
device_index = int(device_input) if device_input.strip() else None
|
||||||
@@ -113,7 +136,7 @@ def main():
|
|||||||
print("Invalid input, using default device.")
|
print("Invalid input, using default device.")
|
||||||
device_index = None
|
device_index = None
|
||||||
|
|
||||||
print(f"\nStarting live transcription & server ingest... (Press Ctrl+C to stop)")
|
print(f"\nStarting live transcription{' & server ingest' if args.ingest else ''}... (Press Ctrl+C to stop)")
|
||||||
|
|
||||||
audio_buffer = []
|
audio_buffer = []
|
||||||
speech_started = False
|
speech_started = False
|
||||||
@@ -154,36 +177,46 @@ def main():
|
|||||||
|
|
||||||
# Prepare payload
|
# Prepare payload
|
||||||
payload = {"original": original_text}
|
payload = {"original": original_text}
|
||||||
# Rule 3: include source language key
|
|
||||||
if detected_lang in TARGET_LANGS or detected_lang == "en":
|
# Include detected language if requested or if it's the bridge
|
||||||
|
if (detected_lang in active_target_langs) or (detected_lang == "en" and args.en):
|
||||||
payload[detected_lang] = original_text
|
payload[detected_lang] = original_text
|
||||||
|
|
||||||
# 2. Bridge to English if not already English
|
# 2. Bridge to English if not already English
|
||||||
if detected_lang != "en":
|
if detected_lang != "en":
|
||||||
bridge_result = mlx_whisper.transcribe(current_audio, path_or_hf_repo=WHISPER_MODEL, task="translate")
|
bridge_result = mlx_whisper.transcribe(current_audio, path_or_hf_repo=WHISPER_MODEL, task="translate")
|
||||||
english_text = bridge_result['text'].strip()
|
english_text = bridge_result['text'].strip()
|
||||||
|
if args.en:
|
||||||
payload["en"] = english_text
|
payload["en"] = english_text
|
||||||
|
print(f"[EN]: {english_text}")
|
||||||
else:
|
else:
|
||||||
english_text = original_text
|
english_text = original_text
|
||||||
|
if args.en:
|
||||||
|
payload["en"] = english_text
|
||||||
|
|
||||||
# 3. Translate from English to other languages
|
# 3. Translate from English to other languages
|
||||||
if english_text:
|
if english_text and translation_engines:
|
||||||
# Limit input length
|
# Limit input length
|
||||||
clean_en = english_text[:247] + "..." if len(english_text) > 250 else english_text
|
clean_en = english_text[:247] + "..." if len(english_text) > 250 else english_text
|
||||||
|
|
||||||
for lang_key, (model, tokenizer) in translation_engines.items():
|
for lang_key, (model, tokenizer) in translation_engines.items():
|
||||||
# Skip if we already filled this (e.g. detected lang was 'es')
|
# Skip if we already filled this (e.g. detected lang was 'es')
|
||||||
if lang_key in payload: continue
|
if lang_key in payload:
|
||||||
|
if lang_key != detected_lang: # Already printed original
|
||||||
|
print(f"[{lang_key.upper()}]: {payload[lang_key]}")
|
||||||
|
continue
|
||||||
|
|
||||||
inputs = tokenizer(clean_en, return_tensors="pt", padding=True).to(device)
|
inputs = tokenizer(clean_en, return_tensors="pt", padding=True).to(device)
|
||||||
with torch.no_grad():
|
with torch.no_grad():
|
||||||
translated_tokens = model.generate(**inputs, max_new_tokens=150)
|
translated_tokens = model.generate(**inputs, max_new_tokens=150)
|
||||||
translated_text = tokenizer.decode(translated_tokens[0], skip_special_tokens=True)
|
translated_text = tokenizer.decode(translated_tokens[0], skip_special_tokens=True)
|
||||||
payload[lang_key] = translated_text
|
payload[lang_key] = translated_text
|
||||||
|
print(f"[{lang_key.upper()}]: {translated_text}")
|
||||||
|
|
||||||
# Queue for background ingestion
|
# Queue for background ingestion if enabled
|
||||||
|
if args.ingest:
|
||||||
ingest_queue.put(payload)
|
ingest_queue.put(payload)
|
||||||
print(f"Sent to ingest: {list(payload.keys())}")
|
# print(f"Sent to ingest: {list(payload.keys())}")
|
||||||
|
|
||||||
audio_buffer = []
|
audio_buffer = []
|
||||||
speech_started = False
|
speech_started = False
|
||||||
|
|||||||
Reference in New Issue
Block a user