From 00231cc747a6a1721ea87693a9fd1352ae64f259 Mon Sep 17 00:00:00 2001 From: Adolfo Reyna Date: Sun, 1 Mar 2026 11:15:59 -0500 Subject: [PATCH] feat: add hallucination detection and watchdog for stuck transcriptions --- README.md | 5 +++ transcribe.py | 109 ++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 111 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 40360b7..bc69c4b 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,10 @@ Run the script using `python3 transcribe.py` with optional flags. ```bash python3 transcribe.py -l ``` +- **Caption system audio (speakers) using a loopback device:** + ```bash + python3 transcribe.py --loopback -es + ``` - **Transcribe and translate to Spanish (screen only):** ```bash python3 transcribe.py -es @@ -55,6 +59,7 @@ Run the script using `python3 transcribe.py` with optional flags. - `-i`, `--ingest`: Enable data transmission to the remote server. - `-l`, `--list-devices`: Show available audio devices and exit. - `-d`, `--device [ID]`: Input device index (bypasses selection prompt). +- `--loopback`: Automatically select a loopback device (e.g., BlackHole, Stereo Mix) to caption system audio. - `-q`, `--quantize`: Use 4-bit quantized Whisper model for faster transcription (Strategy 3). - `-s`, `--stream`: Enable real-time streaming transcription/draft mode (Strategy 1). - `-c`, `--context`: Enable prompt caching/rolling context to help the model maintain sentence continuity across chunks. diff --git a/transcribe.py b/transcribe.py index bfebca7..910337c 100644 --- a/transcribe.py +++ b/transcribe.py @@ -1,10 +1,12 @@ import sys import time +import re import requests import threading import json import argparse from unittest.mock import MagicMock +from collections import Counter # Comprehensive workaround for missing _lzma in some Python builds try: @@ -45,11 +47,38 @@ CHANNELS = 1 SAMPLERATE = 16000 BLOCK_SIZE = 512 VAD_THRESHOLD = 0.5 -BUFFER_LIMIT = SAMPLERATE * 30 audio_queue = queue.Queue() ingest_queue = queue.Queue() +def is_hallucination(text): + """Detect common Whisper hallucinations or high repetition.""" + if not text: return False + + # Common hallucinations + hallucinations = [ + r"thanks? for watching", + r"please subscribe", + r"youtube", + r"click the link", + r"like and subscribe", + r"tuned in", + r"next time", + ] + for pattern in hallucinations: + if re.search(pattern, text, re.IGNORECASE): + return True + + # Check for excessive word repetition (e.g. "Hallelujah" repeated 10 times) + words = text.lower().split() + if len(words) >= 8: + counts = Counter(words) + most_common_word, count = counts.most_common(1)[0] + if count / len(words) > 0.75: + return True + + return False + def callback(indata, frames, time, status): if status: print(status, file=sys.stderr) @@ -91,11 +120,13 @@ def main(): parser.add_argument("-i", "--ingest", action="store_true", help="Enable data transmission to server") parser.add_argument("-l", "--list-devices", action="store_true", help="Show available audio devices and exit") parser.add_argument("-d", "--device", type=int, help="Input device index") + parser.add_argument("--loopback", action="store_true", help="Automatically select a loopback device (e.g., BlackHole, Stereo Mix)") parser.add_argument("-q", "--quantize", action="store_true", help="Use 4-bit quantized Whisper model for speed") parser.add_argument("-s", "--stream", action="store_true", help="Enable real-time streaming transcription (Draft mode)") parser.add_argument("-c", "--context", action="store_true", help="Enable prompt caching/rolling context for better continuity") parser.add_argument("--lang", type=str, help="Hardcode source language (e.g. 'en', 'es') to bypass detection") parser.add_argument("--silence", type=int, default=1000, help="Minimum silence duration in ms to end a chunk (default: 1000)") + parser.add_argument("--max-buffer", type=int, default=20, help="Maximum buffer duration in seconds before forcing a flush (default: 20)") args = parser.parse_args() @@ -107,6 +138,7 @@ def main(): print(sd.query_devices()) return + buffer_limit = SAMPLERATE * args.max_buffer device = "mps" if torch.backends.mps.is_available() else "cpu" print(f"Using device: {device}") @@ -132,9 +164,28 @@ def main(): print("Models loaded.") # 2. Select Audio Device + device_index = None if args.device is not None: device_index = args.device - else: + elif args.loopback: + print("Searching for loopback device...") + devices = sd.query_devices() + for i, dev in enumerate(devices): + name = dev['name'].lower() + if any(keyword in name for keyword in ["blackhole", "loopback", "soundflower", "stereo mix"]): + if dev['max_input_channels'] > 0: + device_index = i + print(f"Using loopback device: {dev['name']} (Index {i})") + print("\n[TIP] For macOS with BlackHole:") + print("1. Open 'Audio MIDI Setup' and create a 'Multi-Output Device'.") + print("2. Select your speakers AND 'BlackHole 2ch'.") + print("3. Set your system output to this 'Multi-Output Device'.") + print("This way you can hear the audio while it is being captioned.\n") + break + if device_index is None: + print("No loopback device found. Falling back to default.") + + if device_index is None and not args.loopback: print("\nAvailable Audio Devices:") print(sd.query_devices()) try: @@ -144,12 +195,19 @@ def main(): print("Invalid input, using default device.") device_index = None + if device_index is not None: + print(f"Using device index: {device_index}") + else: + print("Using system default input device.") + print(f"\nStarting live transcription{' & server ingest' if args.ingest else ''}... (Press Ctrl+C to stop)") audio_buffer = [] speech_started = False last_stream_time = time.time() + last_change_time = time.time() rolling_context = "" + last_draft_text = "" try: with sd.InputStream(samplerate=SAMPLERATE, channels=CHANNELS, callback=callback, blocksize=BLOCK_SIZE, device=device_index): @@ -161,6 +219,7 @@ def main(): if len(audio_buffer) > 0: current_audio = np.concatenate(audio_buffer) audio_tensor = torch.from_numpy(current_audio) + buffer_duration = len(current_audio) / SAMPLERATE speech_timestamps = get_speech_timestamps( audio_tensor, @@ -170,12 +229,28 @@ def main(): min_silence_duration_ms=args.silence ) + # --- STUCK WATCHDOG --- + # If buffer is getting long (>12s) and we haven't had a change in draft for 7s, + # OR if buffer is extremely long (>25s) regardless of draft activity. + time_since_last_change = time.time() - last_change_time + if (buffer_duration > 12.0 and time_since_last_change > 7.0) or (buffer_duration > 25.0): + print(f"\n[SYSTEM]: Transcription watchdog triggered (Buffer: {buffer_duration:.1f}s, No change: {time_since_last_change:.1f}s). Resetting...") + audio_buffer = [] + speech_started = False + rolling_context = "" + last_draft_text = "" + last_change_time = time.time() + if args.stream: + sys.stdout.write("\r\033[K") + sys.stdout.flush() + continue + if len(speech_timestamps) > 0: speech_started = True last_end = speech_timestamps[-1]['end'] buffer_len_samples = len(current_audio) - if (buffer_len_samples - last_end) > (SAMPLERATE * args.silence / 1000) or buffer_len_samples > BUFFER_LIMIT: + if (buffer_len_samples - last_end) > (SAMPLERATE * args.silence / 1000) or buffer_len_samples > buffer_limit: # Clear draft line if it was used if args.stream: @@ -194,11 +269,18 @@ def main(): original_text = transcription_result['text'].strip() detected_lang = transcription_result.get('language', args.lang if args.lang else 'en') + if is_hallucination(original_text): + print(f"\n[SYSTEM]: Hallucination detected, ignoring and resetting context.") + original_text = "" + rolling_context = "" + if original_text: print(f"\n[{detected_lang.upper()}]: {original_text}") + last_change_time = time.time() # Successfully transcribed full segment # Prepare payload payload = {"original": original_text} + # ... (payload construction) # Include detected language if requested or if it's the bridge if (detected_lang in active_target_langs) or (detected_lang == "en" and args.en): @@ -251,6 +333,8 @@ def main(): audio_buffer = [] speech_started = False last_stream_time = time.time() + stuck_draft_count = 0 + last_draft_text = "" elif args.stream and (time.time() - last_stream_time) > 1.0: # Draft transcription @@ -260,13 +344,32 @@ def main(): draft_result = mlx_whisper.transcribe(current_audio, **draft_kwargs) draft_text = draft_result['text'].strip() + if draft_text: + # Update timestamp ONLY if the text actually changed + if draft_text != last_draft_text: + last_change_time = time.time() + last_draft_text = draft_text + sys.stdout.write(f"\r\033[K[DRAFT]: {draft_text}") sys.stdout.flush() # Send draft to ingest if enabled if args.ingest: ingest_queue.put({"draft": draft_text}) + else: + # If Whisper returns empty, check if we've been silent for too long + # even though VAD says there is speech. + if buffer_duration > 10.0 and (time.time() - last_change_time) > 7.0: + print(f"\n[SYSTEM]: Draft is empty while audio continues. Forcing reset...") + audio_buffer = [] + speech_started = False + rolling_context = "" + last_change_time = time.time() + last_draft_text = "" + sys.stdout.write("\r\033[K") + sys.stdout.flush() + continue last_stream_time = time.time()