From 11dbc5ce9a4b12e86c11d5eaa84ed75d931f2e02 Mon Sep 17 00:00:00 2001 From: Adolfo Reyna Date: Fri, 6 Mar 2026 20:02:38 -0500 Subject: [PATCH] feat: add optional speaker-change detection for line cuts --- history.md | 9 ++++++ transcribe.py | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/history.md b/history.md index dbf61db..b6fe41c 100644 --- a/history.md +++ b/history.md @@ -176,3 +176,12 @@ The project now features a high-performance, Apple Silicon-optimized pipeline th - Enforced strict low-latency behavior with `--llm-merge-timeout` (default 0.7s) and automatic fallback to heuristic merges on errors/timeouts. - Added safety guards to reject overly large LLM rewrites and preserve buffered caption state when merge is rejected. - **Outcome:** Merge decisions remain fast and deterministic by default, with optional LLM arbitration for cleaner sentence continuity. + +## Phase 22: Optional Speaker-Change Line Cutting +- **Goal:** Split long live captions earlier when a different person starts speaking. +- **Approach:** + - Added optional `--speaker-change-detect` mode with cosine-similarity comparison of lightweight voice signatures from buffered audio. + - Introduced tuning flags for sensitivity and runtime cost (`--speaker-sim-threshold`, `--speaker-min-buffer`, `--speaker-check-interval`). + - When a probable speaker switch is detected, the current buffered caption line is force-flushed early instead of waiting only for silence. + - Added speaker-state resets on watchdog/hallucination/silence reset paths to avoid stale identity drift. +- **Outcome:** Users can opt into faster caption segmentation at speaker boundaries while keeping default behavior unchanged. diff --git a/transcribe.py b/transcribe.py index 0b449ce..feee197 100644 --- a/transcribe.py +++ b/transcribe.py @@ -156,6 +156,51 @@ def quality_score(mean_logprob, max_compression): compression_penalty = 0.25 * max(0.0, (max_compression or 1.5) - 1.5) return mean_logprob - compression_penalty +def cosine_similarity(vec_a, vec_b): + if vec_a is None or vec_b is None: + return None + denom = (np.linalg.norm(vec_a) * np.linalg.norm(vec_b)) + if denom == 0: + return None + return float(np.dot(vec_a, vec_b) / denom) + +def compute_speaker_signature(audio, samplerate): + """Compute a lightweight speaker signature from averaged log spectrum.""" + if audio is None or len(audio) < int(0.5 * samplerate): + return None + + data = np.asarray(audio, dtype=np.float32) + if data.ndim > 1: + data = np.mean(data, axis=1) + + # Keep only voiced-ish samples to reduce silence/noise influence. + voiced = data[np.abs(data) > 0.01] + if len(voiced) < int(0.3 * samplerate): + voiced = data + + frame_size = 1024 + hop = 512 + if len(voiced) < frame_size: + return None + + window = np.hanning(frame_size).astype(np.float32) + spectra = [] + for i in range(0, len(voiced) - frame_size + 1, hop): + frame = voiced[i:i + frame_size] * window + mag = np.abs(np.fft.rfft(frame)) + if mag.size > 1: + mag = mag[1:257] # Focus on lower-frequency envelope. + spectra.append(np.log1p(mag)) + + if not spectra: + return None + + signature = np.mean(np.stack(spectra, axis=0), axis=0) + norm = np.linalg.norm(signature) + if norm == 0: + return None + return signature / norm + def should_retry_transcription(text, mean_logprob, max_compression, args): if not text: return False @@ -480,6 +525,10 @@ def main(): parser.add_argument("--post-correct-llm-timeout", type=float, default=3.0, help="Timeout seconds for local LLM post-correction (default: 3.0)") parser.add_argument("--llm-merge-decider", action="store_true", help="Use local LLM to validate/refine smart-correct line merges") parser.add_argument("--llm-merge-timeout", type=float, default=0.7, help="Timeout seconds for LLM merge decision (default: 0.7)") + parser.add_argument("--speaker-change-detect", action="store_true", help="Enable lightweight speaker-change detection to force caption line cuts") + parser.add_argument("--speaker-sim-threshold", type=float, default=0.72, help="Cosine similarity threshold below which a speaker change is assumed (default: 0.72)") + parser.add_argument("--speaker-min-buffer", type=float, default=1.6, help="Minimum buffered speech seconds before speaker-change checks (default: 1.6)") + parser.add_argument("--speaker-check-interval", type=float, default=0.6, help="Seconds between speaker-change checks while buffering (default: 0.6)") args = parser.parse_args() if args.post_correct_llm and not args.post_correct: @@ -496,6 +545,11 @@ def main(): if args.llm_merge_decider and not args.post_correct_llm: print("[SYSTEM]: --llm-merge-decider requires --post-correct-llm. Falling back to heuristic merge.") args.llm_merge_decider = False + if args.speaker_change_detect: + print( + "Speaker-change detection enabled " + f"(threshold={args.speaker_sim_threshold}, min_buffer={args.speaker_min_buffer}s)." + ) if args.quantize: WHISPER_MODEL = "mlx-community/whisper-small-mlx-4bit" @@ -578,6 +632,8 @@ def main(): last_draft_text = "" recent_en_lines = deque(maxlen=2) llm_post_correct_state = {"available": True, "warned": False, "merge_warned": False} + speaker_reference_signature = None + last_speaker_check_time = 0.0 try: with sd.InputStream(samplerate=SAMPLERATE, channels=args.channels, callback=callback, blocksize=BLOCK_SIZE, device=device_index): @@ -617,6 +673,7 @@ def main(): rolling_context = "" rolling_context_en = "" recent_en_lines.clear() + speaker_reference_signature = None last_draft_text = "" last_change_time = time.time() if args.stream: @@ -628,8 +685,31 @@ def main(): speech_started = True last_end = speech_timestamps[-1]['end'] buffer_len_samples = len(current_audio) + force_flush_for_speaker_change = False + + if args.speaker_change_detect and buffer_duration >= args.speaker_min_buffer: + now = time.time() + if (now - last_speaker_check_time) >= args.speaker_check_interval: + last_speaker_check_time = now + tail_secs = min(2.0, buffer_duration) + tail_samples = int(tail_secs * SAMPLERATE) + recent_signature = compute_speaker_signature(current_audio[-tail_samples:], SAMPLERATE) + if speaker_reference_signature is None and recent_signature is not None: + speaker_reference_signature = recent_signature + elif recent_signature is not None: + similarity = cosine_similarity(speaker_reference_signature, recent_signature) + if similarity is not None and similarity < args.speaker_sim_threshold: + force_flush_for_speaker_change = True + print( + f"\n[SYSTEM]: Speaker change detected (similarity={similarity:.2f}). " + "Cutting current line." + ) - if (buffer_len_samples - last_end) > (SAMPLERATE * args.silence / 1000) or buffer_len_samples > buffer_limit: + if ( + (buffer_len_samples - last_end) > (SAMPLERATE * args.silence / 1000) + or buffer_len_samples > buffer_limit + or force_flush_for_speaker_change + ): # Clear draft line if it was used if args.stream: @@ -680,6 +760,7 @@ def main(): rolling_context = "" rolling_context_en = "" recent_en_lines.clear() + speaker_reference_signature = None if original_text: print_caption(detected_lang, original_text, leading_newline=True) @@ -811,6 +892,7 @@ def main(): last_stream_time = time.time() stuck_draft_count = 0 last_draft_text = "" + speaker_reference_signature = None elif args.stream and (time.time() - last_stream_time) > 1.0: # Draft transcription @@ -850,6 +932,7 @@ def main(): rolling_context = "" rolling_context_en = "" recent_en_lines.clear() + speaker_reference_signature = None last_change_time = time.time() last_draft_text = "" sys.stdout.write("\r\033[K") @@ -860,6 +943,7 @@ def main(): elif not speech_started and len(current_audio) > SAMPLERATE * 2: audio_buffer = [] + speaker_reference_signature = None except KeyboardInterrupt: print("\nStopped by user.")