feat: add optional speaker-change detection for line cuts

history.md

@@ -176,3 +176,12 @@ The project now features a high-performance, Apple Silicon-optimized pipeline th
     - Enforced strict low-latency behavior with `--llm-merge-timeout` (default 0.7s) and automatic fallback to heuristic merges on errors/timeouts.
     - Added safety guards to reject overly large LLM rewrites and preserve buffered caption state when merge is rejected.
 - **Outcome:** Merge decisions remain fast and deterministic by default, with optional LLM arbitration for cleaner sentence continuity.
+
+## Phase 22: Optional Speaker-Change Line Cutting
+- **Goal:** Split long live captions earlier when a different person starts speaking.
+- **Approach:**
+    - Added optional `--speaker-change-detect` mode with cosine-similarity comparison of lightweight voice signatures from buffered audio.
+    - Introduced tuning flags for sensitivity and runtime cost (`--speaker-sim-threshold`, `--speaker-min-buffer`, `--speaker-check-interval`).
+    - When a probable speaker switch is detected, the current buffered caption line is force-flushed early instead of waiting only for silence.
+    - Added speaker-state resets on watchdog/hallucination/silence reset paths to avoid stale identity drift.
+- **Outcome:** Users can opt into faster caption segmentation at speaker boundaries while keeping default behavior unchanged.

transcribe.py

@@ -156,6 +156,51 @@ def quality_score(mean_logprob, max_compression):
     compression_penalty = 0.25 * max(0.0, (max_compression or 1.5) - 1.5)
     return mean_logprob - compression_penalty
 
+def cosine_similarity(vec_a, vec_b):
+    if vec_a is None or vec_b is None:
+        return None
+    denom = (np.linalg.norm(vec_a) * np.linalg.norm(vec_b))
+    if denom == 0:
+        return None
+    return float(np.dot(vec_a, vec_b) / denom)
+
+def compute_speaker_signature(audio, samplerate):
+    """Compute a lightweight speaker signature from averaged log spectrum."""
+    if audio is None or len(audio) < int(0.5 * samplerate):
+        return None
+
+    data = np.asarray(audio, dtype=np.float32)
+    if data.ndim > 1:
+        data = np.mean(data, axis=1)
+
+    # Keep only voiced-ish samples to reduce silence/noise influence.
+    voiced = data[np.abs(data) > 0.01]
+    if len(voiced) < int(0.3 * samplerate):
+        voiced = data
+
+    frame_size = 1024
+    hop = 512
+    if len(voiced) < frame_size:
+        return None
+
+    window = np.hanning(frame_size).astype(np.float32)
+    spectra = []
+    for i in range(0, len(voiced) - frame_size + 1, hop):
+        frame = voiced[i:i + frame_size] * window
+        mag = np.abs(np.fft.rfft(frame))
+        if mag.size > 1:
+            mag = mag[1:257]  # Focus on lower-frequency envelope.
+            spectra.append(np.log1p(mag))
+
+    if not spectra:
+        return None
+
+    signature = np.mean(np.stack(spectra, axis=0), axis=0)
+    norm = np.linalg.norm(signature)
+    if norm == 0:
+        return None
+    return signature / norm
+
 def should_retry_transcription(text, mean_logprob, max_compression, args):
     if not text:
         return False
@@ -480,6 +525,10 @@ def main():
     parser.add_argument("--post-correct-llm-timeout", type=float, default=3.0, help="Timeout seconds for local LLM post-correction (default: 3.0)")
     parser.add_argument("--llm-merge-decider", action="store_true", help="Use local LLM to validate/refine smart-correct line merges")
     parser.add_argument("--llm-merge-timeout", type=float, default=0.7, help="Timeout seconds for LLM merge decision (default: 0.7)")
+    parser.add_argument("--speaker-change-detect", action="store_true", help="Enable lightweight speaker-change detection to force caption line cuts")
+    parser.add_argument("--speaker-sim-threshold", type=float, default=0.72, help="Cosine similarity threshold below which a speaker change is assumed (default: 0.72)")
+    parser.add_argument("--speaker-min-buffer", type=float, default=1.6, help="Minimum buffered speech seconds before speaker-change checks (default: 1.6)")
+    parser.add_argument("--speaker-check-interval", type=float, default=0.6, help="Seconds between speaker-change checks while buffering (default: 0.6)")
     
     args = parser.parse_args()
     if args.post_correct_llm and not args.post_correct:
@@ -496,6 +545,11 @@ def main():
     if args.llm_merge_decider and not args.post_correct_llm:
         print("[SYSTEM]: --llm-merge-decider requires --post-correct-llm. Falling back to heuristic merge.")
         args.llm_merge_decider = False
+    if args.speaker_change_detect:
+        print(
+            "Speaker-change detection enabled "
+            f"(threshold={args.speaker_sim_threshold}, min_buffer={args.speaker_min_buffer}s)."
+        )
 
     if args.quantize:
         WHISPER_MODEL = "mlx-community/whisper-small-mlx-4bit"
@@ -578,6 +632,8 @@ def main():
     last_draft_text = ""
     recent_en_lines = deque(maxlen=2)
     llm_post_correct_state = {"available": True, "warned": False, "merge_warned": False}
+    speaker_reference_signature = None
+    last_speaker_check_time = 0.0
 
     try:
         with sd.InputStream(samplerate=SAMPLERATE, channels=args.channels, callback=callback, blocksize=BLOCK_SIZE, device=device_index):
@@ -617,6 +673,7 @@ def main():
                         rolling_context = ""
                         rolling_context_en = ""
                         recent_en_lines.clear()
+                        speaker_reference_signature = None
                         last_draft_text = ""
                         last_change_time = time.time()
                         if args.stream:
@@ -628,8 +685,31 @@ def main():
                         speech_started = True
                         last_end = speech_timestamps[-1]['end']
                         buffer_len_samples = len(current_audio)
+                        force_flush_for_speaker_change = False
+
+                        if args.speaker_change_detect and buffer_duration >= args.speaker_min_buffer:
+                            now = time.time()
+                            if (now - last_speaker_check_time) >= args.speaker_check_interval:
+                                last_speaker_check_time = now
+                                tail_secs = min(2.0, buffer_duration)
+                                tail_samples = int(tail_secs * SAMPLERATE)
+                                recent_signature = compute_speaker_signature(current_audio[-tail_samples:], SAMPLERATE)
+                                if speaker_reference_signature is None and recent_signature is not None:
+                                    speaker_reference_signature = recent_signature
+                                elif recent_signature is not None:
+                                    similarity = cosine_similarity(speaker_reference_signature, recent_signature)
+                                    if similarity is not None and similarity < args.speaker_sim_threshold:
+                                        force_flush_for_speaker_change = True
+                                        print(
+                                            f"\n[SYSTEM]: Speaker change detected (similarity={similarity:.2f}). "
+                                            "Cutting current line."
+                                        )
                         
-                        if (buffer_len_samples - last_end) > (SAMPLERATE * args.silence / 1000) or buffer_len_samples > buffer_limit:
+                        if (
+                            (buffer_len_samples - last_end) > (SAMPLERATE * args.silence / 1000)
+                            or buffer_len_samples > buffer_limit
+                            or force_flush_for_speaker_change
+                        ):
                             
                             # Clear draft line if it was used
                             if args.stream:
@@ -680,6 +760,7 @@ def main():
                                 rolling_context = ""
                                 rolling_context_en = ""
                                 recent_en_lines.clear()
+                                speaker_reference_signature = None
 
                             if original_text:
                                 print_caption(detected_lang, original_text, leading_newline=True)
@@ -811,6 +892,7 @@ def main():
                             last_stream_time = time.time()
                             stuck_draft_count = 0
                             last_draft_text = ""
+                            speaker_reference_signature = None
 
                         elif args.stream and (time.time() - last_stream_time) > 1.0:
                             # Draft transcription
@@ -850,6 +932,7 @@ def main():
                                     rolling_context = ""
                                     rolling_context_en = ""
                                     recent_en_lines.clear()
+                                    speaker_reference_signature = None
                                     last_change_time = time.time()
                                     last_draft_text = ""
                                     sys.stdout.write("\r\033[K")
@@ -860,6 +943,7 @@ def main():
                     
                     elif not speech_started and len(current_audio) > SAMPLERATE * 2:
                         audio_buffer = []
+                        speaker_reference_signature = None
 
     except KeyboardInterrupt:
         print("\nStopped by user.")