adolforeyna/whisper-translation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Apple Speech v3 + Freeflow polish + draft streaming

Browse Source 
- Apple SpeechAnalyzer (macOS 26+) binary: --bench (31x RTF), --pipe
  (persistent process, 150ms finals), --live (word-by-word drafts)
- Pipe protocol: 4-byte BE length + wav payload, emits JSONL
  {event:draft|final, text, isFinal, chunk} — 31 drafts for 6s audio (~60ms granularity)
- engine_apple_transcribe.py: ApplePipeTranscriber with
  transcribe() + transcribe_with_draft_callback(), VAD + draft
  queue, new flags --apple-stream (on), --apple-stream-interval,
  --apple-pipe (on). Fixes PIL/transformers import crash by lazy import.
- main_v3.py: engine selector {whisper,apple}, passthrough translate
  when no -es/-fr/-ar, freeflow flags same as v2
- Freeflow polish: deterministic punctuation commands (comma,
  question mark, new paragraph, at sign), filler stripping,
  <keep> protection, skip-clean heuristic, freeflow/qwen/legacy
  prompt styles. Much better final readability vs raw Apple/Whisper.
- main_v2.py, engine_llm.py, engine_distribute.py: integrate freeflow
- bench: Apple 2.12% WER vs Whisper Small 3.74% (Inscribe), CPU
  0mW ANE (measured via powermetrics), 196M EN cryptex per locale.
- Verified: 31 word-by-word drafts, 2 finals, exit 0, bench regression ok.

Freeflow still much better for final polish — Apple wins on speed
and raw accuracy, freeflow wins on readable paragraph output.

Co-authored-by: internal-model
This commit is contained in:
Adolfo Reyna
2026-07-13 21:08:18 -04:00
parent a2b108a5da
commit 80f0bf309f
19 changed files with 2276 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine_llm.py
+155 -70
View File
@@ -18,8 +18,19 @@ import os
import re
import json
import requests
import queue
from collections import deque
from datetime import datetime, timezone
from freeflow_polish import (
build_user_prompt,
deterministic_polish,
guard_against_truncation,
is_clean_enough_to_skip_llm,
match_input_casing,
normalize_formatting,
strip_keep_tags,
system_prompt_for_language,
)
def build_paragraph_messages(previous_refined_context, previous_source_text, new_source_text):
@@ -85,6 +96,15 @@ Hard rules:
], prompt
def build_freeflow_line_messages(text, language=None, model=""):
system_prompt = system_prompt_for_language(language, model=model)
prompt = build_user_prompt(text, language=language)
return [
{"role": "system", "content": system_prompt},
{"role": "user", "content": prompt},
], prompt
def append_llm_request_log(log_path, entry):
if not log_path:
return
@@ -229,11 +249,22 @@ def run_llm_prompt_test(args):
print(response)
if getattr(args, "llm_test_line", None):
messages, prompt = build_line_messages(
getattr(args, "llm_test_prev1", ""),
getattr(args, "llm_test_prev2", ""),
args.llm_test_line,
)
use_freeflow = getattr(args, "post_correct_prompt_style", "freeflow") != "legacy"
if use_freeflow:
preprocessed, substituted = deterministic_polish(args.llm_test_line)
messages, prompt = build_freeflow_line_messages(
substituted,
language=getattr(args, "lang", None),
model=getattr(args, "post_correct_model", ""),
)
metadata_input = preprocessed
else:
messages, prompt = build_line_messages(
getattr(args, "llm_test_prev1", ""),
getattr(args, "llm_test_prev2", ""),
args.llm_test_line,
)
metadata_input = args.llm_test_line
response = send_test_request(
"line",
messages,
@@ -242,7 +273,8 @@ def run_llm_prompt_test(args):
{
"prev1": getattr(args, "llm_test_prev1", ""),
"prev2": getattr(args, "llm_test_prev2", ""),
"input": args.llm_test_line,
"input": metadata_input,
"prompt_style": getattr(args, "post_correct_prompt_style", "freeflow"),
},
)
print("[LLM TEST] Line response:")
@@ -251,6 +283,7 @@ def run_llm_prompt_test(args):
if getattr(args, "llm_test_segments", None):
messages, prompt = build_paragraph_messages(
getattr(args, "llm_test_context", ""),
"",
args.llm_test_segments,
)
response = send_test_request(
@@ -328,6 +361,9 @@ def run_llm_processor(in_queue, out_queue, args):
return normalized
def apply_rule_based_post_correction(text):
if getattr(args, "freeflow_polish", True):
corrected, _ = deterministic_polish(text)
return corrected
corrected = normalize_english_caption(text)
corrected = re.sub(r"\s+", " ", corrected).strip()
corrected = re.sub(r"\s+([,.;!?])", r"\1", corrected)
@@ -417,13 +453,13 @@ def run_llm_processor(in_queue, out_queue, args):
entry["duration_ms"] = round((time.time() - started_at) * 1000, 2)
append_llm_request_log(log_path, entry)
# State: This is the ONLY text we send to the LLM as context
active_context = ""
# State: Keep a short rolling window of refined paragraphs as context.
refined_context_history = deque(maxlen=2)
previous_source_window = ""
recent_lines = deque(maxlen=2)
def call_llm_rolling_refine(new_segments_str, context, previous_source_text):
messages, prompt = build_paragraph_messages(context, previous_source_text, new_segments_str)
def call_llm_rolling_refine(new_segments_str, previous_refined_context, previous_source_text):
messages, prompt = build_paragraph_messages(previous_refined_context, previous_source_text, new_segments_str)
paragraph_options = {
"top_p": getattr(args, "llm_paragraph_top_p", 0.8),
"repeat_penalty": getattr(args, "llm_paragraph_repeat_penalty", 1.0),
@@ -440,20 +476,92 @@ def run_llm_processor(in_queue, out_queue, args):
if not getattr(args, "post_correct", False):
return normalize_english_caption(text)
corrected = apply_rule_based_post_correction(text)
corrected, substituted = deterministic_polish(text) if getattr(args, "freeflow_polish", True) else (apply_rule_based_post_correction(text), text)
if not getattr(args, "post_correct_llm", False):
return corrected
if getattr(args, "post_correct_skip_clean", True) and is_clean_enough_to_skip_llm(corrected):
if getattr(args, "post_correct_debug", False):
print("[LLM] Skipping line LLM polish; deterministic output looks clean.")
return corrected
prev2 = recent_lines[-2] if len(recent_lines) >= 2 else ""
prev1 = recent_lines[-1] if len(recent_lines) >= 1 else ""
messages, prompt = build_line_messages(prev1, prev2, corrected)
candidate = normalize_english_caption(call_llm(messages, prompt, 0.1, "line_warned"))
prompt_style = getattr(args, "post_correct_prompt_style", "freeflow")
if prompt_style == "legacy":
prev2 = recent_lines[-2] if len(recent_lines) >= 2 else ""
prev1 = recent_lines[-1] if len(recent_lines) >= 1 else ""
messages, prompt = build_line_messages(prev1, prev2, corrected)
else:
language = getattr(args, "lang", None)
messages, prompt = build_freeflow_line_messages(
substituted,
language=language,
model=args.post_correct_model if prompt_style == "qwen" else "",
)
candidate = call_llm(messages, prompt, 0.1, "line_warned").strip()
if prompt_style != "legacy":
fallback = guard_against_truncation(candidate, corrected)
if fallback:
return fallback
candidate = strip_keep_tags(candidate)
candidate = normalize_formatting(candidate)
candidate = match_input_casing(candidate, substituted)
else:
candidate = normalize_english_caption(candidate)
if candidate and word_overlap_ratio(corrected, candidate) >= getattr(args, "post_correct_min_overlap", 0.45):
return candidate
return corrected
pending_buffer = []
last_llm_call_time = time.time()
def flush_pending_outputs(processed_items):
nonlocal pending_buffer, previous_source_window
if not processed_items:
return
structured_paragraph = None
paragraph_from_llm = False
if args.llm_paragraph and pending_buffer:
new_batch = " ".join(pending_buffer)
previous_refined_context = "\n\n".join(refined_context_history)
result = call_llm_rolling_refine(new_batch, previous_refined_context, previous_source_window)
if result:
if "\n\n" in result:
parts = result.split("\n\n")
refined_context_history.clear()
for part in parts:
cleaned_part = part.strip()
if cleaned_part:
refined_context_history.append(cleaned_part)
structured_paragraph = result
else:
cleaned_result = result.strip()
if cleaned_result:
refined_context_history.append(cleaned_result)
structured_paragraph = result
paragraph_from_llm = True
else:
structured_paragraph = new_batch
cleaned_batch = new_batch.strip()
if cleaned_batch:
refined_context_history.append(cleaned_batch)
previous_source_window = new_batch
last_index = len(processed_items) - 1
for index, processed_item in enumerate(processed_items):
out_queue.put({
"raw": processed_item["raw"],
"corrected": processed_item["corrected"],
"paragraph": structured_paragraph if index == last_index else None,
"en_bridge": processed_item["en_bridge"],
"used_llm_line": bool(getattr(args, "post_correct_llm", False)),
"used_llm_paragraph": paragraph_from_llm if index == last_index else False,
"detected_lang": processed_item.get("detected_lang"),
"speaker": processed_item.get("speaker"),
"ts": processed_item.get("ts", time.time())
})
pending_buffer = []
while True:
try:
@@ -462,64 +570,41 @@ def run_llm_processor(in_queue, out_queue, args):
if "draft" in item:
out_queue.put(item)
continue
processed_items = []
pending_item = item
raw_text = item.get("original", "").strip()
bridge_text = item.get("en_bridge", raw_text).strip()
if not raw_text: continue
while pending_item is not None:
raw_text = pending_item.get("original", "").strip()
bridge_text = pending_item.get("en_bridge", raw_text).strip()
if raw_text:
corrected_text = post_correct_line(bridge_text)
corrected_text = post_correct_line(bridge_text)
# Hallucination check
words = corrected_text.lower().split()
if not (len(words) > 10 and len(set(words)) < 3):
pending_buffer.append(corrected_text)
recent_lines.append(corrected_text)
processed_items.append({
"raw": raw_text,
"corrected": corrected_text,
"en_bridge": bridge_text,
"detected_lang": pending_item.get("detected_lang"),
"speaker": pending_item.get("speaker"),
"ts": pending_item.get("ts", time.time()),
})
# Hallucination check
words = corrected_text.lower().split()
if len(words) > 10 and len(set(words)) < 3: continue
try:
pending_item = in_queue.get_nowait()
if pending_item is None:
flush_pending_outputs(processed_items)
return
if "draft" in pending_item:
out_queue.put(pending_item)
pending_item = None
except queue.Empty:
pending_item = None
pending_buffer.append(corrected_text)
recent_lines.append(corrected_text)
should_refine = False
if len(pending_buffer) >= 5: should_refine = True
elif len(pending_buffer) >= 2 and corrected_text.endswith((".", "?", "!")): should_refine = True
elif len(pending_buffer) >= 1 and (time.time() - last_llm_call_time) > 15: should_refine = True
structured_paragraph = None
paragraph_from_llm = False
if args.llm_paragraph and should_refine:
new_batch = " ".join(pending_buffer)
result = call_llm_rolling_refine(new_batch, active_context, previous_source_window)
if result:
# Logic to handle paragraph breaks
if "\n\n" in result:
# Split by double newline
parts = result.split("\n\n")
# The last part is the new "Active Context"
active_context = parts[-1].strip()
# The whole result is sent to the user (contains the break)
structured_paragraph = result
else:
# No break, just update the context
active_context = result
structured_paragraph = result
paragraph_from_llm = True
else:
structured_paragraph = new_batch
active_context = new_batch
previous_source_window = new_batch
pending_buffer = []
last_llm_call_time = time.time()
out_queue.put({
"raw": raw_text,
"corrected": corrected_text,
"paragraph": structured_paragraph,
"en_bridge": bridge_text,
"used_llm_line": bool(getattr(args, "post_correct_llm", False)),
"used_llm_paragraph": paragraph_from_llm,
"detected_lang": item.get("detected_lang"),
"speaker": item.get("speaker"),
"ts": item.get("ts", time.time())
})
flush_pending_outputs(processed_items)
except Exception as e:
print(f"[LLM] Error: {e}")