feat: Apple Speech v3 + Freeflow polish + draft streaming
- Apple SpeechAnalyzer (macOS 26+) binary: --bench (31x RTF), --pipe
(persistent process, 150ms finals), --live (word-by-word drafts)
- Pipe protocol: 4-byte BE length + wav payload, emits JSONL
{event:draft|final, text, isFinal, chunk} — 31 drafts for 6s audio (~60ms granularity)
- engine_apple_transcribe.py: ApplePipeTranscriber with
transcribe() + transcribe_with_draft_callback(), VAD + draft
queue, new flags --apple-stream (on), --apple-stream-interval,
--apple-pipe (on). Fixes PIL/transformers import crash by lazy import.
- main_v3.py: engine selector {whisper,apple}, passthrough translate
when no -es/-fr/-ar, freeflow flags same as v2
- Freeflow polish: deterministic punctuation commands (comma,
question mark, new paragraph, at sign), filler stripping,
<keep> protection, skip-clean heuristic, freeflow/qwen/legacy
prompt styles. Much better final readability vs raw Apple/Whisper.
- main_v2.py, engine_llm.py, engine_distribute.py: integrate freeflow
- bench: Apple 2.12% WER vs Whisper Small 3.74% (Inscribe), CPU
0mW ANE (measured via powermetrics), 196M EN cryptex per locale.
- Verified: 31 word-by-word drafts, 2 finals, exit 0, bench regression ok.
Freeflow still much better for final polish — Apple wins on speed
and raw accuracy, freeflow wins on readable paragraph output.
Co-authored-by: internal-model
This commit is contained in:
Adolfo Reyna
+242
@@ -0,0 +1,242 @@
|
||||
import multiprocessing
|
||||
import argparse
|
||||
import json as _json
|
||||
import os
|
||||
import sys
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
from engine_transcribe import run_transcription as run_whisper_transcription, list_audio_devices
|
||||
|
||||
def _lazy_apple():
|
||||
try:
|
||||
from engine_apple_transcribe import run_transcription as run_apple
|
||||
return run_apple
|
||||
except Exception as e:
|
||||
print(f"[Main] Apple engine import failed: {e}")
|
||||
return None
|
||||
|
||||
from engine_llm import run_llm_processor, run_llm_prompt_test
|
||||
# engine_translate is imported lazily inside main() to avoid PIL crash when not needed
|
||||
from engine_distribute import run_distribution
|
||||
|
||||
def _run_translation_passthrough(in_q, out_q, a):
|
||||
"""No MarianMT import — just map LLM output to distribution shape. Top-level for spawn pickling."""
|
||||
import time as _time
|
||||
while True:
|
||||
try:
|
||||
item = in_q.get()
|
||||
if item is None:
|
||||
break
|
||||
if "draft" in item:
|
||||
out_q.put(item)
|
||||
continue
|
||||
raw_text = item.get("raw")
|
||||
corrected = item.get("corrected")
|
||||
paragraph = item.get("paragraph")
|
||||
to_show = paragraph if getattr(a, "only_translate_llm", False) else (paragraph or corrected or raw_text)
|
||||
payload = {
|
||||
"original": raw_text,
|
||||
"corrected": corrected,
|
||||
"paragraph": paragraph,
|
||||
"en_bridge": item.get("en_bridge"),
|
||||
"english_output": to_show or "",
|
||||
"used_llm_line": item.get("used_llm_line", False),
|
||||
"used_llm_paragraph": item.get("used_llm_paragraph", False),
|
||||
"paragraph_fallback": bool(paragraph) and not item.get("used_llm_paragraph", False),
|
||||
"speaker": item.get("speaker"),
|
||||
"ts": item.get("ts", _time.time()),
|
||||
}
|
||||
if getattr(a, "en", False) and to_show:
|
||||
payload["en"] = to_show
|
||||
out_q.put(payload)
|
||||
except Exception as e:
|
||||
print(f"[Translate-passthrough] Error: {e}")
|
||||
|
||||
def parse_temperature_fallback(value):
|
||||
try:
|
||||
return tuple(float(x.strip()) for x in value.split(",") if x.strip())
|
||||
except:
|
||||
return (0.0, 0.2, 0.4, 0.6, 0.8, 1.0)
|
||||
|
||||
def main():
|
||||
config_path = "config.json"
|
||||
defaults = {}
|
||||
if os.path.exists(config_path):
|
||||
with open(config_path, "r") as f:
|
||||
defaults = _json.load(f)
|
||||
|
||||
parser = argparse.ArgumentParser(description="Multi-process Transcription and Translation (v3: whisper + apple).")
|
||||
|
||||
# Engine selector
|
||||
parser.add_argument("--engine", choices=["whisper", "apple"], default=defaults.get("engine", "whisper"),
|
||||
help="Transcription engine: whisper (mlx) or apple (SpeechAnalyzer macOS 26+) (default: whisper)")
|
||||
parser.add_argument("--apple-locale", type=str, default=defaults.get("apple_locale", None),
|
||||
help="Apple locale override e.g. en-US, es-ES, fr-FR (default: auto from --lang)")
|
||||
parser.add_argument("--apple-bench-file", type=str, default=None,
|
||||
help="Quick bench: transcribe a file with apple engine and exit")
|
||||
parser.add_argument("--apple-stream", action="store_true", default=defaults.get("apple_stream", True),
|
||||
help="Apple: enable draft streaming every ~1s while speaking (default: on)")
|
||||
parser.add_argument("--no-apple-stream", action="store_false", dest="apple_stream")
|
||||
parser.add_argument("--apple-stream-interval", type=float, default=defaults.get("apple_stream_interval", 1.0),
|
||||
help="Apple: seconds between draft transcribes (default: 1.0)")
|
||||
parser.add_argument("--apple-pipe", action="store_true", default=defaults.get("apple_pipe", True),
|
||||
help="Apple: keep one Swift process alive via --pipe (default: on, faster)")
|
||||
parser.add_argument("--no-apple-pipe", action="store_false", dest="apple_pipe")
|
||||
|
||||
# Transcribe Args (shared)
|
||||
parser.add_argument("-l", "--list-devices", action="store_true", help="Show available audio devices and exit")
|
||||
parser.add_argument("--model", type=str, default=defaults.get("model", "mlx-community/whisper-base-mlx"))
|
||||
parser.add_argument("--device", type=int, default=defaults.get("device"))
|
||||
parser.add_argument("--lang", type=str, default=defaults.get("lang"))
|
||||
parser.add_argument("--silence", type=int, default=defaults.get("silence", 1000))
|
||||
parser.add_argument("--max-buffer", type=int, default=defaults.get("max_buffer", 20))
|
||||
parser.add_argument("--channels", type=int, default=defaults.get("channels", 1))
|
||||
parser.add_argument("--filter-lang", action="store_true", default=defaults.get("filter_lang", False))
|
||||
parser.add_argument("-q", "--quantize", action="store_true", default=defaults.get("quantize", False))
|
||||
parser.add_argument("-s", "--stream", action="store_true", default=defaults.get("stream", False))
|
||||
parser.add_argument("-c", "--context", action="store_true", default=defaults.get("context", False))
|
||||
parser.add_argument("--speaker-diarization", action="store_true", default=defaults.get("speaker_diarization", False))
|
||||
parser.add_argument("--temperature-fallback", type=parse_temperature_fallback, default=tuple(defaults.get("temperature_fallback", [0.0, 0.2, 0.4, 0.6, 0.8, 1.0])))
|
||||
parser.add_argument("--logprob-threshold", type=float, default=defaults.get("logprob_threshold", -0.8))
|
||||
parser.add_argument("--compression-threshold", type=float, default=defaults.get("compression_threshold", 2.2))
|
||||
parser.add_argument("--verbose", "-v", action="store_true", default=defaults.get("verbose", False))
|
||||
|
||||
# LLM Args
|
||||
parser.add_argument("--post-correct", action="store_true", default=defaults.get("post_correct", False))
|
||||
parser.add_argument("--post-correct-llm", action="store_true", default=defaults.get("post_correct_llm", False))
|
||||
parser.add_argument("--post-correct-model", type=str, default=defaults.get("post_correct_model", "qwen3.5:0.8b"))
|
||||
parser.add_argument("--post-correct-ollama-url", type=str, default=defaults.get("post_correct_ollama_url", "http://127.0.0.1:11434/api/generate"))
|
||||
parser.add_argument("--post-correct-llm-timeout", type=float, default=defaults.get("post_correct_llm_timeout", 8.0))
|
||||
parser.add_argument("--post-correct-keep-alive", type=str, default=defaults.get("post_correct_keep_alive", "30m"))
|
||||
parser.add_argument("--post-correct-warmup-timeout", type=float, default=defaults.get("post_correct_warmup_timeout", 20.0))
|
||||
parser.add_argument("--post-correct-min-overlap", type=float, default=defaults.get("post_correct_min_overlap", 0.45))
|
||||
parser.add_argument("--post-correct-debug", action="store_true", default=defaults.get("post_correct_debug", False))
|
||||
parser.add_argument("--freeflow-polish", action="store_true", default=defaults.get("freeflow_polish", True))
|
||||
parser.add_argument("--no-freeflow-polish", action="store_false", dest="freeflow_polish")
|
||||
parser.add_argument("--post-correct-skip-clean", action="store_true", default=defaults.get("post_correct_skip_clean", True))
|
||||
parser.add_argument("--no-post-correct-skip-clean", action="store_false", dest="post_correct_skip_clean")
|
||||
parser.add_argument("--post-correct-prompt-style", choices=["freeflow", "qwen", "legacy"], default=defaults.get("post_correct_prompt_style", "freeflow"))
|
||||
parser.add_argument("--llm-paragraph", action="store_true", default=defaults.get("llm_paragraph", False))
|
||||
parser.add_argument("--llm-paragraph-temperature", type=float, default=defaults.get("llm_paragraph_temperature", 0.2))
|
||||
parser.add_argument("--llm-paragraph-top-p", type=float, default=defaults.get("llm_paragraph_top_p", 0.8))
|
||||
parser.add_argument("--llm-paragraph-repeat-penalty", type=float, default=defaults.get("llm_paragraph_repeat_penalty", 1.0))
|
||||
parser.add_argument("--llm-request-log-path", type=str, default=defaults.get("llm_request_log_path", "logs/llm_requests.jsonl"))
|
||||
parser.add_argument("--llm-test-line", type=str, default=None)
|
||||
parser.add_argument("--llm-test-prev1", type=str, default="")
|
||||
parser.add_argument("--llm-test-prev2", type=str, default="")
|
||||
parser.add_argument("--llm-test-segments", type=str, default=None)
|
||||
parser.add_argument("--llm-test-context", type=str, default="")
|
||||
parser.add_argument("--llm-test-from-log", type=int, default=None)
|
||||
parser.add_argument("--llm-test-use-logged-model", action="store_true", default=False)
|
||||
|
||||
# Translate Args
|
||||
parser.add_argument("-es", action="store_true", default=defaults.get("es", False))
|
||||
parser.add_argument("-fr", action="store_true", default=defaults.get("fr", False))
|
||||
parser.add_argument("-ar", action="store_true", default=defaults.get("ar", False))
|
||||
parser.add_argument("-en", action="store_true", default=defaults.get("en", False))
|
||||
parser.add_argument("--only-translate-llm", action="store_true", default=defaults.get("only_translate_llm", False))
|
||||
parser.add_argument("--mt-max-chars", type=int, default=defaults.get("mt_max_chars", 250))
|
||||
parser.add_argument("--mt-max-new-tokens", type=int, default=defaults.get("mt_max_new_tokens", 150))
|
||||
parser.add_argument("--mt-num-beams", type=int, default=defaults.get("mt_num_beams", 4))
|
||||
|
||||
# Distribute
|
||||
parser.add_argument("-i", "--ingest", action="store_true", default=defaults.get("ingest", False))
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Auto-enable
|
||||
if args.post_correct_llm and not args.post_correct:
|
||||
args.post_correct = True
|
||||
if args.only_translate_llm and not args.llm_paragraph:
|
||||
print("[Main] Enabling --llm-paragraph because --only-translate-llm was requested.")
|
||||
args.llm_paragraph = True
|
||||
|
||||
# Apple bench file shortcut
|
||||
if args.apple_bench_file:
|
||||
from engine_apple_transcribe import resolve_binary, resolve_locale
|
||||
import subprocess, json
|
||||
binary = resolve_binary()
|
||||
if not binary:
|
||||
print("Apple binary not found. Run cd apple_speech && bash build.sh")
|
||||
sys.exit(1)
|
||||
locale = resolve_locale(args.lang, args.apple_locale)
|
||||
cmd = [str(binary), "--bench", args.apple_bench_file, "--locale", locale, "-v"]
|
||||
print(f"[Main] Running: {' '.join(cmd)}")
|
||||
subprocess.run(cmd)
|
||||
return
|
||||
|
||||
if args.llm_test_line or args.llm_test_segments or args.llm_test_from_log is not None:
|
||||
run_llm_prompt_test(args)
|
||||
return
|
||||
|
||||
if args.list_devices:
|
||||
list_audio_devices()
|
||||
return
|
||||
|
||||
if args.device is None:
|
||||
list_audio_devices()
|
||||
try:
|
||||
val = input("\nSelect input device index (or press Enter for default): ").strip()
|
||||
if val:
|
||||
args.device = int(val)
|
||||
except EOFError:
|
||||
pass
|
||||
except ValueError:
|
||||
print("[Main] Invalid index, using default.")
|
||||
|
||||
# Choose engine
|
||||
if args.engine == "apple":
|
||||
run_transcription_fn = _lazy_apple()
|
||||
if run_transcription_fn is None:
|
||||
print("[Main] Falling back to whisper because apple engine unavailable")
|
||||
run_transcription_fn = run_whisper_transcription
|
||||
else:
|
||||
run_transcription_fn = run_whisper_transcription
|
||||
|
||||
print(f"[Main] Engine: {args.engine}")
|
||||
|
||||
needs_translate = bool(args.es or args.fr or args.ar or args.en)
|
||||
q_trans_to_llm = multiprocessing.Queue()
|
||||
q_llm_to_tl = multiprocessing.Queue()
|
||||
q_tl_to_dist = multiprocessing.Queue()
|
||||
|
||||
p_transcribe = multiprocessing.Process(target=run_transcription_fn, args=(q_trans_to_llm, args))
|
||||
p_llm = multiprocessing.Process(target=run_llm_processor, args=(q_trans_to_llm, q_llm_to_tl, args))
|
||||
|
||||
if not needs_translate:
|
||||
print("[Main] Translate disabled (no -es/-fr/-ar/-en) — using passthrough (no MarianMT/PIL)")
|
||||
p_translate = multiprocessing.Process(target=_run_translation_passthrough, args=(q_llm_to_tl, q_tl_to_dist, args))
|
||||
else:
|
||||
try:
|
||||
from engine_translate import run_translation as _real_translate
|
||||
p_translate = multiprocessing.Process(target=_real_translate, args=(q_llm_to_tl, q_tl_to_dist, args))
|
||||
except Exception as e:
|
||||
print(f"[Main] Translate import failed ({e}) — falling back to passthrough")
|
||||
p_translate = multiprocessing.Process(target=_run_translation_passthrough, args=(q_llm_to_tl, q_tl_to_dist, args))
|
||||
|
||||
p_distribute = multiprocessing.Process(target=run_distribution, args=(q_tl_to_dist, args))
|
||||
|
||||
print("[Main] Starting processes...")
|
||||
p_transcribe.start()
|
||||
p_llm.start()
|
||||
p_translate.start()
|
||||
p_distribute.start()
|
||||
|
||||
try:
|
||||
p_transcribe.join()
|
||||
p_llm.join()
|
||||
p_translate.join()
|
||||
p_distribute.join()
|
||||
except KeyboardInterrupt:
|
||||
print("\n[Main] Stopping processes...")
|
||||
p_transcribe.terminate()
|
||||
p_llm.terminate()
|
||||
p_translate.terminate()
|
||||
p_distribute.terminate()
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == "__main__":
|
||||
multiprocessing.freeze_support()
|
||||
main()
|
||||
Reference in New Issue
Block a user