Initial commit of current state

compare_engines.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""Record yourself once, transcribe with every engine, and compare.
+
+Every accuracy number in ACCURACY.md comes from synthesised speech, which is
+cleaner and more evenly paced than a person at a microphone, and from a
+vocabulary tuned to those same sentences. This runs the same comparison on your
+voice, in your room, which is the only measurement that decides anything.
+
+    .venv/bin/python compare_engines.py                  # record and compare
+    .venv/bin/python compare_engines.py --seconds 12
+    .venv/bin/python compare_engines.py --file clip.wav  # reuse a recording
+    .venv/bin/python compare_engines.py --say "the exact words you spoke"
+"""
+
+import argparse
+import json
+import subprocess
+import sys
+import time
+import wave
+from pathlib import Path
+
+HERE = Path(__file__).parent
+HELPER = HERE / "swift" / "speech-helper"
+RATE = 16000
+
+
+def record(seconds: float, path: Path):
+    import numpy as np
+    import sounddevice as sd
+
+    print(f"Recording {seconds:.0f}s — speak now, ideally something with jargon in it.")
+    for count in (3, 2, 1):
+        print(f"  {count}...", end="\r", flush=True)
+        time.sleep(0.6)
+    print("  GO           ")
+    audio = sd.rec(int(seconds * RATE), samplerate=RATE, channels=1, dtype="int16")
+    sd.wait()
+    peak = int(np.abs(audio).max())
+    with wave.open(str(path), "wb") as w:
+        w.setnchannels(1)
+        w.setsampwidth(2)
+        w.setframerate(RATE)
+        w.writeframes(audio.tobytes())
+    print(f"Recorded, peak amplitude {peak}" + ("  (very quiet — check the mic)" if peak < 1500 else ""))
+    return peak
+
+
+def run_helper(path: Path, terms: list[str], dictation: bool) -> str:
+    args = [str(HELPER), str(path), "--locale", "en-US"]
+    if terms:
+        args += ["--terms", ",".join(terms)]
+    if dictation:
+        args.append("--dictation")
+    result = subprocess.run(args, capture_output=True, text=True, timeout=300)
+    if result.returncode != 0:
+        return f"<helper failed: {result.returncode}>"
+    try:
+        return json.loads(result.stdout).get("text", "")
+    except json.JSONDecodeError:
+        return "<unreadable helper output>"
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--seconds", type=float, default=10.0)
+    parser.add_argument("--file", help="Score an existing 16kHz mono WAV instead of recording.")
+    parser.add_argument("--say", help="What you actually said, to score word error rate.")
+    args = parser.parse_args()
+
+    sys.path.insert(0, str(HERE))
+    from vocabulary import Vocabulary
+
+    vocabulary = Vocabulary(
+        project_dir=HERE,
+        vocabulary_file=HERE / "vocabulary.txt",
+        corrections_file=HERE / "corrections.txt",
+    )
+    terms = vocabulary.terms()
+
+    path = Path(args.file) if args.file else Path("/tmp/compare-engines.wav")
+    if not args.file:
+        record(args.seconds, path)
+
+    engines = [
+        ("SpeechTranscriber (default)", lambda: run_helper(path, [], False)),
+        ("SpeechTranscriber + repair", lambda: vocabulary.repair(run_helper(path, [], False))),
+        ("DictationTranscriber + vocab", lambda: run_helper(path, terms, True)),
+        ("DictationTranscriber + vocab + repair",
+         lambda: vocabulary.repair(run_helper(path, terms, True))),
+    ]
+    try:
+        from apple_stt import _recognize_file, timeout_for
+
+        with wave.open(str(path)) as w:
+            duration = w.getnframes() / w.getframerate()
+        engines.append(
+            ("old SFSpeechRecognizer + vocab + repair",
+             lambda: vocabulary.repair(_recognize_file(str(path), "en-US", timeout_for(duration))))
+        )
+    except Exception:
+        pass
+
+    print()
+    scorer = None
+    if args.say:
+        sys.path.insert(0, "/tmp")
+        try:
+            from wer2 import wer as scorer
+        except ImportError:
+            scorer = None
+
+    for label, run in engines:
+        started = time.time()
+        try:
+            text = run()
+        except Exception as e:
+            text = f"<{e}>"
+        elapsed = time.time() - started
+        suffix = ""
+        if scorer and args.say and not text.startswith("<"):
+            errors, total = scorer(args.say, text)
+            suffix = f"   [WER {100 * errors / max(total, 1):.0f}%]"
+        print(f"  {label}{suffix}")
+        print(f"    {text}")
+        print(f"    ({elapsed:.2f}s)\n")
+
+    print("Pick whichever reads closest to what you said.")
+    print("  default is SpeechTranscriber; --stt-engine apple or")
+    print("  --analyzer-module dictation switch to the older model.")
+
+
+if __name__ == "__main__":
+    main()