#!/usr/bin/env python3 """Record yourself once, transcribe with every engine, and compare. Every accuracy number in ACCURACY.md comes from synthesised speech, which is cleaner and more evenly paced than a person at a microphone, and from a vocabulary tuned to those same sentences. This runs the same comparison on your voice, in your room, which is the only measurement that decides anything. .venv/bin/python compare_engines.py # record and compare .venv/bin/python compare_engines.py --seconds 12 .venv/bin/python compare_engines.py --file clip.wav # reuse a recording .venv/bin/python compare_engines.py --say "the exact words you spoke" """ import argparse import json import subprocess import sys import time import wave from pathlib import Path HERE = Path(__file__).parent HELPER = HERE / "swift" / "speech-helper" RATE = 16000 def record(seconds: float, path: Path): import numpy as np import sounddevice as sd print(f"Recording {seconds:.0f}s — speak now, ideally something with jargon in it.") for count in (3, 2, 1): print(f" {count}...", end="\r", flush=True) time.sleep(0.6) print(" GO ") audio = sd.rec(int(seconds * RATE), samplerate=RATE, channels=1, dtype="int16") sd.wait() peak = int(np.abs(audio).max()) with wave.open(str(path), "wb") as w: w.setnchannels(1) w.setsampwidth(2) w.setframerate(RATE) w.writeframes(audio.tobytes()) print(f"Recorded, peak amplitude {peak}" + (" (very quiet — check the mic)" if peak < 1500 else "")) return peak def run_helper(path: Path, terms: list[str], dictation: bool) -> str: args = [str(HELPER), str(path), "--locale", "en-US"] if terms: args += ["--terms", ",".join(terms)] if dictation: args.append("--dictation") result = subprocess.run(args, capture_output=True, text=True, timeout=300) if result.returncode != 0: return f"" try: return json.loads(result.stdout).get("text", "") except json.JSONDecodeError: return "" def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--seconds", type=float, default=10.0) parser.add_argument("--file", help="Score an existing 16kHz mono WAV instead of recording.") parser.add_argument("--say", help="What you actually said, to score word error rate.") args = parser.parse_args() sys.path.insert(0, str(HERE)) from vocabulary import Vocabulary vocabulary = Vocabulary( project_dir=HERE, vocabulary_file=HERE / "vocabulary.txt", corrections_file=HERE / "corrections.txt", ) terms = vocabulary.terms() path = Path(args.file) if args.file else Path("/tmp/compare-engines.wav") if not args.file: record(args.seconds, path) engines = [ ("SpeechTranscriber (default)", lambda: run_helper(path, [], False)), ("SpeechTranscriber + repair", lambda: vocabulary.repair(run_helper(path, [], False))), ("DictationTranscriber + vocab", lambda: run_helper(path, terms, True)), ("DictationTranscriber + vocab + repair", lambda: vocabulary.repair(run_helper(path, terms, True))), ] try: from apple_stt import _recognize_file, timeout_for with wave.open(str(path)) as w: duration = w.getnframes() / w.getframerate() engines.append( ("old SFSpeechRecognizer + vocab + repair", lambda: vocabulary.repair(_recognize_file(str(path), "en-US", timeout_for(duration)))) ) except Exception: pass print() scorer = None if args.say: sys.path.insert(0, "/tmp") try: from wer2 import wer as scorer except ImportError: scorer = None for label, run in engines: started = time.time() try: text = run() except Exception as e: text = f"<{e}>" elapsed = time.time() - started suffix = "" if scorer and args.say and not text.startswith("<"): errors, total = scorer(args.say, text) suffix = f" [WER {100 * errors / max(total, 1):.0f}%]" print(f" {label}{suffix}") print(f" {text}") print(f" ({elapsed:.2f}s)\n") print("Pick whichever reads closest to what you said.") print(" default is SpeechTranscriber; --stt-engine apple or") print(" --analyzer-module dictation switch to the older model.") if __name__ == "__main__": main()