commit 62d805e565ac8ca6f63e14fda7d0d648fb451ea8 Author: Adolfo Reyna Date: Fri Aug 7 18:15:36 2026 -0400 Initial commit of current state diff --git a/ACCURACY.md b/ACCURACY.md new file mode 100644 index 0000000..6998f64 --- /dev/null +++ b/ACCURACY.md @@ -0,0 +1,125 @@ +# Transcription accuracy + +What was measured, what shipped, and what turned out to be wrong along the way. + +Benchmark throughout: eight sentences of technical speech (85 words) containing +the vocabulary this project actually uses — pipecat, Kokoro, Metamate, +Phabricator, fbsource, SFSpeechRecognizer. The audio is synthesised, so it is +cleaner and more evenly paced than a person at a microphone. Trust the ordering, +not the absolute rates, and prefer `compare_engines.py` on your own voice. + +Scoring follows the [Inscribe +benchmark](https://get-inscribe.com/blog/apple-speech-api-benchmark.html): +lowercased, depunctuated, contractions expanded and **digits written out**. That +last part matters — see the correction below. + +## Where it ended up + +| Configuration | WER | +| --- | --- | +| old `SFSpeechRecognizer`, no help | 20.0% | +| `SpeechTranscriber`, no help | 16.5% | +| old + vocabulary + repair rules | 10.6% | +| `DictationTranscriber` + vocabulary + repair | 9.4% | +| **`SpeechTranscriber` + repair — the default** | **9.4%** | + +The top two tie, but not at equal cost. `SpeechTranscriber` reaches 9.4% with +**five repair rules and no vocabulary**, where the old model needs 39 curated +terms *and* seven rules to match it. It is also better with no configuration at +all, which is the state anyone starts in. + +## The two mistakes I made measuring this + +**Scoring punished the new model for formatting.** My first normaliser +lowercased and stripped punctuation but left digits alone, so +`SpeechTranscriber` was charged errors for writing "400 milliseconds" instead of +"four hundred milliseconds", and "SF speech recognizer" instead of +"SFSpeechRecognizer". Neither is a mishearing. The Inscribe benchmark names this +exact trap. Rescored properly, a 12.9%-vs-14.1% loss for the new model became a +9.4% tie that it wins on effort. + +**"Dilution is cheap" was wrong.** Padding the term list with a thousand random +dictionary words cost only 1.1 points, so auto-generating 100 terms from project +filenames looked safe. It erased the entire benefit — back to no-biasing levels +— because filenames like `bot`, `plan` and `hack` are ordinary English, and +boosting ordinary words drags correct speech onto them. Obscure words are inert +by comparison. Discovered terms are now filtered against the system dictionary, +capped at 12, and ranked behind the hand-written list. + +## What each mechanism is worth + +**Vocabulary biasing** (`vocabulary.txt` → `contextualStrings`) removed 30% of +errors on the old model and costs nothing at runtime. `SpeechTranscriber` +**ignores it** — output is byte-identical with and without terms — so on the +default engine the file now serves only Claude's system prompt and the fallback +engines. Keep it short and specific. + +**Repair rules** (`corrections.txt`) are deterministic substitutions for +mistakes that recur identically. Worth more on the new engine than the old, +because its errors land phonetically close to the target — "Kakoro" for Kokoro, +where the old model produced "coral". Five rules do what 39 terms plus seven +rules do elsewhere. + +**LLM post-correction** took 16.5% to 7.1% on the old engine, and 5-best beat +1-best (7.1% against 8.2%). Not worth a separate pass on the conversation path, +since Claude already reads every transcript — the vocabulary goes into his +system prompt instead, which is free. Worth a real pass for meeting notes. + +**Punctuation and task hint** on the old API measurably do nothing for accuracy. +The new engine punctuates and capitalises automatically. + +**Noise beats all of it.** At 10 dB SNR, WER nearly tripled to 69% and neither +vocabulary, high-pass filtering nor gain normalisation recovered any of it. The +microphone is the highest-leverage component in the pipeline. + +## Which model is which + +| | old `SFSpeechRecognizer` | `DictationTranscriber` | `SpeechTranscriber` | +| --- | --- | --- | --- | +| Model | old | **byte-identical to old** | **new** | +| Honours vocabulary | yes | yes | no | +| Long audio | needs stitching | native | native | +| Punctuation | no | no | automatic | +| Best measured | 10.6% | 9.4% | **9.4%, far less tuning** | + +`DictationTranscriber` is not a different model: it produces character-for- +character identical output to `SFSpeechRecognizer` on the same audio, verified +sentence by sentence rather than inferred from the docs. It is the old +recogniser reached through the new API, and its only advantage is vocabulary +support. + +Both new-API modules handle long speech natively — 40 of 40 sentences recovered +from 83 seconds — so the transcript stitching in `apple_stt.py` is only needed +on the legacy fallback path. + +For scale, the published benchmark measured 5,559 LibriSpeech utterances on an +M2 Pro: SpeechAnalyzer 2.12% clean and 4.56% noisy, against SFSpeechRecognizer's +9.02% and 16.25% — the legacy engine scoring worse than Whisper Tiny. It +validates its harness by reproducing OpenAI's published Whisper numbers within ++0.11 to +0.42. Caveats: English read speech, one machine. + +## Ruled out + +**Denoising.** *When De-noising Hurts* ([arXiv:2512.17562](https://arxiv.org/abs/2512.17562)) +applied MetricGAN+ across 500 recordings and 10 conditions: **all 40 +configurations got worse**, Whisper degrading 8.82% → 25.83% at 10 dB SNR, and +even clean audio losing 1.3–3.2 points. So Pipecat's `RNNoiseFilter` is not +worth the dependency surgery it needs, and the commercial filters (Koala, Krisp, +AIC) are the same bet with a licence. Worth checking macOS Voice Isolation is +off, since it is the same class of processing applied for free. + +**Custom language models.** `SFSpeechLanguageModel` trains on example sentences +and supports X-SAMPA pronunciations, so it should beat `contextualStrings` — but +its data builder is Swift-only, and it cannot be combined with +`SpeechTranscriber` anyway. + +## Still open + +- **Real speech.** Every number here is synthesised audio with config files + tuned around it. `compare_engines.py` records you and runs every engine. +- **The external microphone**, versus the built-in one. Probably the largest + real-world effect available and free to test. +- **5-best into Claude.** `--alternatives` exists in the helper and is unused; + measured 8.2% → 7.1% on the old engine. +- **Confidence-gated clarification** — ask again rather than answer a + low-confidence guess. diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..c31e01f --- /dev/null +++ b/PLAN.md @@ -0,0 +1,203 @@ +# Plan: meeting notes, and shipping this as an internal Mac app + +Two separate pieces of work. The meeting-notes feature is buildable today. The +packaging work is mostly not a coding problem — it's a signing, permissions and +distribution problem, and one policy question that should be answered before +any of it starts. + +Nothing here is implemented yet. + +--- + +## Part 1 — Transcribing the other side of a meeting + +### Is it possible? + +Yes. macOS 26 has two ways to capture audio that is being *played*, neither of +which needs a virtual audio device like BlackHole or Loopback: + +| Approach | API | Notes | +| --- | --- | --- | +| **Core Audio process tap** | `AudioHardwareCreateProcessTap` + `CATapDescription` | Confirmed present in the installed SDK (`AudioHardwareTapping.h`). Can tap one process — e.g. only Zoom — rather than everything. Preferred. | +| **ScreenCaptureKit** | `SCStream` with audio only | `pyobjc-framework-ScreenCaptureKit` 12.2.1 is on PyPI. Simpler bindings, but conceptually "screen recording", and captures system-wide. | + +Both need a TCC grant (Screen Recording / Audio Recording), which is currently +not granted. Same one-time prompt as the other permissions. + +### Design sketch + +Run it as a **second, independent pipeline** in the same process, not as a +branch of the conversation pipeline. It has a different job, a different +lifetime, and must not interfere with turn taking. + +``` +Zoom process audio ──► tap ──► Apple STT ──► transcript writer ──► notes/2026-08-07-standup.md +your microphone ──► tap ──► Apple STT ──┘ +``` + +Points that matter: + +- **Tap the mic separately from the output.** Two streams gives you speaker + attribution for free — output is "them", mic is "you" — without any + diarization model. This is the single biggest quality win available and it's + nearly free. +- **Reuse `apple_stt.py`.** It already transcribes accurately at 0.05–0.10 s + per utterance, and needs no model download. +- **Long-form transcription needs chunking.** `SFSpeechRecognizer` is built for + utterances, not hour-long meetings; recognition tasks end on their own. The + notes pipeline will need to segment on silence and restart tasks, and that + behaviour needs to be verified against a real long meeting before trusting + it. This is the main technical unknown. +- macOS 26 also ships `SpeechAnalyzer`/`SpeechTranscriber`, which is *designed* + for long-form audio and would be a better fit — but it's a Swift-only API and + unreachable from pyobjc. See the packaging section; this is one of the + stronger arguments for a Swift shell later. + +### Turning it on and off + +Two triggers, and they're not equivalent: + +- **Automatic on Zoom.** Detection is trivial — Zoom is running right now + (`/Applications/zoom.us.app`, pid 89302) and installs its own + `ZoomAudioDevice` Core Audio driver. But *process running* is a poor signal: + Zoom idles in the background for hours. Better to trigger on Zoom actually + holding an active audio stream, or on a calendar event from + `meta calendar` — which knows when a meeting is genuinely scheduled. +- **By voice.** Don't do this by pattern-matching transcripts. Claude is + already in the loop, so expose `start_meeting_notes` / `stop_meeting_notes` + as SDK MCP tools and let him call them. That handles "start taking notes", + "actually stop that", and "are you recording?" without any string matching, + and it's a handful of lines. + +### Before building this: the consent question + +This records other people. Worth settling first, because it shapes the design +and it's much cheaper to answer now: + +1. **Meta already has an official AI notetaker** with transcripts and + summaries, reachable from `meta calendar`. If that covers the need, this + feature is redundant — and it's already been through review. +2. **Some jurisdictions require all-party consent** to record a conversation. + A personal tool that silently captures colleagues is a different thing from + one that announces itself. +3. **Distributing a recorder internally is a compliance matter**, not just an + engineering one. This is the single biggest risk to the "share it with the + team" goal, and it applies to the meeting-notes feature specifically — not + to the voice assistant. + +Concrete suggestions if it goes ahead: default it **off**, require an explicit +per-meeting start, make it obvious when it's running, and keep transcripts +local with a retention policy. Get a read from Privacy/Legal before it goes to +anyone else's machine. + +**Recommendation:** build it for your own use behind an off-by-default flag, +and treat "ship the notes feature to the team" as a separate decision gated on +that review. The voice assistant itself carries none of this baggage and can +ship first. + +--- + +## Part 2 — Packaging as an internal Mac app + +### What's actually hard + +Not the code. Three things: + +**1. Permissions are the whole reason to do this.** You just granted Input +Monitoring to your terminal and noted it'll take effect on restart — that +fragility *is* the argument for a real app. TCC grants attach to a code-signed +bundle identity. Today the grants belong to your terminal, so they're shared +with everything else you run there and break when the terminal changes. A +signed `.app` with a stable bundle ID asks once, keeps it, and shows up in +System Settings under its own name. This app needs four: Microphone, Speech +Recognition, Input Monitoring, and (for meeting notes) Screen/Audio Recording. + +**2. Signing.** `security find-identity -p codesigning` returns **0 valid +identities** on this machine. Unsigned or ad-hoc-signed apps have their TCC +grants invalidated on every rebuild, which makes the app unusable in practice. +This needs a real signing identity from whoever owns Mac app distribution. +It also intersects with the binary-approval policy on managed Macs: a +hand-built `.app` handed to a colleague will be killed on launch unless it +comes through the sanctioned channel. + +**3. Distribution.** **Managed Software Center** is installed — that's the +internal channel, and the answer to "how do I share this". It also solves +signing and approval, because packages that ship through it are already +handled. The work is conforming to whatever that team requires, not inventing +a mechanism. + +### Size: fixable, and the fix is cheap + +The venv is 1.7 GB, which is a non-starter for distribution. Almost all of it +is for engines that are no longer the default: + +| Package | Size | Needed by | +| --- | --- | --- | +| torch | 529 MB | mlx-whisper only | +| claude_agent_sdk `_bundled` | 260 MB | its bundled CLI — **we deliberately don't use it** | +| mlx | 183 MB | mlx-whisper | +| llvmlite + numba | 156 MB | mlx-whisper | +| scipy + sympy | 171 MB | mlx-whisper (scipy also used by the resampler) | +| onnxruntime | 69 MB | Kokoro, Silero VAD | +| av | 45 MB | faster-whisper | + +Switching the shipped default to **Apple STT + Apple TTS + push-to-talk** drops +Whisper, Kokoro *and* Silero. I verified this: importing only the Apple-path +modules pulls in `scipy` and nothing else heavy, whereas importing `bot.py` +pulls in all nine. A realistic app is **250–300 MB**, most of which is the +Python runtime and pyobjc. + +Two concrete prerequisites: + +- Make the Whisper/Kokoro imports lazy. `bot.py` imports them at module scope + today, so they'd be bundled even when unused. +- Exclude `claude_agent_sdk/_bundled` (260 MB). The app must call the installed + `claude` CLI anyway — it's the only build that can reach the gateway — so + **Claude Code at Meta becomes a documented prerequisite**, not something the + app ships. + +### Shape of the app + +A **menu-bar app** fits the interaction model: no window, hold a key to talk, a +menu for voice/model/notes toggles, and an indicator showing when the mic is +live and when notes are recording. + +| Option | Effort | Trade-off | +| --- | --- | --- | +| **A. Python + py2app + `rumps`** | Low | Reuses everything as-is. Bundles a Python runtime; py2app + pyobjc + code signing is fiddly but well-trodden. | +| **B. Swift shell, Python core as a subprocess** | High | Native menu bar and a much cleaner signing/TCC story. Also unlocks `SpeechAnalyzer` for long-form meeting transcription. | +| **C. Stay a CLI, distribute as an internal package** | Lowest | No TCC identity of its own, so the permissions problem stays. Fine for a handful of engineers, not for a team. | + +**Recommendation: A for a pilot, with B as the path if it gets real adoption.** +A gets it onto a few machines quickly and proves whether anyone wants it. B is +justified once it needs to survive OS upgrades and support non-engineers, and +its extra value is concentrated exactly where A is weakest — signing and +long-form transcription. + +### Suggested sequence + +1. **Decide the notes/consent question.** Blocks part 1 only; everything else + can proceed in parallel. +2. **Slim the dependencies.** Lazy imports, exclude the bundled CLI, verify the + Apple-only path runs with Whisper/Kokoro uninstalled. Do this first — it's + pure cleanup with no external dependency, and it makes every later step + smaller. +3. **Prototype the process tap** against Zoom, behind an off-by-default flag, + for your own use. Verify long-meeting chunking on a real call. +4. **Talk to whoever owns Managed Software Center** about signing and packaging + requirements before building the bundle. Their answer may dictate the app's + shape, and finding that out after building option A would be expensive. +5. **Build the menu-bar bundle** (option A), signed, and install it on your own + machine. Confirm all four TCC grants survive a rebuild and a reboot — that's + the acceptance test that matters. +6. **Pilot with two or three teammates** before any wider push. + +### Open questions + +- Who owns Mac app signing and Managed Software Center packaging? +- Does the official `meta calendar` notetaker already cover the meeting-notes + need well enough to drop that feature entirely? +- Should the app require Claude Code at Meta as a prerequisite (simple, 260 MB + smaller, and the only thing that authenticates) or attempt to bundle a CLI? +- Is a menu-bar app the right shape, or would a Raycast/Alfred-style overlay + suit how people actually work better? diff --git a/README.md b/README.md new file mode 100644 index 0000000..7da6b24 --- /dev/null +++ b/README.md @@ -0,0 +1,382 @@ +# Local voice conversation with Claude Code + +Talk to Claude out loud. Speech recognition and speech synthesis run entirely on +this Mac; only Claude itself is remote, reached through the same `claude` CLI +and the same auth a terminal session uses. + +``` +mic → hold fn → SpeechTranscriber (macOS 26) → Claude Agent SDK → Kokoro → speakers +``` + +| Stage | What runs | Where | +| --- | --- | --- | +| Turn taking | hold the fn key, any app | local | +| Speech to text | `SpeechTranscriber`, macOS 26's on-device model | local | +| The conversation | Claude Code, with tools | Meta AI Gateway | +| Text to speech | Kokoro (kokoro-onnx) | local | +| Audio I/O | portaudio via sounddevice | local | + +## Usage + +```bash +cd ~/voice-agent +./talk +``` + +**Hold the 🌐 fn key and speak; release to send.** This works whatever app has +focus, so you can keep working while you talk. Speaking again cuts Claude off +mid-sentence. Ctrl-C to stop. + +If Input Monitoring isn't granted (see below), it falls back to **SPACE to start +talking, SPACE again to send** — which also stays available as a backstop even +when the hold key is working. + +Useful flags: + +```bash +./talk --list-devices # see microphones and speakers +./talk --list-voices # see macOS system voices +./talk --input-device "Adolfo i16" # pick a device by index or name substring +./talk --tts apple --voice Moira # Irish macOS voice instead of Kokoro +./talk --voice am_michael # any Kokoro voice id +./talk --hold-key right-option # a different hold key, or 'none' +./talk --voice-activity # hands-free instead of push-to-talk +./talk --claude-model claude-opus-5 # trade latency for capability +./talk --allow-writes # give Claude Edit, Write and Bash too +./talk --cwd ~/some/project # work somewhere other than ~/Workspace +./talk --load-settings # load your ~/.claude plugins and skills +./talk --log-level DEBUG # watch the frames flow +``` + +If something misbehaves, run the self-test first — it isolates the broken stage +instead of making you read pipeline logs: + +```bash +.venv/bin/python selftest.py +``` + +## First run + +Kokoro downloads about 350 MB of voice models into `~/.cache/pipecat` the first +time. Apple's recognizer needs no download. + +**macOS will need microphone permission.** The first `./talk` should prompt. If +it doesn't, or if the self-test reports "captured pure silence", add your +terminal under System Settings → Privacy & Security → Microphone. + +The speech model downloads itself the first time a locale is used. + +## Turn taking, and why it's push-to-talk + +Detecting the end of a turn by listening for silence is both slow and wrong +here. Slow, because Silero waits 0.2 s of silence and the turn strategy waits +another 0.6 s in case you resume — about **0.8 s of dead air on every turn**, +before Claude has even been asked. Wrong, because with no acoustic echo +cancellation the microphone hears the speakers, so Kokoro's voice gets +transcribed and Claude answers himself. + +A key fixes both at once. The turn ends the instant you say it does, and audio +outside a keypress is never transcribed, so self-hearing is structurally +impossible rather than merely guarded against. + +Holding **fn** is the default because a Quartz event tap sees it from any app, +and — unlike a terminal, which only ever receives key *presses* — reports +releases too, making real hold-to-talk possible. It needs **Input Monitoring** +for your terminal, under System Settings → Privacy & Security → Input +Monitoring. Restart the terminal afterwards. + +Two things worth knowing about the fn key specifically. If it opens the emoji +picker, set System Settings → Keyboard → "Press 🌐 to" → **Do Nothing**; the tap +is listen-only by design, so it observes the key without stealing it from +whatever else you have bound. And without Input Monitoring, macOS still returns +a valid-looking event tap and simply never delivers events to it — a silent +failure — which is why the SPACE toggle stays armed as a backstop even in hold +mode. `--hold-key none` disables the tap entirely. + +`--voice-activity` switches back to hands-free. That path keeps the echo guard +(mic ignored while Claude speaks, plus `--echo-tail`, default 0.4 s), and +`--barge-in` disables even that — headphones only, or Claude will interrupt +himself. + +## Where the delay goes + +Measured on this machine, from the end of your sentence to the first audio out: + +| Stage | Push-to-talk | Voice activity | +| --- | --- | --- | +| Deciding your turn ended | ~0 (keypress) | ~0.8 s | +| Apple speech to text | 0.05–0.10 s | 0.05–0.10 s | +| Claude, first token (Sonnet) | ~1.4 s | ~1.4 s | +| Kokoro, first audio | ~0.65 s | ~0.65 s | + +Claude dominates what's left, which is why the default model is Sonnet rather +than the largest available — for conversation, first-token latency beats raw +capability. `--claude-model` trades back the other way. + +Kokoro's share is the price of sentence-at-a-time synthesis: it waits for a +sentence boundary before speaking so the prosody is right. Short first +sentences therefore start talking sooner, which is part of why `VOICE_STYLE` +asks for brevity. + +## Voices + +`--tts kokoro` (default) is a local neural voice. `--tts apple` uses the macOS +system voices, which are what `say` and Spoken Content offer. + +For Irish, `--tts apple --voice Moira` works and is the only en-IE voice macOS +ships. It's slightly *faster* than Kokoro here — 0.66 s to first audio versus +0.98 s — but only the "super-compact" variant is installed, which sounds +noticeably synthetic. Downloading the Enhanced or Premium version of Moira from +System Settings → Accessibility → Spoken Content → System Voice → English +(Ireland) makes it much better, and requires no code change. + +**Siri's voices are not available**, Irish or otherwise. Apple doesn't expose +them to third-party apps, so neither `say` nor `AVSpeechSynthesizer` can see +them — `--list-voices` shows everything that is reachable. + +## Which speech engine + +The default is **`SpeechTranscriber`**, macOS 26's new on-device model, reached +through the Swift helper in `swift/`. It needs that helper built and approved +(see below); without it the agent falls back to the older dictation model +automatically. + +To hear the difference on your own voice rather than trusting a benchmark: + +```bash +.venv/bin/python compare_engines.py # records you, runs every engine +``` + +`--stt-engine apple` or `--analyzer-module dictation` switch to the older model, +which is worse bare but honours the vocabulary file. + +### Building the helper + +```bash +swift/build.sh +``` + +On this managed Mac the fresh binary is killed on launch (exit 137) until it has +been through binary approval, and **every rebuild needs approving again** since +the hash changes. `build.sh` says so when it happens, and the agent degrades to +the older engine meanwhile rather than failing. + +## Getting the words right + +Two files control how well unusual words are heard, and both are meant to be +edited: + +- **`vocabulary.txt`** — terms the recognizer should expect. Measured at 23.5% + → 16.5% word error rate on technical speech. Add the names, jargon and + product names you actually say out loud. +- **`corrections.txt`** — `heard => replacement` rules for mistakes that recur + identically. Took it to 14.1%. Add a rule once you've seen the *same* wrong + word twice. + +A handful of terms is also discovered automatically from the project — unusual +filenames, class names, git branches and authors — and from what Claude has +been saying, since that predicts what you'll say next. + +**Keep `vocabulary.txt` short and relevant.** Padding the list out is actively +harmful, and not in the way you'd expect: a thousand random dictionary words +cost only 1.1 points, but a hundred names harvested from this project erased +the entire benefit. The problem is ordinary words — biasing towards "bot", +"plan" or "hack" drags correct speech onto them. Discovered terms are therefore +filtered against the system dictionary and capped at a dozen, and the total is +capped at 40. Terms you write yourself are always kept, ordinary or not, on the +assumption that you meant it. + +Claude also gets the list in his system prompt, so he can resolve a mangled +transcript while answering instead of needing a separate correction pass. That +recovers most of the benefit of one at no latency cost. + +`--no-vocabulary` turns all of it off; `--vocabulary-file` points somewhere else. + +## Personality, and where Claude works + +By default Claude works in **`~/Workspace`** (created if missing), not in this +repo — the assistant is for everyday use, not for editing itself. `--cwd` points +it elsewhere. + +Three files there shape how it behaves, all git-tracked so you can see how they +drift and roll back if they drift wrong: + +| File | What it does | +| --- | --- | +| `AGENTS.md` | Claude's personality, loaded on every launch | +| `vocabulary.txt` | words the recognizer should expect | +| `corrections.txt` | fixes for words it gets wrong the same way each time | + +They're created from the `*.example` templates in this repo the first time you +point at a new workspace, then they're yours to grow. + +Two things worth knowing: + +- **HTML comments are stripped** before Claude sees the file, so you can leave + yourself notes in `` without them acting as instructions. +- **Keep it short.** It's prepended to every turn, and a long file makes replies + longer and more written-sounding. Describe character and habits; the + formatting rules live in `VOICE_STYLE` in `bot.py` and are applied *after* the + personality, so they win on "no markdown, keep it brief". + +`CLAUDE.md` in the same directory also works — the CLI picks that one up by +itself. `AGENTS.md` is handled here because the CLI ignores it. + +## Long-term memory + +The agent reads your Metamate personal brain at startup — `briefing.md`, +`preferences.md`, `profile.md` — so it already knows what you're working on. +Ask "what am I most overdue on" and it answers from your actual pinned notes. +Costs about 1.6 s at launch, cached locally so a VPN drop doesn't lose it. + +The 31 project names under `projects/` also become vocabulary, since +"CIP-Unified-Cooldown" and "pSMSL" are exactly the words a recogniser fumbles. + +It writes back too, but narrowly. The agent supplies three tools; **when to use +them is not in the agent** — it's in `~/Workspace/.claude/skills/memory/`, since +the discipline is your setup rather than a property of the code. Edit that file +to change the behaviour. + +| Tool | Goes to | For | +| --- | --- | --- | +| `remember_correction` | `corrections.txt` (local) | a word the recognizer misheard | +| `remember_preference` | brain `preferences.md` | how you want to be worked with | +| `remember_note` | brain `notes.md` | a technique or lesson worth keeping | + +Corrections stay local because they're about this microphone and this +recognizer — meaningless on another machine. The other two are true of you +regardless, so they belong in the brain. + +Entries land under a `## Learned in voice sessions` heading so they never get +tucked inside a section you wrote, and it stays obvious which lines came from +the agent. The skill also tells it never to touch `briefing.md`, which the +daily cron owns. + +`--no-brain` turns all of it off. + +## Skills and the journal + +Drop a skill in `~/Workspace/.claude/skills//SKILL.md` and it's available +in conversation — verified, not assumed: a test skill placed there was +discovered and invoked with no extra configuration. What makes it work is +loading the workspace as a project source (`setting_sources=["project"]`) plus +`skills="all"`; without the first, a skills folder there is silently ignored. + +There's a README in that folder covering the format and how to write skills +that sound right when spoken rather than read. + +Every turn is logged to `~/Workspace/journal.jsonl` — what was heard, what was +answered. The journal sits immediately after the transcript repair rather than +at the end of the pipeline, because the user aggregator *consumes* +`TranscriptionFrame`s; anything downstream of it never sees what you said. The +reply arrives separately, through the LLM's own callback. It's gitignored, being a verbatim record of everything said near the +microphone. It exists so the next round of accuracy work can be measured on +real conversations rather than synthesised audio. + +## What Claude is allowed to do + +**The shell is on by default**, because most of what you'd ask about out loud +lives behind `meta` — experiments, memory, tasks, calendar — and without it the +agent can only apologise. `Write`, `Edit` and `NotebookEdit` are denied. + +Be clear-eyed about what that means: a shell can write files perfectly well, so +denying the edit tools is a speed bump against casual edits, **not** a security +boundary. `--read-only` is the real boundary — it denies the shell too, leaving +only reading and searching. `--allow-writes` denies nothing. + +Worth knowing **how** that's enforced, because the obvious way doesn't work. +Passing `allowed_tools` with `permission_mode="bypassPermissions"` restricts +nothing — measured: Claude ran `Bash` while it was absent from that list, with +no denial recorded. Only naming tools in `disallowed_tools` actually blocks +them. If you change this, verify by asking it to run a shell command rather +than trusting the config to mean what it looks like. + +Formatting is stripped before anything is spoken, so a stray `**bold**` never +becomes "asterisk asterisk". `spoken_text.py` wraps pipecat's +`MarkdownTextFilter` — which runs after sentence aggregation, so markdown split +across streaming chunks is already reassembled — and adds the cases it misses: +strikethrough, bullet dashes, `snake_case` (read as words), and `3 * 4`, which +the base filter turns into "3 4" rather than "3 times 4". + +Prompting alone wasn't enough for this. Claude is told not to emit markdown and +mostly doesn't, but asking is probabilistic and hearing it once is enough. + +Claude is also told to write for speech rather than for a screen: short answers, +no markdown, no URLs, and a spoken heads-up before long tool calls. That prompt +lives in `VOICE_STYLE` in `bot.py` and is the first thing to edit if the replies +don't sound the way you want. + +## Files + +- `bot.py` — assembles the pipeline; all the tuning knobs are here +- `apple_stt.py` — Apple dictation model as a Pipecat STT service +- `claude_llm.py` — Pipecat processor that puts Claude Code in the LLM slot +- `speech_analyzer_stt.py` — drives the Swift helper; `swift/SpeechHelper.swift` +- `compare_engines.py` — record yourself, run every engine, compare +- `brain.py` — reads and writes the Metamate personal brain +- `memory_tools.py` — the remember_* tools Claude calls +- `journal.py` — one JSON line per turn +- `vocabulary.py` — term biasing and repair rules; templates in `*.example.txt` +- `transcript_repair.py` — applies the repair rules to every transcription +- `push_to_talk.py` — keyboard turn control, the default +- `global_hotkey.py` — system-wide hold-key watcher via a Quartz event tap +- `apple_tts.py` — macOS system voices as a Pipecat TTS service +- `echo_guard.py` — self-hearing guard, used only in `--voice-activity` mode +- `sounddevice_transport.py` — microphone and speaker transport +- `selftest.py` — per-stage diagnostics +- `talk` — launcher, sets the CA bundle and runs `bot.py` + +## Why this doesn't look like a stock Pipecat example + +Five things forced changes worth knowing about before you edit anything. + +**The SDK's bundled CLI can't authenticate here.** `claude-agent-sdk` ships its +own copy of Claude Code inside the wheel and prefers it over the one on `PATH`. +That copy is stock Anthropic: it knows nothing about this org's AI Gateway or +the `apiKeyHelper` in `/Library/Application Support/ClaudeCode/managed-settings.json`, +so every turn comes back "Invalid API key · Fix external API key". `bot.py` +passes `cli_path=shutil.which("claude")` to force the installed CLI. This is +easy to miss because it works fine when launched from inside a Claude Code +session, which leaks the gateway environment variables to its children. + +**Apple's recognizer isn't in Pipecat.** `apple_stt.py` adds it as a +`SegmentedSTTService`. The one real trap: results arrive through the +CoreFoundation runloop, so waiting on a `threading.Event` deadlocks — nothing +pumps the runloop and the handler never fires. The service pumps it in 10 ms +slices and yields to asyncio in between. Also, a Python exception escaping the +result handler crosses back into Objective-C and aborts the process, so that +handler catches everything. + +**No PyAudio.** Pipecat's `LocalAudioTransport` needs PyAudio, which has no +macOS wheel and must be compiled against a Homebrew portaudio. Homebrew can't +write to `/opt/homebrew` here, and a locally compiled binary won't run until +it's been through the approval process. `sounddevice_transport.py` is a port of +that transport onto sounddevice, whose wheel ships a prebuilt portaudio. Same +frame contract, so it drops into any Pipecat pipeline. + +**Python doesn't trust the TLS proxy.** `pip` and any library that downloads +models fail certificate verification until pointed at the system CA bundle. +`.venv/pip.conf` handles pip; `talk` exports `SSL_CERT_FILE` for everything +else. If you add a dependency, install it through `.venv/bin/python -m pip` so +it picks up `pip.conf`. + +**Claude keeps its own history.** `ClaudeCodeLLM` isn't a Pipecat `LLMService` +subclass — the Agent SDK already owns conversation state and tool calling, so +wrapping it in Pipecat's context machinery would mean two systems tracking the +same conversation. It's a plain `FrameProcessor` that reads the newest user +message off each `LLMContextFrame` and emits text frames. The Pipecat context +still exists, but only so the aggregators can detect when a turn has ended. + +## Known rough edges + +- **Launching from inside a Claude Code session doesn't work.** The `claude` CLI + tries to apply its own `sandbox-exec` profile, which macOS refuses when the + parent is already sandboxed, and it exits with code 71. Use a normal terminal. +- **The SPACE fallback is a toggle, not hold-to-talk.** A terminal only receives + key presses, never releases. Only the event-tap path can do true + hold-to-talk, which is why it's the default. +- **Only Moira is available in Irish, at the lowest quality tier.** Download the + Enhanced or Premium variant from System Settings to fix that. +- **Whisper hallucinates on silence** when you fall back to it, emitting things + like "Thank you." for breath noise. `_NOISE_TRANSCRIPTS` in `claude_llm.py` + filters the common ones; add to that set if you hit others. diff --git a/apple_stt.py b/apple_stt.py new file mode 100644 index 0000000..03f5b2e --- /dev/null +++ b/apple_stt.py @@ -0,0 +1,359 @@ +"""Speech-to-text using Apple's on-device dictation model via SFSpeechRecognizer. + +This is the same recognizer macOS Dictation uses. With +``requiresOnDeviceRecognition`` set, audio never leaves the machine, there is no +model to download, and there is no Metal shader compilation — which is what +makes it a better fit here than Whisper. + +Results come back through the CoreFoundation runloop, so waiting on a +`threading.Event` deadlocks: nothing pumps the runloop and the handler is never +called. Everything here pumps it in short slices instead, yielding to asyncio +between them. + +Requires Dictation to be switched on in System Settings > Keyboard. +""" + +import asyncio +import os +import tempfile +import wave +from collections.abc import AsyncGenerator +from difflib import SequenceMatcher + +from loguru import logger + +from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame +from pipecat.services.settings import STTSettings +from pipecat.services.stt_service import SegmentedSTTService +from pipecat.transcriptions.language import Language +from pipecat.utils.time import time_now_iso8601 + +try: + import Speech + from Foundation import NSURL, NSDate, NSLocale, NSRunLoop +except ImportError as e: # pragma: no cover - depends on pyobjc being installed + raise ImportError( + "Apple speech recognition needs pyobjc: pip install pyobjc-framework-Speech" + ) from e + +_AUTH_STATUS = {0: "not determined", 1: "denied", 2: "restricted", 3: "authorized"} +_PUMP_SLICE = 0.01 + +# SFSpeechRecognitionTaskState +_TASK_COMPLETED = 4 + +# Recognition runs far faster than real time (a 61s file finishes in about 1.3s), +# but scale the deadline with the audio anyway so a long utterance can't be +# dropped by a fixed ceiling. +_TIMEOUT_BASE = 10.0 +_TIMEOUT_PER_AUDIO_SECOND = 0.5 + +# How much of a new transcript must still match the previous one for it to count +# as a refinement rather than the recognizer having started over. +_CONTINUATION_RATIO = 0.5 + +# How alike two finished passes must be to be judged the same speech re-read. +_SAME_AUDIO_RATIO = 0.6 + + +def _pump(seconds: float = _PUMP_SLICE): + """Give the runloop a chance to deliver Speech framework callbacks.""" + NSRunLoop.currentRunLoop().runUntilDate_(NSDate.dateWithTimeIntervalSinceNow_(seconds)) + + +def _authorize(timeout: float = 20.0) -> int: + """Return the speech authorization status, prompting once if undetermined.""" + status = Speech.SFSpeechRecognizer.authorizationStatus() + if status != 0: + return status + + box: dict[str, int] = {} + Speech.SFSpeechRecognizer.requestAuthorization_(lambda s: box.setdefault("status", s)) + + waited = 0.0 + while "status" not in box and waited < timeout: + _pump(0.05) + waited += 0.05 + return box.get("status", 0) + + +def _make_recognizer(locale: str | None): + if locale: + recognizer = Speech.SFSpeechRecognizer.alloc().initWithLocale_( + NSLocale.localeWithLocaleIdentifier_(locale) + ) + else: + recognizer = Speech.SFSpeechRecognizer.alloc().init() + + if recognizer is None: + raise RuntimeError(f"No speech recognizer available for locale {locale!r}") + if not recognizer.isAvailable(): + raise RuntimeError("Speech recognizer is not available right now") + return recognizer + + +class _Transcript: + """Stitch a recognition back together across its internal restarts. + + On long audio the recognizer does not extend one transcript to the end. It + builds one up, then silently starts over from a later point in the audio, + and the single final result covers only that last stretch — so reading the + final result alone loses everything said earlier. + + A restart has to be recognised from the text itself. Partial results carry + no segment timestamps, so the only reliable marker is the transcript + ceasing to be a refinement of the previous one: a growing transcript keeps + almost all of its prefix even when the recognizer revises a word, whereas a + restart drops from hundreds of characters back to a few that share nothing + with what came before. + + Restarts are of two kinds, and conflating them is what produces doubled + text. Observed on a 61s recording: the recognizer transcribes the whole + thing, starts over and transcribes the whole thing again slightly + differently, then emits the last second as its only final result. The + re-pass has to replace its predecessor while the tail is appended, so + passes that begin with the same words are treated as the same audio and + only the fullest is kept. + """ + + def __init__(self): + self._passes: list[str] = [] + self._current = "" + + def add(self, start: float, text: str): + if self._restarted(text): + self._close_pass() + # Within a pass each result supersedes the last, so keep the newest. + self._current = text + + def close(self): + """Fold the in-progress pass in. Call once recognition has finished.""" + self._close_pass() + + def _restarted(self, text: str) -> bool: + """Whether this result abandons the running transcript instead of refining it. + + Compared by prefix because it runs on every partial and a refinement + always keeps its opening intact. + """ + if not self._current or not text: + return False + shared = len(os.path.commonprefix([self._current, text])) + return shared < min(len(self._current), len(text)) * _CONTINUATION_RATIO + + def _close_pass(self): + finished, self._current = self._current, "" + if not finished.strip(): + return + for i, existing in enumerate(self._passes): + if self._same_audio(existing, finished): + # Same stretch of audio transcribed again; keep the fuller read. + if len(finished) > len(existing): + self._passes[i] = finished + return + self._passes.append(finished) + + @staticmethod + def _same_audio(a: str, b: str) -> bool: + """Whether two finished passes cover the same speech. + + Prefix matching is too strict here: a second pass corrects mistakes from + the first, often within the opening few words, so overall similarity is + what distinguishes a re-read from genuinely new audio. Only runs when a + pass closes, so the cost doesn't matter. + """ + return SequenceMatcher(None, a, b).ratio() >= _SAME_AUDIO_RATIO + + def text(self) -> str: + parts = [*self._passes, self._current] + return " ".join(part.strip() for part in parts if part.strip()) + + +def _start_recognition(path: str, locale: str | None, terms: list[str] | None = None) -> dict: + """Kick off a recognition task. The returned dict fills in from the handler.""" + recognizer = _make_recognizer(locale) + request = Speech.SFSpeechURLRecognitionRequest.alloc().initWithURL_( + NSURL.fileURLWithPath_(path) + ) + request.setRequiresOnDeviceRecognition_(True) + if terms: + # Biasing the decoder towards expected words measured 23.5% -> 16.5% WER. + request.setContextualStrings_(terms) + # Partial results are what make the stitching above possible: the text from + # a segment is only ever visible while that segment is the current one. + request.setShouldReportPartialResults_(True) + + box: dict = {"transcript": _Transcript()} + + def handler(result, error): + # This crosses back into Objective-C, which aborts the whole process on + # an escaping Python exception. Nothing here may raise. + try: + if error is not None: + box["error"] = str(error.localizedDescription()) + return + if result is None: + return + transcription = result.bestTranscription() + segments = transcription.segments() + start = float(segments[0].timestamp()) if segments else 0.0 + box["transcript"].add(start, str(transcription.formattedString())) + except Exception as e: # pragma: no cover - defensive + box["error"] = f"result handler failed: {e}" + + # Keep the task alive for as long as the caller holds the box. + box["_task"] = recognizer.recognitionTaskWithRequest_resultHandler_(request, handler) + return box + + +def _is_done(box: dict) -> bool: + if "error" in box: + return True + task = box.get("_task") + return task is not None and task.state() == _TASK_COMPLETED + + +def _finish(box: dict, timed_out: bool, timeout: float) -> str: + if "error" in box: + raise RuntimeError(box["error"]) + if timed_out: + raise TimeoutError(f"Speech recognition timed out after {timeout}s") + box["transcript"].close() + return box["transcript"].text() + + +def timeout_for(audio_seconds: float) -> float: + """A recognition deadline that scales with how much audio there is.""" + return _TIMEOUT_BASE + _TIMEOUT_PER_AUDIO_SECOND * audio_seconds + + +def _recognize_file(path: str, locale: str | None, timeout: float) -> str: + """Blocking recognition, for startup checks before the event loop matters.""" + box = _start_recognition(path, locale) + waited = 0.0 + while not _is_done(box) and waited < timeout: + _pump() + waited += _PUMP_SLICE + return _finish(box, waited >= timeout, timeout) + + +async def _recognize_file_async( + path: str, locale: str | None, timeout: float, terms: list[str] | None = None +) -> str: + """Recognition that keeps the asyncio loop breathing between runloop slices.""" + box = _start_recognition(path, locale, terms) + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while not _is_done(box) and loop.time() < deadline: + _pump() + await asyncio.sleep(0.005) + return _finish(box, loop.time() >= deadline, timeout) + + +def probe(locale: str = "en-US") -> tuple[bool, str]: + """Check whether Apple speech recognition can actually be used. + + Returns (available, explanation). Only an attempted recognition settles + this: a disabled Dictation subsystem shows up as an error on the first + request rather than through any status flag. The probe feeds it silence, so + "no speech detected" is the healthy answer — it means the subsystem ran. + """ + status = _authorize() + if status != 3: + return False, f"speech recognition authorization is {_AUTH_STATUS.get(status, status)}" + + try: + recognizer = _make_recognizer(locale) + except RuntimeError as e: + return False, str(e) + if not recognizer.supportsOnDeviceRecognition(): + return False, "this Mac has no on-device recognition model installed" + + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f: + silence = f.name + try: + with wave.open(silence, "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(16000) + w.writeframes(b"\x00\x00" * 4000) + _recognize_file(silence, locale, timeout=1.0) + except TimeoutError: + pass # The subsystem answered but never finalized; good enough. + except RuntimeError as e: + reason = str(e) + if "no speech" in reason.lower(): + pass # The expected reply to a silent file. + elif "disabled" in reason.lower(): + return False, ( + "Dictation is turned off — enable System Settings > Keyboard > Dictation" + ) + else: + return False, reason + except Exception as e: + return False, str(e) + finally: + os.unlink(silence) + + return True, "on-device dictation model ready" + + +class AppleSpeechSTTService(SegmentedSTTService): + """Transcribe VAD-delimited speech segments with Apple's dictation model.""" + + def __init__( + self, + *, + locale: str = "en-US", + language: Language = Language.EN_US, + vocabulary=None, + **kwargs, + ): + # The recognizer picks its model from the locale, so there is no model + # field to set; Pipecat wants every settings field initialized anyway. + super().__init__(settings=STTSettings(model=None, language=locale), **kwargs) + self._locale = locale + self._language = language + # Read per utterance rather than cached, so terms learned during the + # conversation reach the next recognition. + self._vocabulary = vocabulary + + def can_generate_metrics(self) -> bool: + return True + + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Transcribe one speech segment. + + Args: + audio: The segment as a WAV container, per ``wants_wav_segments``. + """ + await self.start_processing_metrics() + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f: + f.write(audio) + path = f.name + # WAV header is 44 bytes; the rest is 16-bit mono at the pipeline rate. + audio_seconds = max(0.0, (len(audio) - 44) / 2 / self.sample_rate) + try: + terms = self._vocabulary.terms() if self._vocabulary else None + text = await _recognize_file_async( + path, self._locale, timeout_for(audio_seconds), terms + ) + except TimeoutError: + # The segment held no recognizable speech. + await self.stop_processing_metrics() + return + except Exception as e: + await self.stop_processing_metrics() + yield ErrorFrame(error=f"Apple speech recognition failed: {e}") + return + finally: + os.unlink(path) + + await self.stop_processing_metrics() + + text = text.strip() + if not text: + return + + logger.debug(f"Transcription: [{text}]") + yield TranscriptionFrame(text, self._user_id, time_now_iso8601(), self._language) diff --git a/apple_tts.py b/apple_tts.py new file mode 100644 index 0000000..737ea04 --- /dev/null +++ b/apple_tts.py @@ -0,0 +1,119 @@ +"""Text-to-speech using the macOS system voices. + +Drives the `say` binary rather than AVSpeechSynthesizer. `say` can render +straight to signed 16-bit little-endian PCM at a chosen sample rate, which is +exactly what the pipeline wants, and it avoids reading raw AVAudioPCMBuffer +channel pointers through pyobjc. + +Note that Siri's voices are not reachable this way. Apple does not expose them +to third-party apps; `say` and AVSpeechSynthesizer only see the voices listed +under System Settings > Accessibility > Spoken Content. +""" + +import asyncio +import os +import tempfile +import wave +from collections.abc import AsyncGenerator + +from loguru import logger + +from pipecat.audio.utils import create_stream_resampler +from pipecat.frames.frames import ErrorFrame, Frame, TTSAudioRawFrame +from pipecat.services.settings import TTSSettings +from pipecat.services.tts_service import TTSService + +SAY = "/usr/bin/say" + +# Chunked so playback can start before the whole file is read. +_CHUNK_FRAMES = 2400 # 100ms at 24kHz + + +def available_voices() -> list[tuple[str, str]]: + """Return (name, language) for every installed system voice.""" + import AVFoundation as AV + + return [(v.name(), v.language()) for v in AV.AVSpeechSynthesisVoice.speechVoices()] + + +def find_voice(name: str) -> tuple[str, str] | None: + """Look up an installed voice by name, case-insensitively.""" + for voice_name, language in available_voices(): + if voice_name.lower() == name.lower(): + return voice_name, language + return None + + +class AppleTTSService(TTSService): + """Speak text with a macOS system voice. + + Args: + voice: An installed system voice name, e.g. "Moira" for Irish English. + rate_wpm: Speaking rate in words per minute. None uses the voice default. + """ + + def __init__(self, *, voice: str = "Moira", rate_wpm: int | None = None, **kwargs): + super().__init__( + push_start_frame=True, + push_stop_frames=True, + settings=TTSSettings(model=None, voice=voice, language=None), + **kwargs, + ) + self._voice = voice + self._rate_wpm = rate_wpm + self._resampler = create_stream_resampler() + + if find_voice(voice) is None: + names = ", ".join(sorted(n for n, _ in available_voices())[:8]) + raise ValueError( + f"macOS has no voice named {voice!r}. Installed voices include: {names}… " + "Add more under System Settings > Accessibility > Spoken Content > System Voice." + ) + + def can_generate_metrics(self) -> bool: + return True + + async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: + """Synthesize one chunk of text.""" + await self.start_tts_usage_metrics(text) + + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f: + path = f.name + try: + command = [SAY, "-v", self._voice, "-o", path] + if self._rate_wpm: + command += ["-r", str(self._rate_wpm)] + command += [f"--data-format=LEI16@{self.sample_rate}", "--", text] + + process = await asyncio.create_subprocess_exec( + *command, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.PIPE, + ) + _, stderr = await process.communicate() + if process.returncode != 0: + detail = stderr.decode(errors="replace").strip() + yield ErrorFrame(error=f"say failed ({process.returncode}): {detail}") + return + + with wave.open(path, "rb") as wav: + source_rate = wav.getframerate() + while chunk := wav.readframes(_CHUNK_FRAMES): + await self.stop_ttfb_metrics() + if source_rate != self.sample_rate: + chunk = await self._resampler.resample( + chunk, source_rate, self.sample_rate + ) + yield TTSAudioRawFrame( + audio=chunk, + sample_rate=self.sample_rate, + num_channels=1, + context_id=context_id, + ) + except Exception as e: + logger.exception(f"Apple TTS failed: {e}") + yield ErrorFrame(error=f"Apple TTS failed: {e}") + finally: + await self.stop_ttfb_metrics() + if os.path.exists(path): + os.unlink(path) diff --git a/bot.py b/bot.py new file mode 100644 index 0000000..03cd4db --- /dev/null +++ b/bot.py @@ -0,0 +1,583 @@ +#!/usr/bin/env python3 +"""Local voice conversation with Claude Code. + +Everything but Claude itself runs on this machine: Apple's on-device dictation +model for speech-to-text, Kokoro for text-to-speech, Silero for voice activity +detection. Claude runs through the Claude Agent SDK, which drives the same +`claude` CLI — and the same auth — as a terminal session. +""" + +import argparse +import asyncio +import re +import shutil +import sys +from pathlib import Path + +from claude_agent_sdk import ClaudeAgentOptions, SandboxSettings +from loguru import logger + +from brain import Brain +from claude_llm import ClaudeCodeLLM +from echo_guard import EchoGuardUserMuteStrategy +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import TTSSpeakFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.worker import PipelineParams, PipelineWorker +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, + LLMUserAggregatorParams, +) +from pipecat.processors.audio.vad_processor import VADProcessor +from pipecat.services.kokoro.tts import KokoroTTSService +from pipecat.services.whisper.stt import MLXModel, WhisperSTTService, WhisperSTTServiceMLX +from pipecat.transcriptions.language import Language +from pipecat.turns.user_start.external_user_turn_start_strategy import ( + ExternalUserTurnStartStrategy, +) +from pipecat.turns.user_stop.external_user_turn_stop_strategy import ( + ExternalUserTurnStopStrategy, +) +from pipecat.turns.user_stop.speech_timeout_user_turn_stop_strategy import ( + SpeechTimeoutUserTurnStopStrategy, +) +from pipecat.turns.user_turn_strategies import UserTurnStrategies +from pipecat.workers.runner import WorkerRunner +from global_hotkey import HOLD_KEYS +from push_to_talk import PushToTalk +from journal import Journal +from memory_tools import build_server +from transcript_repair import TranscriptRepair +from vocabulary import Vocabulary +from spoken_text import SpokenTextFilter +from sounddevice_transport import SoundDeviceTransport, SoundDeviceTransportParams, list_devices + +# Every recognizer here expects 16 kHz; Kokoro synthesizes at 24 kHz. Matching +# both natively avoids a resample on the hot path in each direction. +STT_SAMPLE_RATE = 16000 +TTS_SAMPLE_RATE = 24000 + +VOICE_STYLE = """ +You are talking to the user out loud, over a microphone and speakers. Your +replies are spoken by a text-to-speech voice, so write them the way you would +say them: + +- No markdown, bullet points, code blocks, emoji, or URLs. None of it survives + being read aloud. +- Keep answers to a few sentences unless asked to go deeper. This is a + conversation, not a document. +- Spell out things that only make sense visually. Say "line forty-two of + bot dot py" rather than pasting a path. +- If you need to run tools, say what you're doing in a short phrase first, so + the silence is explained. +- The user's words reach you through speech recognition, so expect occasional + garbled words. Ask rather than guess when it matters. +""".strip() + +# Enforcement is by denial, not by allow-list. Measured: with +# permission_mode="bypassPermissions", passing allowed_tools does NOT restrict +# anything — Claude ran Bash while it was absent from that list, with no denial +# recorded. Only disallowed_tools blocks. +# +# The shell is on by default because the point of this assistant is asking about +# work out loud, and most of that lives behind `meta` — experiments, memory, +# tasks, calendar. Without a shell it can only apologise. +# +# Be clear-eyed about what the default denies: a shell can write files perfectly +# well, so withholding Edit and Write is a speed bump against casual edits, not +# a security boundary. --read-only is the real boundary. +FILE_EDIT_TOOLS = ["Write", "Edit", "NotebookEdit"] +SHELL_TOOLS = ["Bash", "BashOutput", "KillShell"] + +# Left unset, the CLI picks whatever the interactive session would use, which +# here resolves to a model this account can't reach headlessly ("Access to Fable +# is currently restricted"). Naming one explicitly avoids that, and for a spoken +# conversation time-to-first-token matters more than the extra capability. +DEFAULT_CLAUDE_MODEL = "claude-sonnet-4-6" + +# Where Claude works, and where the personality file lives. Deliberately not +# this repo: the assistant is for everyday use, not for editing itself. +DEFAULT_WORKSPACE = Path.home() / "Workspace" + +# The CLI reads CLAUDE.md from the working directory on its own but ignores +# AGENTS.md, so that one is loaded here and appended to the system prompt. +PERSONALITY_FILE = "AGENTS.md" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--list-devices", action="store_true", help="Print audio devices and exit." + ) + parser.add_argument("--input-device", help="Microphone index or name substring.") + parser.add_argument("--output-device", help="Speaker index or name substring.") + parser.add_argument( + "--stt-engine", + choices=["auto", "analyzer", "apple", "mlx", "cpu"], + default="auto", + help=( + "analyzer is macOS 26's SpeechAnalyzer and measured best; apple is the " + "older dictation API; mlx and cpu run Whisper. auto tries them in that " + "order, falling back on whatever is available." + ), + ) + parser.add_argument( + "--analyzer-module", + choices=["transcriber", "dictation"], + default="transcriber", + help=( + "Which SpeechAnalyzer module: transcriber is macOS 26's new model; " + "dictation is the older one, which honours the vocabulary." + ), + ) + parser.add_argument( + "--whisper-model", + help="Whisper model, for the mlx and cpu engines. Downloaded on first use.", + ) + parser.add_argument( + "--tts", + choices=["kokoro", "apple"], + default="kokoro", + help="kokoro is a local neural voice; apple uses the macOS system voices.", + ) + parser.add_argument( + "--voice", + help="Kokoro voice id (default af_heart) or macOS voice name (default Moira).", + ) + parser.add_argument( + "--voice-rate", type=int, help="Words per minute, macOS voices only." + ) + parser.add_argument( + "--list-voices", action="store_true", help="Print macOS system voices and exit." + ) + parser.add_argument( + "--claude-model", + default=DEFAULT_CLAUDE_MODEL, + help="Claude model. Voice wants fast first tokens more than raw capability.", + ) + parser.add_argument( + "--voice-activity", + action="store_true", + help=( + "End turns by detecting silence instead of a keypress. Hands-free, but " + "adds about 0.8s per turn and lets the mic hear the speakers." + ), + ) + parser.add_argument( + "--hold-key", + default="fn", + choices=[*HOLD_KEYS, "none"], + help=( + "Modifier to hold while speaking, recognised in any app. " + "'none' falls back to a spacebar toggle in this terminal." + ), + ) + parser.add_argument( + "--ptt-key", + default=" ", + help="Key that toggles the microphone when not using a hold key.", + ) + parser.add_argument( + "--cwd", + default=str(DEFAULT_WORKSPACE), + help=( + "Where Claude's tools point, and where AGENTS.md is read from. " + "Created if it doesn't exist." + ), + ) + parser.add_argument( + "--allow-writes", + action="store_true", + help="Also allow Edit and Write, on top of the shell it already has.", + ) + parser.add_argument( + "--read-only", + action="store_true", + help="Deny the shell too, so it can only read and search. Overrides --allow-writes.", + ) + parser.add_argument( + "--barge-in", + action="store_true", + help=( + "With --voice-activity, let your voice interrupt Claude mid-sentence. " + "Headphones only: on speakers the mic hears Claude and he interrupts himself." + ), + ) + parser.add_argument( + "--echo-tail", + type=float, + default=0.4, + help=( + "With --voice-activity, seconds to keep ignoring the mic after Claude " + "stops speaking." + ), + ) + parser.add_argument( + "--load-settings", + action="store_true", + help=( + "Load your ~/.claude settings, plugins and skills. Off by default because " + "they add seconds to startup and a voice agent needs few of them." + ), + ) + parser.add_argument( + "--greeting", + default="I'm listening.", + help="Spoken on startup. Pass an empty string to start silent.", + ) + parser.add_argument( + "--vocabulary-file", + help="Terms to bias the recognizer towards. Defaults to vocabulary.txt here.", + ) + parser.add_argument( + "--no-brain", + action="store_true", + help="Skip the Metamate personal brain: no long-term context, no remembering.", + ) + parser.add_argument( + "--no-vocabulary", + action="store_true", + help="Turn off vocabulary biasing and transcript repair.", + ) + parser.add_argument("--log-level", default="INFO") + return parser.parse_args() + + +def as_device(value: str | None) -> int | str | None: + """Accept either a device index or a name substring.""" + if value is None: + return None + return int(value) if value.isdigit() else value + + +def build_stt(args: argparse.Namespace, vocabulary): + engine = args.stt_engine + + # Preferred. Scored the way the published benchmark scores — spelling digits + # out, so an engine is not punished for writing "400" — SpeechTranscriber + # reaches 9.4% with five repair rules and no vocabulary at all, matching the + # older model's best while needing far less hand-tuning, and beating it raw + # (16.5% against 20.0%). It also needs no transcript stitching on long + # speech. Requires the Swift helper built and approved, so fall back quietly. + if engine in ("auto", "analyzer"): + from speech_analyzer_stt import SpeechAnalyzerSTTService, probe + + available, reason = probe() + if available: + model = ( + "SpeechTranscriber, the new macOS 26 model" + if args.analyzer_module == "transcriber" + else "DictationTranscriber, the older model" + ) + logger.info(f"Speech to text: {model} — {reason}") + return SpeechAnalyzerSTTService( + vocabulary=vocabulary, module=args.analyzer_module + ) + if engine == "analyzer": + raise SystemExit(f"SpeechAnalyzer is unavailable: {reason}") + logger.info(f"SpeechAnalyzer unavailable ({reason}); using the dictation model.") + engine = "apple" + + if engine in ("auto", "apple"): + from apple_stt import AppleSpeechSTTService, probe + + available, reason = probe() + if available: + logger.info(f"Speech to text: Apple dictation model ({reason})") + return AppleSpeechSTTService(vocabulary=vocabulary) + if engine == "apple": + raise SystemExit(f"Apple speech recognition is unavailable: {reason}") + logger.warning(f"Apple speech recognition unavailable ({reason}); falling back to Whisper.") + engine = "cpu" + + if engine == "mlx": + model = args.whisper_model or MLXModel.LARGE_V3_TURBO_Q4.value + logger.info(f"Speech to text: MLX Whisper ({model})") + return WhisperSTTServiceMLX( + settings=WhisperSTTServiceMLX.Settings(model=model, language=Language.EN) + ) + + # base.en transcribed the benchmark phrase as accurately as distil-medium.en + # while taking 0.3s instead of 2.2s, which matters a lot in a conversation. + model = args.whisper_model or "base.en" + logger.info(f"Speech to text: faster-whisper on CPU ({model})") + return WhisperSTTService(settings=WhisperSTTService.Settings(model=model, language=Language.EN)) + + +def build_tts(args: argparse.Namespace): + if args.tts == "apple": + from apple_tts import AppleTTSService, find_voice + + voice = args.voice or "Moira" + found = find_voice(voice) + logger.info(f"Text to speech: macOS voice {voice} ({found[1] if found else '?'})") + return AppleTTSService( + voice=voice, rate_wpm=args.voice_rate, text_filters=[SpokenTextFilter()] + ) + + if args.voice_rate: + logger.warning("--voice-rate only applies to --tts apple; ignoring it.") + voice = args.voice or "af_heart" + logger.info(f"Text to speech: Kokoro {voice}") + return KokoroTTSService( + settings=KokoroTTSService.Settings(voice=voice, language=Language.EN), + text_filters=[SpokenTextFilter()], + ) + + +def build_turn_taking(args: argparse.Namespace): + """Decide what opens and closes a turn. + + Returns the processor that sits right after the transport, the turn + strategies for the aggregator, and any mute strategies. + """ + if args.voice_activity: + logger.info("Turn taking: voice activity detection") + return ( + VADProcessor(vad_analyzer=SileroVADAnalyzer()), + # The default stop strategy loads a separate smart-turn model; plain + # VAD silence detection is enough and keeps startup fast. + UserTurnStrategies(stop=[SpeechTimeoutUserTurnStopStrategy()]), + # Without a keypress gating the mic, the speakers feed straight back + # into it and Claude answers himself. + [] if args.barge_in else [EchoGuardUserMuteStrategy(tail_secs=args.echo_tail)], + ) + + # Push-to-talk drives both ends of the turn itself, so no VAD, no smart-turn + # model, and no echo guard — audio outside a keypress is never transcribed. + return ( + PushToTalk( + hold_key=None if args.hold_key == "none" else args.hold_key, + toggle_key=args.ptt_key, + ), + UserTurnStrategies( + start=[ExternalUserTurnStartStrategy()], + stop=[ExternalUserTurnStopStrategy()], + ), + [], + ) + + +def build_vocabulary(args: argparse.Namespace, brain=None) -> Vocabulary | None: + if args.no_vocabulary: + logger.info("Vocabulary biasing disabled.") + return None + + # These live in the workspace, not here: they are learned state that grows + # with use, and keeping them beside AGENTS.md means git tracks how they + # change. This repo only carries the starting templates. + here = Path(__file__).parent + workspace = Path(args.cwd) + vocabulary_file = ( + Path(args.vocabulary_file) if args.vocabulary_file else workspace / "vocabulary.txt" + ) + corrections_file = workspace / "corrections.txt" + + for target, template in ( + (vocabulary_file, here / "vocabulary.example.txt"), + (corrections_file, here / "corrections.example.txt"), + ): + if not target.exists() and template.exists(): + target.write_text(template.read_text()) + logger.info(f"Created {target} from the template") + + vocabulary = Vocabulary( + project_dir=workspace, + vocabulary_file=vocabulary_file, + corrections_file=corrections_file, + ) + if brain and brain.projects: + # "CIP-Unified-Cooldown", "pSMSL-RT-Cache" — the words most likely to be + # spoken and least likely to be recognised. + vocabulary.add_terms(brain.projects) + terms = vocabulary.terms() + logger.info(f"Vocabulary: biasing towards {len(terms)} terms, e.g. {', '.join(terms[:6])}") + return vocabulary + + +def _denied_tools(args: argparse.Namespace) -> list[str]: + if getattr(args, "read_only", False): + return [*SHELL_TOOLS, *FILE_EDIT_TOOLS] + if args.allow_writes: + return [] + return FILE_EDIT_TOOLS + + +def read_personality(cwd: str | None) -> str: + """Load AGENTS.md from the working directory, if it's there. + + HTML comments are stripped, so the file can carry notes to whoever edits it + without those notes reaching Claude as instructions. + """ + if not cwd: + return "" + path = Path(cwd) / PERSONALITY_FILE + if not path.exists(): + return "" + text = re.sub(r"", "", path.read_text(), flags=re.DOTALL).strip() + if text: + logger.info(f"Personality: {path} ({len(text)} chars)") + return text + + +def build_claude_options(args: argparse.Namespace, vocabulary=None, brain=None) -> ClaudeAgentOptions: + # The SDK bundles its own stock Claude Code binary and prefers it over the + # one on PATH. That build knows nothing about this org's gateway or its + # managed apiKeyHelper, so every turn fails with "Invalid API key" unless we + # point it back at the installed CLI. + cli_path = shutil.which("claude") + if cli_path: + logger.info(f"Claude CLI: {cli_path}") + else: + logger.warning("No `claude` on PATH; falling back to the SDK's bundled CLI.") + + # Naming the domain vocabulary lets Claude resolve a garbled transcript + # while answering it. That recovers most of what a separate correction pass + # would, at no latency cost, because he already reads every transcript. + # Personality first, then the voice rules, so the constraints of speaking + # aloud get the last word over anything the personality file asks for. + style = "" + if personality := read_personality(args.cwd): + style += personality + "\n\n" + style += VOICE_STYLE + if brain and (memory := brain.prompt_block()): + # Only the context goes in the prompt. When to write back is described + # by the memory skill in the workspace, which Claude picks up on its own. + style += "\n\n" + memory + if vocabulary and (terms := vocabulary.prompt_block()): + style += ( + "\n\nSpeech recognition mangles unusual words. When a transcript is " + "close to one of these, assume that is what was said and carry on " + "without remarking on it:\n" + terms + ) + + memory_server = build_server(workspace=Path(args.cwd), brain=brain) + + return ClaudeAgentOptions( + cli_path=cli_path, + mcp_servers={"memory": memory_server}, + system_prompt={"type": "preset", "preset": "claude_code", "append": style}, + disallowed_tools=_denied_tools(args), + permission_mode="bypassPermissions", + cwd=args.cwd, + model=args.claude_model, + # "project" makes the workspace's own .claude/skills discoverable; + # without it a skills folder there is silently ignored. + setting_sources=( + ["user", "project", "local"] if args.load_settings else ["project"] + ), + skills="all", + include_partial_messages=True, # Speak as tokens arrive instead of per message. + # The CLI's own sandbox uses sandbox-exec, which fails when this process + # is already running inside one. + sandbox=SandboxSettings(enabled=False), + ) + + +async def main() -> int: + args = parse_args() + + if args.list_devices: + print(list_devices()) + return 0 + + if args.list_voices: + from apple_tts import available_voices + + for name, language in sorted(available_voices(), key=lambda v: (v[1], v[0])): + print(f" {language:<8} {name}") + return 0 + + logger.remove() + logger.add(sys.stderr, level=args.log_level) + + workspace = Path(args.cwd) + if not workspace.exists(): + workspace.mkdir(parents=True) + logger.info(f"Created {workspace}") + logger.info(f"Workspace: {workspace}") + + transport = SoundDeviceTransport( + SoundDeviceTransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + audio_in_sample_rate=STT_SAMPLE_RATE, + audio_out_sample_rate=TTS_SAMPLE_RATE, + input_device=as_device(args.input_device), + output_device=as_device(args.output_device), + ) + ) + + brain = None + if not args.no_brain: + brain = Brain() + brain.load() + + vocabulary = build_vocabulary(args, brain) + stt = build_stt(args, vocabulary) + journal = Journal(workspace / "journal.jsonl") + + def on_reply(text: str): + journal.record_reply(text) + if vocabulary: + vocabulary.observe(text) + + llm = ClaudeCodeLLM( + options=build_claude_options(args, vocabulary, brain), + observer=on_reply, + ) + + tts = build_tts(args) + + # Claude keeps its own history; this context exists so Pipecat can decide + # when a turn has ended. + context = LLMContext() + turn_source, turn_strategies, mute_strategies = build_turn_taking(args) + + user_aggregator, assistant_aggregator = LLMContextAggregatorPair( + context, + user_params=LLMUserAggregatorParams( + user_turn_strategies=turn_strategies, + user_mute_strategies=mute_strategies, + ), + ) + + pipeline = Pipeline( + [ + transport.input(), + turn_source, + stt, + *([TranscriptRepair(vocabulary)] if vocabulary else []), + journal, + user_aggregator, + llm, + tts, + transport.output(), + assistant_aggregator, + ] + ) + + worker = PipelineWorker( + pipeline, + params=PipelineParams(enable_metrics=True, enable_usage_metrics=True), + # Off, because "idle" is meaningless here. The timer only resets on + # speech frames, so it cannot tell an abandoned session from a user + # who hasn't pressed the key for a while, or from Claude spending two + # minutes inside a subagent — and on firing it cancels the worker *and* + # the runner, killing the conversation mid-answer. Waiting quietly is + # this program's normal state. + idle_timeout_secs=None, + ) + + if args.greeting: + await worker.queue_frames([TTSSpeakFrame(args.greeting)]) + + runner = WorkerRunner() + await runner.add_workers(worker) + await runner.run() + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/brain.py b/brain.py new file mode 100644 index 0000000..a300f15 --- /dev/null +++ b/brain.py @@ -0,0 +1,192 @@ +"""Read from and write to the Metamate personal brain. + +The brain is the long-term memory: `briefing.md` carries active work and pinned +reminders, `preferences.md` how Adolfo likes to be worked with, `profile.md` who +he is, and `projects/` a directory per workstream. It lives remotely and is +reached through `meta agents.memory`, so everything here shells out. + +Two directions: + +- **In.** The three context files go into the system prompt at startup, so the + agent knows what's active without being told. Project directory names also + become vocabulary, since "CIP-Unified-Cooldown" and "pSMSL" are exactly the + words a recogniser mangles. +- **Out.** Durable things learned in conversation are appended back, following + the discipline the brain's own README sets out: lasting preferences to + `preferences.md`, techniques and lessons to `notes.md`. + +`briefing.md` is deliberately *not* written to automatically. A daily cron owns +that file, and an agent appending to it unprompted would fight the cron and +corrupt the one file everything else reads first. +""" + +import json +import subprocess +from pathlib import Path + +from loguru import logger + +META = "meta" +CONTEXT_FILES = ("briefing.md", "preferences.md", "profile.md") + +# Writes go only to files a human owns, never to cron-managed ones. +WRITABLE = { + "preference": "preferences.md", + "note": "notes.md", +} + +# Learned entries get their own section so they never land inside a hand-written +# one, and so it stays obvious which lines the agent added. +LEARNED_HEADING = "## Learned in voice sessions" + +_READ_TIMEOUT = 25.0 +_WRITE_TIMEOUT = 25.0 + +# Used only when the brain can't be reached, so a flight or a VPN drop doesn't +# cost the agent all of its context. +CACHE = Path.home() / ".cache" / "voice-agent" / "brain-context.json" + + +def _run(args: list[str], timeout: float) -> tuple[bool, str]: + try: + result = subprocess.run( + [META, "agents.memory", *args], + capture_output=True, + text=True, + timeout=timeout, + ) + except (OSError, subprocess.SubprocessError) as e: + return False, str(e) + if result.returncode != 0: + return False, (result.stderr.strip() or result.stdout.strip())[:200] + return True, result.stdout + + +class Brain: + """The personal brain, or a graceful no-op when it can't be reached.""" + + def __init__(self): + self._context: dict[str, str] = {} + self._projects: list[str] = [] + self.reachable = False + + def load(self) -> bool: + """Fetch context and project names. Falls back to cache when offline.""" + ok, out = _run( + ["read-batch", f"--paths={','.join(CONTEXT_FILES)}", "--output=json"], + _READ_TIMEOUT, + ) + if ok: + try: + payload = json.loads(out) + except json.JSONDecodeError: + ok = False + else: + self._context = { + name: entry.get("content", "") + for name, entry in payload.items() + if isinstance(entry, dict) and entry.get("content") + } + self.reachable = bool(self._context) + + if self.reachable: + self._projects = self._list_projects() + self._save_cache() + logger.info( + f"Brain: loaded {len(self._context)} context files, " + f"{len(self._projects)} projects" + ) + return True + + if self._load_cache(): + logger.warning(f"Brain unreachable ({out[:80]}); using cached context.") + return True + logger.warning(f"Brain unavailable: {out[:120]}") + return False + + def _list_projects(self) -> list[str]: + ok, out = _run(["list", "--path=projects", "-l", "200"], _READ_TIMEOUT) + if not ok: + return [] + names = [] + for line in out.splitlines(): + parts = line.split() + # Rows look like "NAME dir -"; skip the header and rules. + if len(parts) >= 2 and parts[1] == "dir": + names.append(parts[0]) + return names + + def _save_cache(self): + try: + CACHE.parent.mkdir(parents=True, exist_ok=True) + CACHE.write_text( + json.dumps({"context": self._context, "projects": self._projects}) + ) + except OSError: + pass + + def _load_cache(self) -> bool: + try: + payload = json.loads(CACHE.read_text()) + except (OSError, json.JSONDecodeError): + return False + self._context = payload.get("context", {}) + self._projects = payload.get("projects", []) + return bool(self._context) + + @property + def projects(self) -> list[str]: + return self._projects + + def prompt_block(self) -> str: + """The context files, framed as reference rather than as instructions. + + Framing matters: briefing.md is dense markdown with links and bold, and + without a clear label the agent starts answering in the same register — + which is wrong out loud. + """ + if not self._context: + return "" + sections = [ + f"### {name}\n{text.strip()}" + for name, text in self._context.items() + if text.strip() + ] + if not sections: + return "" + return ( + "Below is your memory of Adolfo's work, from his personal brain. " + "Treat it as reference you already know, not as something to read " + "back. It is written notes — never mirror their formatting when you " + "speak.\n\n" + "\n\n".join(sections) + ) + + def append(self, kind: str, content: str) -> tuple[bool, str]: + """Append a line to one of the writable brain files. + + Args: + kind: A key of ``WRITABLE`` — "preference" or "note". + content: One line, already phrased as a durable statement. + """ + path = WRITABLE.get(kind) + if not path: + return False, f"nothing writable for {kind!r}; expected {sorted(WRITABLE)}" + if not content.strip(): + return False, "refusing to write empty content" + + # Append lands at the end of the file, which would tuck the new line + # inside whatever the last section happens to be. These are curated + # files, so learned entries get their own heading the first time. + body = content.strip() + ok, existing = _run(["read", f"--path=/{path}"], _READ_TIMEOUT) + if ok and LEARNED_HEADING not in existing: + body = f"\n{LEARNED_HEADING}\n{body}" + + ok, out = _run( + ["append", f"--path={path}", f"--content={body}"], + _WRITE_TIMEOUT, + ) + if ok: + logger.info(f"Brain: appended a {kind} to {path}") + return True, path + return False, out diff --git a/claude_llm.py b/claude_llm.py new file mode 100644 index 0000000..2092e41 --- /dev/null +++ b/claude_llm.py @@ -0,0 +1,267 @@ +"""A Pipecat processor that puts Claude Code in the LLM slot of a voice pipeline. + +This drives the Claude Agent SDK rather than the raw Anthropic API, so the +conversation runs through the same `claude` CLI (and the same auth) that a +terminal session uses, and Claude keeps its tools. + +The SDK owns conversation history, so the Pipecat context is used only for turn +detection: this reads the newest user message off each `LLMContextFrame` and +emits the `LLMFullResponseStartFrame` / `LLMTextFrame` / +`LLMFullResponseEndFrame` sequence the TTS service downstream expects. +""" + +import asyncio + +from claude_agent_sdk import ( + AssistantMessage, + ClaudeAgentOptions, + ClaudeSDKClient, + ResultMessage, + StreamEvent, + TextBlock, + ToolUseBlock, +) +from loguru import logger + +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + Frame, + InterruptionFrame, + LLMContextFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMTextFrame, + StartFrame, + TTSSpeakFrame, +) +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor + +# Whisper hallucinates these on silence or breath noise. Treating them as speech +# makes the agent answer questions nobody asked. +_NOISE_TRANSCRIPTS = { + "", + ".", + "thank you.", + "thanks for watching!", + "you", + "bye.", + "okay.", + "[blank_audio]", + "[silence]", +} + + +class ClaudeCodeLLM(FrameProcessor): + """Runs each user utterance through a persistent Claude Code session.""" + + def __init__( + self, *, options: ClaudeAgentOptions, observer=None, working_phrase: str | None = "One moment.", **kwargs + ): + super().__init__(**kwargs) + # Said once per turn if a tool runs before any answer has begun. Silence + # while Claude works reads as a crash: in one recorded session four of + # nine turns went unanswered because tool-heavy turns took over a minute + # with no sound, and speaking again to check cancels the turn in flight. + self._working_phrase = working_phrase + self._said_working = False + self._options = options + self._client: ClaudeSDKClient | None = None + self._turn_task: asyncio.Task | None = None + # Called with each completed reply. What Claude is talking about + # predicts what the user will say next, so this feeds the recognizer. + # + # Not named _observer: FrameProcessor owns that attribute and reassigns + # it during pipeline setup, so it would be silently replaced by + # pipecat's own observer and then called with the wrong signature. + self._on_reply = observer + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, StartFrame): + await self.push_frame(frame, direction) + await self._connect() + elif isinstance(frame, (EndFrame, CancelFrame)): + await self._cancel_turn() + await self._disconnect() + await self.push_frame(frame, direction) + elif isinstance(frame, InterruptionFrame): + await self._cancel_turn() + await self.push_frame(frame, direction) + elif isinstance(frame, LLMContextFrame): + await self._maybe_start_turn(_latest_user_text(frame.context)) + else: + await self.push_frame(frame, direction) + + async def _connect(self): + if self._client: + return + logger.info("Starting Claude Code session...") + client = ClaudeSDKClient(options=self._options) + try: + await client.connect() + except Exception as e: + self._client = None + logger.error( + f"Could not start the Claude CLI: {e}\n" + "Exit code -9 means the CLI was killed rather than failing on its own. " + "On a managed Mac that is usually either the binary still awaiting " + "approval, or its sandbox failing to apply because this process is " + "already inside one — in which case run `./talk` from a normal terminal." + ) + return + self._client = client + logger.info("Claude Code session ready.") + + async def _disconnect(self): + if not self._client: + return + await self._client.disconnect() + self._client = None + + async def _maybe_start_turn(self, text: str): + utterance = text.strip() + if utterance.lower() in _NOISE_TRANSCRIPTS or len(utterance) < 2: + logger.debug(f"Ignoring noise transcript: {utterance!r}") + return + if not self._client: + logger.warning("Transcript arrived before the Claude session was ready; dropping it.") + return + + # A new utterance supersedes whatever Claude was in the middle of saying. + await self._cancel_turn() + + logger.info(f"You: {utterance}") + self._turn_task = self.create_task(self._run_turn(utterance)) + + async def _cancel_turn(self): + if not self._turn_task: + return + task, self._turn_task = self._turn_task, None + if self._client: + try: + await self._client.interrupt() + except Exception as e: + logger.debug(f"Interrupt failed (session may be idle): {e}") + await self.cancel_task(task) + + async def _say_working(self, spoken: list[str]): + """Break the silence before a slow tool call, once per turn.""" + if self._said_working or spoken or not self._working_phrase: + return + self._said_working = True + await self.push_frame(TTSSpeakFrame(self._working_phrase)) + + async def _run_turn(self, utterance: str): + await self.push_frame(LLMFullResponseStartFrame()) + self._said_working = False + spoken: list[str] = [] + try: + await self._client.query(utterance) + async for message in self._client.receive_response(): + # Subagent chatter carries a parent tool id; only speak the main thread. + if getattr(message, "parent_tool_use_id", None): + continue + + if isinstance(message, StreamEvent): + text = _text_delta(message) + if text: + spoken.append(text) + await self.push_frame(LLMTextFrame(text)) + elif isinstance(message, AssistantMessage): + for block in message.content: + if isinstance(block, ToolUseBlock): + logger.info(f" [tool] {block.name}") + await self._say_working(spoken) + elif isinstance(block, TextBlock) and not self._streams_partials(): + spoken.append(block.text) + await self.push_frame(LLMTextFrame(block.text)) + elif isinstance(message, ResultMessage): + if message.is_error: + _log_result_error(message) + except asyncio.CancelledError: + raise + except Exception as e: + logger.exception(f"Claude turn failed: {e}") + finally: + if spoken: + reply = "".join(spoken) + logger.info(f"Claude: {reply}") + if self._on_reply: + # Isolated deliberately: this frame ends the turn for the + # TTS and the aggregator, so nothing optional may prevent + # it being pushed. + try: + self._on_reply(reply) + except Exception as e: + logger.warning(f"Reply observer failed: {e}") + await self.push_frame(LLMFullResponseEndFrame()) + + def _streams_partials(self) -> bool: + return self._options.include_partial_messages + + +def _log_result_error(message: ResultMessage): + """Report a failed turn using whichever field actually says something. + + ``result`` is often None on a failure — it holds the reply text, and a turn + that failed has none — so logging it alone produced "Claude returned an + error: None" and told us nothing. The diagnosis lives in the other fields. + """ + details = { + field: value + for field in ( + "subtype", + "stop_reason", + "terminal_reason", + "api_error_status", + "errors", + "permission_denials", + "result", + ) + if (value := getattr(message, field, None)) + } + + # Cutting Claude off mid-answer is a normal part of talking, and the CLI + # reports it the same way it reports a genuine failure. Don't cry wolf. + if details.get("terminal_reason") in ( + "interrupted", + "cancelled", + # What the CLI reports when interrupt() lands mid-stream, which is + # simply what talking over Claude looks like from its side. + "aborted_streaming", + ): + logger.debug(f"Turn interrupted: {details}") + return + + logger.error(f"Claude turn failed: {details or 'no detail reported'}") + + +def _latest_user_text(context: LLMContext) -> str: + """Return the text of the most recent user message in the context.""" + for message in reversed(context.get_messages()): + if not isinstance(message, dict) or message.get("role") != "user": + continue + content = message.get("content") + if isinstance(content, str): + return content + if isinstance(content, list): + return " ".join( + part.get("text", "") + for part in content + if isinstance(part, dict) and part.get("type") == "text" + ) + return "" + + +def _text_delta(event: StreamEvent) -> str | None: + """Pull assistant text out of a raw stream event, ignoring thinking and tool input.""" + raw = event.event or {} + if raw.get("type") != "content_block_delta": + return None + delta = raw.get("delta") or {} + if delta.get("type") != "text_delta": + return None + return delta.get("text") or None diff --git a/compare_engines.py b/compare_engines.py new file mode 100755 index 0000000..49facf7 --- /dev/null +++ b/compare_engines.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Record yourself once, transcribe with every engine, and compare. + +Every accuracy number in ACCURACY.md comes from synthesised speech, which is +cleaner and more evenly paced than a person at a microphone, and from a +vocabulary tuned to those same sentences. This runs the same comparison on your +voice, in your room, which is the only measurement that decides anything. + + .venv/bin/python compare_engines.py # record and compare + .venv/bin/python compare_engines.py --seconds 12 + .venv/bin/python compare_engines.py --file clip.wav # reuse a recording + .venv/bin/python compare_engines.py --say "the exact words you spoke" +""" + +import argparse +import json +import subprocess +import sys +import time +import wave +from pathlib import Path + +HERE = Path(__file__).parent +HELPER = HERE / "swift" / "speech-helper" +RATE = 16000 + + +def record(seconds: float, path: Path): + import numpy as np + import sounddevice as sd + + print(f"Recording {seconds:.0f}s — speak now, ideally something with jargon in it.") + for count in (3, 2, 1): + print(f" {count}...", end="\r", flush=True) + time.sleep(0.6) + print(" GO ") + audio = sd.rec(int(seconds * RATE), samplerate=RATE, channels=1, dtype="int16") + sd.wait() + peak = int(np.abs(audio).max()) + with wave.open(str(path), "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(RATE) + w.writeframes(audio.tobytes()) + print(f"Recorded, peak amplitude {peak}" + (" (very quiet — check the mic)" if peak < 1500 else "")) + return peak + + +def run_helper(path: Path, terms: list[str], dictation: bool) -> str: + args = [str(HELPER), str(path), "--locale", "en-US"] + if terms: + args += ["--terms", ",".join(terms)] + if dictation: + args.append("--dictation") + result = subprocess.run(args, capture_output=True, text=True, timeout=300) + if result.returncode != 0: + return f"" + try: + return json.loads(result.stdout).get("text", "") + except json.JSONDecodeError: + return "" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--seconds", type=float, default=10.0) + parser.add_argument("--file", help="Score an existing 16kHz mono WAV instead of recording.") + parser.add_argument("--say", help="What you actually said, to score word error rate.") + args = parser.parse_args() + + sys.path.insert(0, str(HERE)) + from vocabulary import Vocabulary + + vocabulary = Vocabulary( + project_dir=HERE, + vocabulary_file=HERE / "vocabulary.txt", + corrections_file=HERE / "corrections.txt", + ) + terms = vocabulary.terms() + + path = Path(args.file) if args.file else Path("/tmp/compare-engines.wav") + if not args.file: + record(args.seconds, path) + + engines = [ + ("SpeechTranscriber (default)", lambda: run_helper(path, [], False)), + ("SpeechTranscriber + repair", lambda: vocabulary.repair(run_helper(path, [], False))), + ("DictationTranscriber + vocab", lambda: run_helper(path, terms, True)), + ("DictationTranscriber + vocab + repair", + lambda: vocabulary.repair(run_helper(path, terms, True))), + ] + try: + from apple_stt import _recognize_file, timeout_for + + with wave.open(str(path)) as w: + duration = w.getnframes() / w.getframerate() + engines.append( + ("old SFSpeechRecognizer + vocab + repair", + lambda: vocabulary.repair(_recognize_file(str(path), "en-US", timeout_for(duration)))) + ) + except Exception: + pass + + print() + scorer = None + if args.say: + sys.path.insert(0, "/tmp") + try: + from wer2 import wer as scorer + except ImportError: + scorer = None + + for label, run in engines: + started = time.time() + try: + text = run() + except Exception as e: + text = f"<{e}>" + elapsed = time.time() - started + suffix = "" + if scorer and args.say and not text.startswith("<"): + errors, total = scorer(args.say, text) + suffix = f" [WER {100 * errors / max(total, 1):.0f}%]" + print(f" {label}{suffix}") + print(f" {text}") + print(f" ({elapsed:.2f}s)\n") + + print("Pick whichever reads closest to what you said.") + print(" default is SpeechTranscriber; --stt-engine apple or") + print(" --analyzer-module dictation switch to the older model.") + + +if __name__ == "__main__": + main() diff --git a/corrections.example.txt b/corrections.example.txt new file mode 100644 index 0000000..8479ddc --- /dev/null +++ b/corrections.example.txt @@ -0,0 +1,35 @@ +# Fixes for words the recognizer gets wrong the same way every time. +# +# heard => replacement +# +# Matching is case-insensitive and word-bounded, so "coral voice" is rewritten +# mid-sentence but "chorale" is left alone. Everything after a # is ignored. +# +# This is the blunt instrument, and that is the point: it is exact, testable, +# and costs nothing at runtime. Vocabulary biasing (vocabulary.txt) is the +# softer tool that stops the mistake happening at all — reach for that first, +# and add a rule here only once you have seen the SAME wrong word more than +# once. A rule is blind to context, so make each one specific enough that it +# cannot fire on ordinary speech: prefer "coral voice" over bare "coral". +# +# These were observed in testing; delete any that don't match how you speak. +# The two engines mishear differently, so both sets are here — the rules are +# specific enough not to collide. + +# SpeechTranscriber (the default). Its errors are phonetically close, which is +# what makes short rules like these enough. +Kakoro => Kokoro +Pipika => Pipecat +Metemma => Metamate +echo tale => echo tail +graph QL => GraphQL + +# The older dictation model, used by --stt-engine apple and --analyzer-module +# dictation. It fails further from the target, so it needs vocabulary biasing +# as well as these. +coral voice => Kokoro voice +pit transport => Pipecat transport +pipe cat => Pipecat +LN point => endpoint +fab ricator => Phabricator +meta mate => Metamate diff --git a/echo_guard.py b/echo_guard.py new file mode 100644 index 0000000..81d4eb2 --- /dev/null +++ b/echo_guard.py @@ -0,0 +1,43 @@ +"""Stop the agent from hearing itself. + +On laptop speakers with no acoustic echo cancellation, the microphone picks up +whatever Kokoro just said. The recognizer transcribes it, the aggregator treats +it as a user turn, and the agent answers its own greeting — which is exactly +what happens without this. + +Pipecat's ``AlwaysUserMuteStrategy`` mutes while the bot is speaking, but +unmutes the instant playback ends, and the tail of that audio is still in the +input buffer. This keeps the mute up for a short while afterwards. +""" + +import time + +from pipecat.frames.frames import BotStartedSpeakingFrame, BotStoppedSpeakingFrame, Frame +from pipecat.turns.user_mute.base_user_mute_strategy import BaseUserMuteStrategy + + +class EchoGuardUserMuteStrategy(BaseUserMuteStrategy): + """Mute the user while the bot speaks, plus a tail to let echo drain. + + Args: + tail_secs: How long to stay muted after playback ends. + """ + + def __init__(self, *, tail_secs: float = 0.4): + super().__init__() + self._tail_secs = tail_secs + self._bot_speaking = False + self._stopped_at = 0.0 + + async def process_frame(self, frame: Frame) -> bool: + await super().process_frame(frame) + + if isinstance(frame, BotStartedSpeakingFrame): + self._bot_speaking = True + elif isinstance(frame, BotStoppedSpeakingFrame): + self._bot_speaking = False + self._stopped_at = time.monotonic() + + if self._bot_speaking: + return True + return time.monotonic() - self._stopped_at < self._tail_secs diff --git a/global_hotkey.py b/global_hotkey.py new file mode 100644 index 0000000..d0f8011 --- /dev/null +++ b/global_hotkey.py @@ -0,0 +1,136 @@ +"""Watch a modifier key system-wide, so push-to-talk works from any app. + +Reading stdin only works while the terminal has focus, which defeats the point +of a voice assistant you talk to while doing something else. A Quartz event tap +sees key events no matter what is frontmost, and — unlike stdin — reports +releases as well as presses, so this can be true hold-to-talk. + +The tap runs on its own thread with its own CFRunLoop and hands state changes +back to asyncio via ``call_soon_threadsafe``. + +Requires Input Monitoring permission for whichever app runs this (your +terminal), granted under System Settings > Privacy & Security > Input +Monitoring. +""" + +import asyncio +import threading +from collections.abc import Callable + +from loguru import logger + +import Quartz +from CoreFoundation import ( + CFMachPortCreateRunLoopSource, + CFRunLoopAddSource, + CFRunLoopGetCurrent, + CFRunLoopRun, + CFRunLoopStop, + kCFRunLoopCommonModes, +) + +# (keycode, modifier mask) for the keys worth holding. Modifier keycodes arrive +# on flagsChanged events, so one handler covers all of them. +HOLD_KEYS: dict[str, tuple[int, int]] = { + "fn": (63, Quartz.kCGEventFlagMaskSecondaryFn), + "right-option": (61, Quartz.kCGEventFlagMaskAlternate), + "right-command": (54, Quartz.kCGEventFlagMaskCommand), + "right-control": (62, Quartz.kCGEventFlagMaskControl), + "right-shift": (60, Quartz.kCGEventFlagMaskShift), +} + + +def permission_granted() -> bool: + """Whether this process may observe keyboard events.""" + return bool(Quartz.CGPreflightListenEventAccess()) + + +def request_permission() -> bool: + """Ask for Input Monitoring, which prompts once and then opens Settings.""" + return bool(Quartz.CGRequestListenEventAccess()) + + +class HoldKeyMonitor: + """Report press and release of one modifier key, from anywhere in the OS. + + Args: + key: A name from ``HOLD_KEYS``. + on_change: Called with True on press and False on release, on the + asyncio loop. + loop: The loop to deliver callbacks on. + """ + + def __init__(self, *, key: str, on_change: Callable[[bool], None], loop): + if key not in HOLD_KEYS: + raise ValueError(f"Unsupported hold key {key!r}. Choose from {list(HOLD_KEYS)}.") + self._key = key + self._keycode, self._mask = HOLD_KEYS[key] + self._on_change = on_change + self._loop = loop + self._thread: threading.Thread | None = None + self._runloop = None + self._tap = None + self._down = False + self._ready = threading.Event() + self._started_ok = False + + def start(self) -> bool: + """Begin watching. Returns False if the tap could not be created.""" + self._thread = threading.Thread(target=self._run, name="hold-key-tap", daemon=True) + self._thread.start() + self._ready.wait(timeout=5) + return self._started_ok + + def stop(self): + if self._runloop is not None: + CFRunLoopStop(self._runloop) + self._runloop = None + + def _run(self): + tap = Quartz.CGEventTapCreate( + Quartz.kCGSessionEventTap, + Quartz.kCGHeadInsertEventTap, + # Listen only: we observe the key without swallowing it, so we never + # break whatever else the user has bound to it. + Quartz.kCGEventTapOptionListenOnly, + Quartz.CGEventMaskBit(Quartz.kCGEventFlagsChanged), + self._callback, + None, + ) + if tap is None: + self._started_ok = False + self._ready.set() + return + + self._tap = tap + source = CFMachPortCreateRunLoopSource(None, tap, 0) + self._runloop = CFRunLoopGetCurrent() + CFRunLoopAddSource(self._runloop, source, kCFRunLoopCommonModes) + Quartz.CGEventTapEnable(tap, True) + + self._started_ok = True + self._ready.set() + CFRunLoopRun() + + def _callback(self, proxy, event_type, event, refcon): + # macOS disables a tap that takes too long; turn it back on. + if event_type in ( + Quartz.kCGEventTapDisabledByTimeout, + Quartz.kCGEventTapDisabledByUserInput, + ): + if self._tap is not None: + Quartz.CGEventTapEnable(self._tap, True) + return event + + try: + keycode = Quartz.CGEventGetIntegerValueField( + event, Quartz.kCGKeyboardEventKeycode + ) + if keycode == self._keycode: + down = bool(Quartz.CGEventGetFlags(event) & self._mask) + if down != self._down: + self._down = down + self._loop.call_soon_threadsafe(self._on_change, down) + except Exception as e: # never let an exception cross back into C + logger.debug(f"Hold-key tap callback error: {e}") + return event diff --git a/journal.py b/journal.py new file mode 100644 index 0000000..13e58dd --- /dev/null +++ b/journal.py @@ -0,0 +1,71 @@ +"""Record what was said and what came back, one line of JSON per turn. + +Every improvement so far has come from measuring something rather than guessing +at it, and there is no record of real conversations to measure. This writes one +so questions like "which words does it mishear most often" have an answer that +isn't synthesised audio. + +The two halves arrive by different routes, which is the awkward part. A +transcript reaches this processor as a frame, but replies never do: the user +aggregator *consumes* TranscriptionFrames rather than forwarding them, so a +processor placed late enough to see LLMTextFrames is already too late to see +the transcript. Sitting early and taking the reply through the LLM's own +callback is the only placement that sees both. + +Deliberately not in git: it is append-only, grows without bound, and is a +verbatim record of everything said near the microphone. +""" + +import json +from datetime import datetime +from pathlib import Path + +from loguru import logger + +from pipecat.frames.frames import CancelFrame, EndFrame, Frame, TranscriptionFrame +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor + + +class Journal(FrameProcessor): + """Log each turn as JSONL. Place it just after the transcript repair.""" + + def __init__(self, path: Path, **kwargs): + super().__init__(**kwargs) + self._path = path + self._heard: str | None = None + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, TranscriptionFrame): + # A second transcript before any reply means the previous turn was + # abandoned — record it anyway, since an unanswered utterance is + # exactly the kind of failure worth being able to find later. + if self._heard: + self._write(self._heard, None) + self._heard = frame.text + elif isinstance(frame, (EndFrame, CancelFrame)) and self._heard: + self._write(self._heard, None) + self._heard = None + + await self.push_frame(frame, direction) + + def record_reply(self, reply: str): + """Called by the LLM with each completed answer.""" + self._write(self._heard, reply) + self._heard = None + + def _write(self, heard: str | None, reply: str | None): + if not heard and not reply: + return + entry = { + "at": datetime.now().isoformat(timespec="seconds"), + "heard": heard, + "reply": reply, + } + try: + self._path.parent.mkdir(parents=True, exist_ok=True) + with self._path.open("a") as f: + f.write(json.dumps(entry) + "\n") + except OSError as e: + logger.debug(f"Could not write the journal: {e}") diff --git a/memory_tools.py b/memory_tools.py new file mode 100644 index 0000000..7e7bd32 --- /dev/null +++ b/memory_tools.py @@ -0,0 +1,97 @@ +"""Tools that let the agent improve itself between conversations. + +Three kinds of thing get learned in a voice conversation, and they belong in +different places: + +- **How a word was misheard** is about this microphone and this recogniser. It + goes to `corrections.txt` in the workspace — local, because it would be + meaningless on another machine. +- **A lasting preference** — "stop explaining so much" — goes to the personal + brain's `preferences.md`, because it is true of Adolfo regardless of which + assistant is listening. +- **A technique or lesson** goes to the brain's `notes.md`, same reasoning. + +This module is only the mechanism. *When* to record something, and which file +it belongs in, is personal setup rather than a property of the agent, so that +lives in a skill at `~/Workspace/.claude/skills/memory/`. Editing the discipline +means editing that file, not this one. + +Everything written lands in a git-tracked file or the brain, so it can be +reviewed and undone. +""" + +from pathlib import Path + +from claude_agent_sdk import create_sdk_mcp_server, tool +from loguru import logger + +SERVER_NAME = "memory" + +# The tool names Claude sees, and must be allowed to call. +TOOL_NAMES = [ + f"mcp__{SERVER_NAME}__remember_correction", + f"mcp__{SERVER_NAME}__remember_preference", + f"mcp__{SERVER_NAME}__remember_note", +] + +def build_server(*, workspace: Path, brain=None): + """Create the in-process MCP server exposing the memory tools.""" + corrections_file = workspace / "corrections.txt" + + @tool( + "remember_correction", + "Record that speech recognition misheard a word, so it is fixed from now on.", + {"heard": str, "intended": str}, + ) + async def remember_correction(args): + heard = (args.get("heard") or "").strip() + intended = (args.get("intended") or "").strip() + if not heard or not intended or heard.lower() == intended.lower(): + return {"content": [{"type": "text", "text": "Nothing to record."}]} + + # A one-word rule risks firing on ordinary speech; a phrase is safer. + rule = f"{heard} => {intended}" + existing = corrections_file.read_text() if corrections_file.exists() else "" + if rule.lower() in existing.lower(): + return {"content": [{"type": "text", "text": "Already known."}]} + + with corrections_file.open("a") as f: + if not existing.endswith("\n"): + f.write("\n") + f.write(f"{rule}\n") + logger.info(f"Learned correction: {rule}") + return {"content": [{"type": "text", "text": f"Recorded: {rule}"}]} + + @tool( + "remember_preference", + "Record a lasting preference about how Adolfo wants to be worked with.", + {"preference": str}, + ) + async def remember_preference(args): + return _to_brain(brain, "preference", args.get("preference")) + + @tool( + "remember_note", + "Record a technique, lesson or fact worth having in future sessions.", + {"note": str}, + ) + async def remember_note(args): + return _to_brain(brain, "note", args.get("note")) + + return create_sdk_mcp_server( + name=SERVER_NAME, + tools=[remember_correction, remember_preference, remember_note], + ) + + +def _to_brain(brain, kind: str, text: str | None) -> dict: + text = (text or "").strip() + if not text: + return {"content": [{"type": "text", "text": "Nothing to record."}]} + if brain is None or not brain.reachable: + logger.warning(f"Brain unreachable; dropped {kind}: {text[:60]}") + return {"content": [{"type": "text", "text": "Memory unavailable right now."}]} + + ok, detail = brain.append(kind, f"- {text}") + message = f"Recorded to {detail}." if ok else f"Could not record: {detail}" + return {"content": [{"type": "text", "text": message}]} diff --git a/push_to_talk.py b/push_to_talk.py new file mode 100644 index 0000000..c047dbe --- /dev/null +++ b/push_to_talk.py @@ -0,0 +1,199 @@ +"""Keyboard-driven turn taking: nothing is heard until you say so. + +Voice activity detection has to guess when you've finished a sentence, and it +guesses slowly — Silero waits 0.2s of silence, then the turn strategy waits +another 0.6s in case you say more. A key removes both the waiting and the +guessing: the turn ends the moment you say it does. + +It also makes echo structurally impossible. Transcription only runs over audio +captured while the key is engaged, so the agent cannot hear its own voice +however loud the speakers are. + +Two input modes: + +- **hold** (default): a Quartz event tap watches a modifier key system-wide, so + it works whatever app has focus, and reports releases as well as presses — + real hold-to-talk. Needs Input Monitoring permission. +- **toggle**: reads stdin. Works with no permissions, but only while the + terminal has focus, and a terminal never sees key releases — so it's press to + start, press again to send. +""" + +import asyncio +import atexit +import sys +import termios +import tty + +from loguru import logger + +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + Frame, + StartFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, + VADUserStartedSpeakingFrame, + VADUserStoppedSpeakingFrame, +) +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor + + +class PushToTalk(FrameProcessor): + """Turn a key into the microphone's on-air switch. + + Emits the VAD frames the segmented STT uses to cut audio into utterances, + and the user-turn frames the external turn strategies use to open and close + a turn — so one key drives both. + + Args: + hold_key: Modifier to hold, from ``global_hotkey.HOLD_KEYS``. None + selects the stdin toggle instead. + toggle_key: Character that toggles talking in stdin mode. + """ + + def __init__(self, *, hold_key: str | None = "fn", toggle_key: str = " ", **kwargs): + super().__init__(**kwargs) + self._hold_key = hold_key + self._toggle_key = toggle_key + self._talking = False + self._monitor = None + self._fd: int | None = None + self._saved_term: list | None = None + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, StartFrame): + await self.push_frame(frame, direction) + self._start_input() + elif isinstance(frame, (EndFrame, CancelFrame)): + self._stop_input() + await self.push_frame(frame, direction) + else: + await self.push_frame(frame, direction) + + # ------------------------------------------------------------------ input + + def _start_input(self): + holding = bool(self._hold_key) and self._start_hold_mode() + # Always arm the toggle as well. Without Input Monitoring, macOS still + # hands back a valid-looking event tap and simply never delivers events + # to it — so hold mode can fail silently, and this guarantees there is + # always some way to talk. + self._start_toggle_mode(primary=not holding) + + def _start_hold_mode(self) -> bool: + from global_hotkey import HoldKeyMonitor, permission_granted, request_permission + + if not permission_granted(): + logger.warning("Input Monitoring permission is not granted; requesting it.") + request_permission() + if not permission_granted(): + logger.warning( + "Still not granted. Add your terminal under System Settings > " + "Privacy & Security > Input Monitoring, then restart it. " + "Falling back to the spacebar toggle for now." + ) + return False + + monitor = HoldKeyMonitor( + key=self._hold_key, + on_change=self._on_hold_change, + loop=self.get_event_loop(), + ) + if not monitor.start(): + logger.warning("Could not create the keyboard event tap; using the toggle instead.") + return False + + self._monitor = monitor + label = "🌐 fn" if self._hold_key == "fn" else self._hold_key + logger.info(f"Hold {label} to talk — works in any app, release to send.") + if self._hold_key == "fn": + logger.info( + "If fn also opens the emoji picker, set System Settings > Keyboard > " + "'Press 🌐 to' → Do Nothing." + ) + return True + + def _start_toggle_mode(self, *, primary: bool = True): + if not sys.stdin.isatty(): + if primary: + logger.warning("stdin is not a terminal; push-to-talk is disabled.") + return + + self._fd = sys.stdin.fileno() + self._saved_term = termios.tcgetattr(self._fd) + # Restore the terminal even on an unhandled exception, or the shell is + # left with echo off and no line editing. + atexit.register(self._restore_terminal) + tty.setcbreak(self._fd) + self.get_event_loop().add_reader(self._fd, self._on_stdin_readable) + + label = "SPACE" if self._toggle_key == " " else repr(self._toggle_key) + if primary: + logger.info(f"Push to talk: press {label} to start speaking, {label} again to send.") + else: + logger.info(f"({label} also works as a toggle, if the hold key goes quiet.)") + + def _stop_input(self): + if self._monitor: + self._monitor.stop() + self._monitor = None + if self._fd is not None: + try: + self.get_event_loop().remove_reader(self._fd) + except Exception: + pass + self._restore_terminal() + + def _restore_terminal(self): + if self._fd is not None and self._saved_term is not None: + termios.tcsetattr(self._fd, termios.TCSADRAIN, self._saved_term) + self._saved_term = None + + def _on_hold_change(self, down: bool): + self.create_task(self._start_talking() if down else self._stop_talking()) + + def _on_stdin_readable(self): + try: + char = sys.stdin.read(1) + except (OSError, ValueError): + return + if not char: + return + # Ctrl-C doesn't raise KeyboardInterrupt in cbreak mode; deliver it. + if char == "\x03": + self._restore_terminal() + raise KeyboardInterrupt + if char == self._toggle_key: + self.create_task(self._toggle()) + + # ------------------------------------------------------------------ turns + + async def _start_talking(self): + if self._talking: + return + self._talking = True + logger.info("🎤 listening") + # ExternalUserTurnStartStrategy hardcodes enable_interruptions=False, on + # the assumption that whatever drives it externally handles this. That's + # us: without this, talking over Claude doesn't stop him. + await self.broadcast_interruption() + await self.push_frame(VADUserStartedSpeakingFrame()) + await self.push_frame(UserStartedSpeakingFrame()) + + async def _stop_talking(self): + if not self._talking: + return + self._talking = False + logger.info("… sent") + await self.push_frame(VADUserStoppedSpeakingFrame()) + await self.push_frame(UserStoppedSpeakingFrame()) + + async def _toggle(self): + if self._talking: + await self._stop_talking() + else: + await self._start_talking() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2d7f9c5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,26 @@ +# Install with the venv's pip so it picks up .venv/pip.conf, which points TLS +# verification at the system CA bundle: +# +# .venv/bin/python -m pip install -r requirements.txt + +pipecat-ai[silero,kokoro,mlx-whisper]==1.7.0 + +# Apple's on-device dictation model — the default speech-to-text engine. +# AVFoundation also enumerates the macOS system voices for --tts apple. +pyobjc-framework-Speech==12.2.1 +pyobjc-framework-AVFoundation==12.2.1 + +# Quartz event tap, for watching the hold-to-talk key from any app. +pyobjc-framework-Quartz==12.2.1 + +# Microphone and speakers. Used instead of PyAudio because its wheel bundles a +# prebuilt portaudio, and this machine can neither install Homebrew's nor +# compile one. +sounddevice==0.5.5 + +# Claude, driving the same `claude` CLI a terminal session uses. +claude-agent-sdk==0.2.131 + +# Whisper fallbacks. faster-whisper is also imported unconditionally by +# pipecat's whisper module, so it is required even when unused. +faster-whisper==1.2.1 diff --git a/selftest.py b/selftest.py new file mode 100755 index 0000000..3076903 --- /dev/null +++ b/selftest.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +"""Check each piece of the voice pipeline independently. + +Run this first when something isn't working — it tells you which stage is at +fault instead of making you read pipeline logs. +""" + +import asyncio +import sys +import time +import wave +from pathlib import Path + +PHRASE = "The quick brown fox jumps over the lazy dog." +KOKORO_CACHE = Path.home() / ".cache/pipecat/kokoro-onnx" + +results: list[tuple[str, bool, str]] = [] + + +def record(name: str, ok: bool, detail: str): + results.append((name, ok, detail)) + print(f" {'PASS' if ok else 'FAIL'} {name}: {detail}") + + +def check_devices(): + import sounddevice as sd + + try: + default_in, default_out = sd.default.device + names = sd.query_devices() + record( + "audio devices", + True, + f"in={names[default_in]['name']!r}, out={names[default_out]['name']!r}", + ) + except Exception as e: + record("audio devices", False, str(e)) + + +def check_microphone(): + import numpy as np + import sounddevice as sd + + try: + print(" ... recording 3 seconds, please say something") + rec = sd.rec(3 * 16000, samplerate=16000, channels=1, dtype="int16") + sd.wait() + peak = int(np.abs(rec).max()) + except Exception as e: + record("microphone", False, str(e)) + return None + + if peak == 0: + record( + "microphone", + False, + "captured pure silence — grant your terminal microphone access in " + "System Settings > Privacy & Security > Microphone", + ) + return None + record("microphone", True, f"peak amplitude {peak}") + return rec + + +def check_hold_key(): + """Hold-to-talk needs Input Monitoring; without it we fall back to a toggle.""" + from global_hotkey import permission_granted + + if permission_granted(): + record("hold-to-talk permission", True, "Input Monitoring granted") + else: + record( + "hold-to-talk (optional)", + False, + "Input Monitoring not granted — add your terminal under System Settings > " + "Privacy & Security > Input Monitoring and restart it. SPACE toggle still works.", + ) + + +def check_macos_voices(): + from apple_tts import available_voices, find_voice + + voices = available_voices() + irish = [name for name, language in voices if language == "en-IE"] + if not irish: + record( + "macos voices (optional)", + False, + "no Irish (en-IE) voice installed; add one under System Settings > " + "Accessibility > Spoken Content > System Voice", + ) + return + record( + "macos voices", + bool(find_voice("Moira")), + f"{len(voices)} installed, Irish: {', '.join(irish)}", + ) + + +def check_kokoro(): + from kokoro_onnx import Kokoro + + try: + kokoro = Kokoro(str(KOKORO_CACHE / "kokoro-v1.0.onnx"), str(KOKORO_CACHE / "voices-v1.0.bin")) + samples, rate = kokoro.create(PHRASE, voice="af_heart", speed=1.0, lang="en-us") + except Exception as e: + record("kokoro tts", False, str(e)) + return None + record("kokoro tts", True, f"{len(samples) / rate:.2f}s of audio at {rate} Hz") + return samples, rate + + +def write_wav(synthesized, path="/tmp/voice-agent-probe.wav"): + import numpy as np + + samples, rate = synthesized + pcm = (np.clip(samples, -1, 1) * 32767).astype(np.int16) + with wave.open(path, "wb") as f: + f.setnchannels(1) + f.setsampwidth(2) + f.setframerate(rate) + f.writeframes(pcm.tobytes()) + return path + + +def matches(heard: str) -> bool: + return heard.lower().strip(" .") == PHRASE.lower().strip(" .") + + +def check_apple_stt(synthesized): + """Transcribe Kokoro's own output — a full loop through the audio stack.""" + from apple_stt import _recognize_file, probe + + available, reason = probe() + if not available: + record("apple speech to text", False, reason) + return + if synthesized is None: + record("apple speech to text", False, "skipped, Kokoro produced no audio to transcribe") + return + + started = time.time() + try: + heard = _recognize_file(write_wav(synthesized), "en-US", 20.0).strip() + except Exception as e: + record("apple speech to text", False, str(e)) + return + record("apple speech to text", matches(heard), f"heard {heard!r} in {time.time() - started:.2f}s") + + +def check_mlx_whisper(synthesized): + """Optional — only matters if you want --stt-engine mlx.""" + if synthesized is None: + record("mlx whisper (optional)", False, "skipped, no audio to transcribe") + return + + import mlx_whisper + import numpy as np + import soxr + + samples, rate = synthesized + audio = soxr.resample(samples.astype(np.float32), rate, 16000) + try: + heard = mlx_whisper.transcribe( + audio, path_or_hf_repo="mlx-community/whisper-large-v3-turbo-q4", language="en" + )["text"].strip() + except Exception as e: + detail = str(e).splitlines()[0] + record("mlx whisper (optional)", False, f"{detail} — use --stt-engine apple or cpu") + return + record("mlx whisper (optional)", matches(heard), f"heard {heard!r}") + + +async def check_claude(): + from claude_agent_sdk import ClaudeSDKClient, ResultMessage, StreamEvent + + from bot import build_claude_options + from claude_llm import _text_delta + + import argparse + + from bot import DEFAULT_CLAUDE_MODEL + + options = build_claude_options( + argparse.Namespace( + allow_writes=False, + cwd=None, + claude_model=DEFAULT_CLAUDE_MODEL, + load_settings=False, + ) + ) + try: + async with ClaudeSDKClient(options=options) as client: + await client.query("Reply with exactly one word: ready") + spoken = [] + async for message in client.receive_response(): + if isinstance(message, StreamEvent): + text = _text_delta(message) + if text: + spoken.append(text) + elif isinstance(message, ResultMessage) and message.is_error: + record("claude session", False, f"error: {message.result}") + return + except Exception as e: + hint = "" + if "-9" in str(e): + hint = " — the CLI was killed applying its own sandbox; run this from a normal terminal" + record("claude session", False, f"{e}{hint}") + return + record( + "claude session", + True, + f"replied {''.join(spoken).strip()!r} using {DEFAULT_CLAUDE_MODEL}", + ) + + +async def main(): + print("Checking the voice pipeline...\n") + check_devices() + check_microphone() + check_hold_key() + check_macos_voices() + synthesized = check_kokoro() + check_apple_stt(synthesized) + check_mlx_whisper(synthesized) + await check_claude() + + required_failures = [ + name for name, ok, _ in results if not ok and not name.endswith("(optional)") + ] + print() + if required_failures: + print(f"{len(required_failures)} check(s) failed: {', '.join(required_failures)}") + return 1 + print("Everything works. Run ./talk to start a conversation.") + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/sounddevice_transport.py b/sounddevice_transport.py new file mode 100644 index 0000000..0f77e76 --- /dev/null +++ b/sounddevice_transport.py @@ -0,0 +1,168 @@ +"""Pipecat local-audio transport backed by sounddevice instead of PyAudio. + +Pipecat ships `pipecat.transports.local.audio.LocalAudioTransport`, but it needs +PyAudio, which has no macOS wheel and must be compiled against a Homebrew +portaudio. This machine blocks both, so we talk to the same portaudio through +sounddevice, whose wheel bundles a prebuilt dylib. The frame contract is +identical to the upstream transport. +""" + +import asyncio +from concurrent.futures import ThreadPoolExecutor + +import sounddevice as sd +from loguru import logger + +from pipecat.frames.frames import InputAudioRawFrame, OutputAudioRawFrame, StartFrame +from pipecat.processors.frame_processor import FrameProcessor +from pipecat.transports.base_input import BaseInputTransport +from pipecat.transports.base_output import BaseOutputTransport +from pipecat.transports.base_transport import BaseTransport, TransportParams + + +class SoundDeviceTransportParams(TransportParams): + """Configuration for the sounddevice transport. + + Parameters: + input_device: sounddevice device index or name substring. None uses the default. + output_device: sounddevice device index or name substring. None uses the default. + """ + + input_device: int | str | None = None + output_device: int | str | None = None + + +class SoundDeviceInputTransport(BaseInputTransport): + """Captures microphone audio and pushes it into the pipeline.""" + + _params: SoundDeviceTransportParams + + def __init__(self, params: SoundDeviceTransportParams): + super().__init__(params) + self._in_stream: sd.RawInputStream | None = None + self._sample_rate = 0 + + async def start(self, frame: StartFrame): + await super().start(frame) + + if self._in_stream: + return + + self._sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate + blocksize = int(self._sample_rate / 100) * 2 # 20ms + + self._in_stream = sd.RawInputStream( + samplerate=self._sample_rate, + blocksize=blocksize, + device=self._params.input_device, + channels=self._params.audio_in_channels, + dtype="int16", + callback=self._audio_in_callback, + ) + self._in_stream.start() + + device_name = sd.query_devices(self._in_stream.device, "input")["name"] + logger.info(f"Microphone: {device_name} @ {self._sample_rate} Hz") + + await self.set_transport_ready(frame) + + async def cleanup(self): + await super().cleanup() + if self._in_stream: + self._in_stream.stop() + self._in_stream.close() + self._in_stream = None + + def _audio_in_callback(self, indata, frame_count, time_info, status): + if status: + logger.trace(f"Audio input status: {status}") + + frame = InputAudioRawFrame( + audio=bytes(indata), + sample_rate=self._sample_rate, + num_channels=self._params.audio_in_channels, + ) + + asyncio.run_coroutine_threadsafe(self.push_audio_frame(frame), self.get_event_loop()) + + +class SoundDeviceOutputTransport(BaseOutputTransport): + """Plays pipeline audio out through the speakers.""" + + _params: SoundDeviceTransportParams + + def __init__(self, params: SoundDeviceTransportParams): + super().__init__(params) + self._out_stream: sd.RawOutputStream | None = None + self._sample_rate = 0 + # Writes are serialized by the pipeline, so one worker is enough. + self._executor = ThreadPoolExecutor(max_workers=1) + + async def start(self, frame: StartFrame): + await super().start(frame) + + if self._out_stream: + return + + self._sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate + + self._out_stream = sd.RawOutputStream( + samplerate=self._sample_rate, + device=self._params.output_device, + channels=self._params.audio_out_channels, + dtype="int16", + ) + self._out_stream.start() + + device_name = sd.query_devices(self._out_stream.device, "output")["name"] + logger.info(f"Speaker: {device_name} @ {self._sample_rate} Hz") + + await self.set_transport_ready(frame) + + async def cleanup(self): + await super().cleanup() + if self._out_stream: + self._out_stream.stop() + self._out_stream.close() + self._out_stream = None + + async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: + if not self._out_stream: + return False + await self.get_event_loop().run_in_executor( + self._executor, self._out_stream.write, frame.audio + ) + return True + + +class SoundDeviceTransport(BaseTransport): + """Local microphone + speaker transport.""" + + def __init__(self, params: SoundDeviceTransportParams): + super().__init__() + self._params = params + self._input: SoundDeviceInputTransport | None = None + self._output: SoundDeviceOutputTransport | None = None + + def input(self) -> FrameProcessor: + if not self._input: + self._input = SoundDeviceInputTransport(self._params) + return self._input + + def output(self) -> FrameProcessor: + if not self._output: + self._output = SoundDeviceOutputTransport(self._params) + return self._output + + +def list_devices() -> str: + """Render the audio devices sounddevice can see, for `--list-devices`.""" + lines = [] + for index, device in enumerate(sd.query_devices()): + capability = [] + if device["max_input_channels"]: + capability.append(f"in:{device['max_input_channels']}") + if device["max_output_channels"]: + capability.append(f"out:{device['max_output_channels']}") + lines.append(f" [{index}] {device['name']} ({', '.join(capability)})") + return "\n".join(lines) diff --git a/speech_analyzer_stt.py b/speech_analyzer_stt.py new file mode 100644 index 0000000..cc7b5f8 --- /dev/null +++ b/speech_analyzer_stt.py @@ -0,0 +1,200 @@ +"""Speech-to-text through macOS 26's SpeechAnalyzer, via a Swift helper. + +`SpeechTranscriber` is the accurate recogniser on this OS, and it is Swift-only +— built on actors and AsyncSequence, with no Objective-C surface — so pyobjc +cannot reach it. `swift/speech-helper` is a small binary that does, and this +drives it as a subprocess: one process per utterance, JSON on stdout. + +Two modules sit behind the analyzer and the choice between them is not obvious, +so it was measured on eight sentences of technical speech: + +| module | WER | +|-----------------------------------|-------| +| SpeechTranscriber | 18.8% | +| SpeechTranscriber + vocabulary | 18.8% | +| DictationTranscriber | 25.9% | +| DictationTranscriber + vocabulary | 17.6% | + +`SpeechTranscriber` has the better acoustic model but **ignores** +`AnalysisContext.contextualStrings` — output is byte-identical with and without +terms. `DictationTranscriber` is weaker bare yet consumes them, and biasing +matters more than the model on jargon, so it is the default here. Add repair +rules on top and it reaches 12.9%, the best measured configuration. + +Both avoid the session limit that forces the transcript stitching in +`apple_stt.py`: 40 of 40 sentences recovered from 83 seconds of audio, with no +reassembly. + +The subprocess costs a few tens of milliseconds per utterance. Against a +recogniser that runs far faster than real time, that is a fair trade. +""" + +import asyncio +import json +import os +import subprocess +import tempfile +from collections.abc import AsyncGenerator +from pathlib import Path + +from loguru import logger + +from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame +from pipecat.services.settings import STTSettings +from pipecat.services.stt_service import SegmentedSTTService +from pipecat.transcriptions.language import Language +from pipecat.utils.time import time_now_iso8601 + +HELPER = Path(__file__).parent / "swift" / "speech-helper" + +# Generous: the helper may be downloading the on-device model on first use. +_FIRST_RUN_TIMEOUT = 300.0 +_TIMEOUT_BASE = 15.0 +_TIMEOUT_PER_AUDIO_SECOND = 0.5 + +# A locally built binary that has not been through this Mac's approval process +# is SIGKILLed on launch rather than failing in any legible way. +_KILLED = -9 + + +def _needs_approval(returncode: int) -> bool: + return returncode in (_KILLED, 137) + + +async def _run_helper(args: list[str], timeout: float) -> tuple[int, str, str]: + process = await asyncio.create_subprocess_exec( + str(HELPER), + *args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout) + except asyncio.TimeoutError: + process.kill() + raise + return process.returncode, stdout.decode(errors="replace"), stderr.decode(errors="replace") + + +def probe(locale: str = "en-US") -> tuple[bool, str]: + """Check whether the helper exists, is allowed to run, and has its model. + + Blocking on purpose: this runs while the pipeline is being assembled, which + may already be inside an event loop, so it cannot start one of its own. + """ + if not HELPER.exists(): + return False, f"helper not built — run {HELPER.parent}/build.sh" + + try: + result = subprocess.run( + [str(HELPER), "--check", "--locale", locale], + capture_output=True, + text=True, + timeout=_FIRST_RUN_TIMEOUT, + ) + except subprocess.TimeoutExpired: + return False, "helper timed out during its availability check" + except OSError as e: + return False, str(e) + returncode, stdout, stderr = result.returncode, result.stdout, result.stderr + + if _needs_approval(returncode): + return False, ( + "the helper was killed on launch, which on this managed Mac means the " + f"binary is still awaiting approval. Request approval for {HELPER}, " + "then try again." + ) + if returncode != 0: + return False, (stderr.strip() or stdout.strip() or f"helper exited {returncode}") + + try: + payload = json.loads(stdout) + except json.JSONDecodeError: + return False, f"unreadable helper output: {stdout[:120]!r}" + if not payload.get("available"): + return False, f"no SpeechTranscriber model for {locale}" + return True, f"SpeechAnalyzer ready, assets {payload.get('assets', 'unknown')}" + + +class SpeechAnalyzerSTTService(SegmentedSTTService): + """Transcribe VAD-delimited segments with SpeechTranscriber.""" + + def __init__( + self, + *, + locale: str = "en-US", + language: Language = Language.EN_US, + vocabulary=None, + want_alternatives: bool = False, + module: str = "dictation", + **kwargs, + ): + super().__init__(settings=STTSettings(model=None, language=locale), **kwargs) + self._locale = locale + self._language = language + self._vocabulary = vocabulary + self._want_alternatives = want_alternatives + # "dictation" consumes the vocabulary; "transcriber" has the stronger + # acoustic model but ignores it. See the module docstring. + self._module = module + + def can_generate_metrics(self) -> bool: + return True + + async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Transcribe one speech segment. + + Args: + audio: The segment as a WAV container, per ``wants_wav_segments``. + """ + await self.start_processing_metrics() + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f: + f.write(audio) + path = f.name + + audio_seconds = max(0.0, (len(audio) - 44) / 2 / self.sample_rate) + args = [path, "--locale", self._locale] + if self._vocabulary and (terms := self._vocabulary.terms()): + args += ["--terms", ",".join(terms)] + if self._want_alternatives: + args.append("--alternatives") + if self._module == "dictation": + args.append("--dictation") + + try: + returncode, stdout, stderr = await _run_helper( + args, _TIMEOUT_BASE + _TIMEOUT_PER_AUDIO_SECOND * audio_seconds + ) + except asyncio.TimeoutError: + await self.stop_processing_metrics() + yield ErrorFrame(error="SpeechAnalyzer helper timed out") + return + except Exception as e: + await self.stop_processing_metrics() + yield ErrorFrame(error=f"SpeechAnalyzer helper failed: {e}") + return + finally: + os.unlink(path) + + await self.stop_processing_metrics() + + if returncode != 0: + detail = stderr.strip() or stdout.strip() or f"exited {returncode}" + yield ErrorFrame(error=f"SpeechAnalyzer helper failed: {detail}") + return + + try: + payload = json.loads(stdout) + except json.JSONDecodeError: + yield ErrorFrame(error=f"unreadable helper output: {stdout[:120]!r}") + return + if "error" in payload: + yield ErrorFrame(error=f"SpeechAnalyzer: {payload['error']}") + return + + text = (payload.get("text") or "").strip() + if not text: + return + + logger.debug(f"Transcription: [{text}]") + yield TranscriptionFrame(text, self._user_id, time_now_iso8601(), self._language) diff --git a/spoken_text.py b/spoken_text.py new file mode 100644 index 0000000..25b805e --- /dev/null +++ b/spoken_text.py @@ -0,0 +1,43 @@ +"""Strip formatting that only makes sense on a screen, before it is spoken. + +`VOICE_STYLE` asks Claude not to emit markdown, and mostly he doesn't — but +asking is probabilistic and hearing "asterisk asterisk aloud asterisk asterisk" +once is enough. This is the deterministic half. + +Pipecat's `MarkdownTextFilter` does most of it and runs *after* sentence +aggregation, so formatting split across streaming chunks is already reassembled +by the time it sees the text. What it leaves behind, measured: + + 'Use ~~strike~~' -> 'Use ~~strike~~' still spoken as tildes + '- first bullet' -> '- first bullet' spoken as "dash" + 'the sample_rate is' -> 'the sample_rate is' spoken as "underscore" + 'Multiply 3 * 4' -> 'Multiply 3 4' the "times" is lost + +so this subclass handles those four and tidies the spacing. +""" + +import re + +from pipecat.utils.text.markdown_text_filter import MarkdownTextFilter + +# Run before the markdown filter, which would otherwise eat the asterisk. +_TIMES = re.compile(r"(?<=\d)\s*\*\s*(?=\d)") + +_STRIKETHROUGH = re.compile(r"~~(.+?)~~") +# Only at the start of a line, so a hyphenated word is untouched. +_LIST_MARKER = re.compile(r"^[ \t]*[-*•]\s+", re.MULTILINE) +# Identifiers read better as words: "sample_rate" -> "sample rate". +_UNDERSCORE_WORD = re.compile(r"(?<=\w)_(?=\w)") +_EXTRA_SPACE = re.compile(r"[ \t]{2,}") + + +class SpokenTextFilter(MarkdownTextFilter): + """Markdown filtering, plus the leftovers that matter when read aloud.""" + + async def filter(self, text: str) -> str: + text = _TIMES.sub(" times ", text) + text = await super().filter(text) + text = _STRIKETHROUGH.sub(r"\1", text) + text = _LIST_MARKER.sub("", text) + text = _UNDERSCORE_WORD.sub(" ", text) + return _EXTRA_SPACE.sub(" ", text) diff --git a/swift/SpeechHelper.swift b/swift/SpeechHelper.swift new file mode 100644 index 0000000..3cd9ef8 --- /dev/null +++ b/swift/SpeechHelper.swift @@ -0,0 +1,170 @@ +// Transcribe a WAV file with macOS 26's SpeechAnalyzer. +// +// The accurate recogniser on this OS — SpeechTranscriber, driven by +// SpeechAnalyzer — is a Swift-only API built on actors and AsyncSequence, so +// pyobjc cannot reach it. This is the smallest thing that can: read a file, +// print JSON, exit. bot.py runs it as a subprocess. +// +// Why bother, versus the SFSpeechRecognizer path in apple_stt.py: +// - Roughly four times more accurate on published benchmarks. +// - No session length limit, which is what forces the transcript-stitching +// workaround in apple_stt.py. +// - Still supports vocabulary biasing, via AnalysisContext on the analyzer +// rather than on the request. +// - Returns ranked alternatives, which measurably help LLM post-correction. +// +// Usage: +// speech-helper [--locale en-US] [--terms a,b,c] [--alternatives] +// speech-helper --check [--locale en-US] + +import AVFoundation +import Foundation +import Speech + +struct Options { + var path: String? + var locale = "en-US" + var terms: [String] = [] + var alternatives = false + var check = false + var dictation = false +} + +func parseArguments() -> Options { + var options = Options() + var arguments = Array(CommandLine.arguments.dropFirst()) + while let argument = arguments.first { + arguments.removeFirst() + switch argument { + case "--locale": + options.locale = arguments.isEmpty ? options.locale : arguments.removeFirst() + case "--terms": + let raw = arguments.isEmpty ? "" : arguments.removeFirst() + options.terms = raw.split(separator: ",").map { + $0.trimmingCharacters(in: .whitespaces) + }.filter { !$0.isEmpty } + case "--alternatives": + options.alternatives = true + case "--dictation": + options.dictation = true + case "--check": + options.check = true + default: + if options.path == nil { options.path = argument } + } + } + return options +} + +func emit(_ payload: [String: Any]) { + let data = try! JSONSerialization.data(withJSONObject: payload, options: [.sortedKeys]) + FileHandle.standardOutput.write(data) + FileHandle.standardOutput.write("\n".data(using: .utf8)!) +} + +func fail(_ message: String) -> Never { + emit(["error": message]) + exit(1) +} + +/// Download the on-device model if this locale hasn't been used before. +/// +/// The first run for a locale has to fetch assets; later runs return +/// immediately. Without this the analyzer fails rather than installing them. +func ensureModel(for module: any SpeechModule) async throws { + if let request = try await AssetInventory.assetInstallationRequest(supporting: [module]) { + try await request.downloadAndInstall() + } +} + +func transcribe(_ options: Options) async throws { + guard let path = options.path else { fail("no audio file given") } + let url = URL(fileURLWithPath: path) + guard FileManager.default.fileExists(atPath: path) else { fail("no such file: \(path)") } + + let locale = Locale(identifier: options.locale) + + // SpeechTranscriber has the better acoustic model but ignores + // AnalysisContext.contextualStrings; DictationTranscriber is the older + // model and does consume them. Which trade wins depends on the speech. + let module: any SpeechModule = options.dictation + ? DictationTranscriber(locale: locale, preset: .shortDictation) + : SpeechTranscriber( + locale: locale, + preset: options.alternatives ? .transcriptionWithAlternatives : .transcription + ) + try await ensureModel(for: module) + + // Vocabulary biasing lives on the analyzer's context here, not on the + // recognition request as it did in the old API — so the accurate model and + // term biasing can be used together. + let context = AnalysisContext() + if !options.terms.isEmpty { + context.contextualStrings = [.general: options.terms] + } + + let file = try AVAudioFile(forReading: url) + let analyzer = try await SpeechAnalyzer( + inputAudioFile: file, + modules: [module], + analysisContext: context, + finishAfterFile: true + ) + + // Results arrive per utterance and are concatenated; unlike the old API + // there is no restart to stitch around. + var pieces: [String] = [] + var alternatives: [String] = [] + if let dictationModule = module as? DictationTranscriber { + for try await result in dictationModule.results { + let text = String(result.text.characters) + if !text.isEmpty { pieces.append(text) } + } + } else if let speechModule = module as? SpeechTranscriber { + for try await result in speechModule.results { + let text = String(result.text.characters) + if !text.isEmpty { pieces.append(text) } + if options.alternatives { + alternatives.append(contentsOf: result.alternatives.map { String($0.characters) }) + } + } + } + try await analyzer.finalizeAndFinishThroughEndOfInput() + + var payload: [String: Any] = [ + "text": pieces.joined(separator: " ").trimmingCharacters(in: .whitespaces) + ] + if options.alternatives { payload["alternatives"] = alternatives } + emit(payload) +} + +/// Report whether this machine can run the model, without transcribing. +func check(_ options: Options) async throws { + let locale = Locale(identifier: options.locale) + let supported = await SpeechTranscriber.supportedLocales.contains { + $0.identifier(.bcp47) == locale.identifier(.bcp47) + } + let transcriber = SpeechTranscriber(locale: locale, preset: .transcription) + let status = await AssetInventory.status(forModules: [transcriber]) + emit([ + "available": supported, + "locale": options.locale, + "assets": String(describing: status), + ]) +} + +@main +struct SpeechHelper { + static func main() async { + let options = parseArguments() + do { + if options.check { + try await check(options) + } else { + try await transcribe(options) + } + } catch { + fail(String(describing: error)) + } + } +} diff --git a/swift/build.sh b/swift/build.sh new file mode 100755 index 0000000..e4ca582 --- /dev/null +++ b/swift/build.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Build the SpeechAnalyzer helper. +# +# On a managed Mac the freshly built binary is SIGKILLed (exit 137) until it has +# been through the binary approval process, and the approval takes a few minutes +# to sync. Rebuilding changes the binary, so it needs approving again. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$HERE" + +swiftc -O -parse-as-library SpeechHelper.swift -o speech-helper +echo "built $HERE/speech-helper" + +if ./speech-helper --check >/dev/null 2>&1; then + echo "it runs — ./speech-helper --check for details" +else + status=$? + if [ "$status" -eq 137 ]; then + echo "built, but killed on launch (137): request binary approval for" + echo " $HERE/speech-helper" + echo "then wait a few minutes for it to sync." + else + echo "built, but exited $status — run ./speech-helper --check to see why" + fi +fi diff --git a/swift/speech-helper b/swift/speech-helper new file mode 100755 index 0000000..cf2f502 Binary files /dev/null and b/swift/speech-helper differ diff --git a/talk b/talk new file mode 100755 index 0000000..bd54857 --- /dev/null +++ b/talk @@ -0,0 +1,14 @@ +#!/bin/bash +# Launch the local voice conversation. +# +# Python doesn't trust the corporate TLS proxy out of the box, so point its +# HTTP clients at the system CA bundle before Hugging Face and Kokoro try to +# download their models. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +export SSL_CERT_FILE=/etc/ssl/cert.pem +export REQUESTS_CA_BUNDLE=/etc/ssl/cert.pem + +exec "$HERE/.venv/bin/python" "$HERE/bot.py" "$@" diff --git a/test_idle.py b/test_idle.py new file mode 100644 index 0000000..5aa51db --- /dev/null +++ b/test_idle.py @@ -0,0 +1,51 @@ +"""A long tool call must not get the pipeline cancelled. + +Regression test: pipecat's idle timer only resets on speech frames, so a slow +turn looked identical to an abandoned session and cancelled the worker and the +runner mid-answer. +""" +import asyncio, sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent)) +from pipecat.frames.frames import EndFrame, Frame, TTSSpeakFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.worker import PipelineParams, PipelineWorker +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.workers.runner import WorkerRunner + +class Probe(FrameProcessor): + async def process_frame(self, f, d): + await super().process_frame(f, d) + await self.push_frame(f, d) + +async def run(idle_timeout_secs, label, *, expect_cancelled, quiet_for=4.0): + worker = PipelineWorker( + Pipeline([Probe()]), + params=PipelineParams(), + idle_timeout_secs=idle_timeout_secs, + ) + cancelled = {"yes": False} + @worker.event_handler("on_idle_timeout") + async def _(w): + cancelled["yes"] = True + + runner = WorkerRunner(handle_sigint=False) + await runner.add_workers(worker) + task = asyncio.create_task(runner.run()) + await asyncio.sleep(quiet_for) # stand in for a slow tool call + alive = not task.done() + await worker.queue_frames([EndFrame()]) + try: + await asyncio.wait_for(task, timeout=10) + except Exception: + pass + got_cancelled = cancelled["yes"] or not alive + verdict = "PASS" if got_cancelled == expect_cancelled else "FAIL" + print(f" {verdict} {label}: cancelled={got_cancelled} (expected {expect_cancelled})") + +async def main(): + # The first shows the bug is real; the second shows the fix holds. + await run(2.0, "a short idle timeout still cancels", expect_cancelled=True) + await run(None, "disabled: a slow turn survives", expect_cancelled=False) + +asyncio.run(main()) diff --git a/test_journal.py b/test_journal.py new file mode 100644 index 0000000..4e1fae6 --- /dev/null +++ b/test_journal.py @@ -0,0 +1,51 @@ +"""The journal must capture both halves of a turn. + +Regression test for a placement bug: the journal originally sat at the end of +the pipeline, where the user aggregator has already consumed the transcript, so +it never recorded anything at all. +""" +import asyncio, json, sys, tempfile +from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent)) +from journal import Journal +from pipecat.frames.frames import EndFrame, TranscriptionFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.worker import PipelineWorker +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import ( + LLMContextAggregatorPair, LLMUserAggregatorParams) +from pipecat.turns.user_start.external_user_turn_start_strategy import ExternalUserTurnStartStrategy +from pipecat.turns.user_stop.external_user_turn_stop_strategy import ExternalUserTurnStopStrategy +from pipecat.turns.user_turn_strategies import UserTurnStrategies +from pipecat.utils.time import time_now_iso8601 +from pipecat.workers.runner import WorkerRunner + +async def main(): + path = Path(tempfile.mkdtemp()) / "journal.jsonl" + journal = Journal(path) + ctx = LLMContext() + ua, _ = LLMContextAggregatorPair(ctx, user_params=LLMUserAggregatorParams( + user_turn_strategies=UserTurnStrategies( + start=[ExternalUserTurnStartStrategy()], stop=[ExternalUserTurnStopStrategy()]))) + worker = PipelineWorker(Pipeline([journal, ua])) + runner = WorkerRunner(handle_sigint=False) + await runner.add_workers(worker) + task = asyncio.create_task(runner.run()) + await asyncio.sleep(0.3) + + await worker.queue_frames([ + TranscriptionFrame("what is the sample rate", "u", time_now_iso8601(), None)]) + await asyncio.sleep(0.5) + journal.record_reply("Sixteen kilohertz.") + await asyncio.sleep(0.3) + await worker.queue_frames([EndFrame()]) + await task + + rows = [json.loads(l) for l in path.read_text().splitlines() if l.strip()] + print(f" {'PASS' if rows else 'FAIL'} journal file written ({len(rows)} entries)") + if rows: + r = rows[0] + print(f" {'PASS' if r['heard'] else 'FAIL'} captured what was heard: {r['heard']!r}") + print(f" {'PASS' if r['reply'] else 'FAIL'} captured the reply: {r['reply']!r}") + +asyncio.run(main()) diff --git a/test_observer.py b/test_observer.py new file mode 100644 index 0000000..e911214 --- /dev/null +++ b/test_observer.py @@ -0,0 +1,48 @@ +"""The turn must close even if the reply observer explodes, and the observer +must be the one we passed rather than pipecat's.""" +import asyncio, argparse, sys +sys.path.insert(0, "/Users/adolforeyna/voice-agent") +from pathlib import Path +from bot import build_claude_options +from claude_llm import ClaudeCodeLLM +from pipecat.frames.frames import EndFrame, Frame, LLMContextFrame, LLMFullResponseEndFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.worker import PipelineWorker +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.workers.runner import WorkerRunner + +class Sink(FrameProcessor): + def __init__(self): + super().__init__(); self.ended = asyncio.Event() + async def process_frame(self, f, d): + await super().process_frame(f, d) + if isinstance(f, LLMFullResponseEndFrame): self.ended.set() + await self.push_frame(f, d) + +async def run(observer, label): + args = argparse.Namespace(allow_writes=False, cwd=str(Path.home()/"Workspace"), + claude_model="claude-sonnet-4-6", load_settings=False) + llm = ClaudeCodeLLM(options=build_claude_options(args, None, None), observer=observer) + sink = Sink() + worker = PipelineWorker(Pipeline([llm, sink])) + runner = WorkerRunner(handle_sigint=False) + await runner.add_workers(worker) + task = asyncio.create_task(runner.run()) + await asyncio.sleep(0.4) + ctx = LLMContext(); ctx.add_message({"role":"user","content":"Say hi in three words."}) + await worker.queue_frames([LLMContextFrame(context=ctx)]) + try: + await asyncio.wait_for(sink.ended.wait(), timeout=90) + print(f" PASS {label}: turn closed") + except asyncio.TimeoutError: + print(f" FAIL {label}: LLMFullResponseEndFrame never arrived") + await worker.queue_frames([EndFrame()]); await task + +async def main(): + seen = [] + await run(seen.append, "normal observer") + print(f" {'PASS' if seen else 'FAIL'} observer actually called ({len(seen)} reply)") + def boom(_): raise RuntimeError("observer blew up") + await run(boom, "observer raises") +asyncio.run(main()) diff --git a/test_spoken_text.py b/test_spoken_text.py new file mode 100644 index 0000000..3114b1e --- /dev/null +++ b/test_spoken_text.py @@ -0,0 +1,35 @@ +"""Formatting must never reach the speakers. + +Regression test for hearing "asterisk asterisk aloud asterisk asterisk". +""" +import asyncio, sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent)) +from spoken_text import SpokenTextFilter + +CASES = [ + ("It is annoying to hear **aloud** asterisks.", "It is annoying to hear aloud asterisks."), + ("Use *italics* and `code` and ~~strike~~.", "Use italics and code and strike."), + ("- first\n- second", "first\nsecond"), + ("Check `bot.py`, the **sample_rate** is 16000.", "Check bot.py, the sample rate is 16000."), + ("See [the docs](https://example.com) for more.", "See the docs for more."), + ("Multiply 3 * 4.", "Multiply 3 times 4."), + ("A plain sentence.", "A plain sentence."), + ("The well-known trade-off is fine.", "The well-known trade-off is fine."), +] + +async def main(): + f = SpokenTextFilter() + bad = 0 + for text, want in CASES: + got = await f.filter(text) + if got.strip() != want.strip(): + bad += 1 + print(f" FAIL {text!r}\n got {got!r}\n want {want!r}") + print(f" {'PASS' if not bad else 'FAIL'} {len(CASES) - bad}/{len(CASES)} spoken-text cases") + # Nothing that reads as punctuation noise should survive. + joined = "".join([await f.filter(t) for t, _ in CASES]) + for ch in "*`~#": + print(f" {'PASS' if ch not in joined else 'FAIL'} no {ch!r} reaches the speakers") + +asyncio.run(main()) diff --git a/test_working_phrase.py b/test_working_phrase.py new file mode 100644 index 0000000..8687ce2 --- /dev/null +++ b/test_working_phrase.py @@ -0,0 +1,48 @@ +"""Silence during a slow tool call must be broken. + +From a recorded session: four of nine turns went unanswered because tool-heavy +turns ran for over a minute with no sound, and speaking again to check whether +it was alive cancelled the turn in flight. +""" +import asyncio, sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent)) +from claude_llm import ClaudeCodeLLM +from pipecat.frames.frames import TTSSpeakFrame + +class Recorder(ClaudeCodeLLM): + def __init__(self, **kw): + super().__init__(options=None, **kw) + self.pushed = [] + async def push_frame(self, frame, direction=None): + self.pushed.append(frame) + +async def main(): + # Tool runs before anything has been said: the user hears nothing otherwise. + llm = Recorder() + llm._said_working = False + await llm._say_working(spoken=[]) + said = [f.text for f in llm.pushed if isinstance(f, TTSSpeakFrame)] + print(f" {'PASS' if said else 'FAIL'} speaks up when a tool runs first: {said}") + + # Only once per turn, however many tools run. + await llm._say_working(spoken=[]) + await llm._say_working(spoken=[]) + said = [f.text for f in llm.pushed if isinstance(f, TTSSpeakFrame)] + print(f" {'PASS' if len(said) == 1 else 'FAIL'} says it only once per turn ({len(said)})") + + # Already answering: adding a filler would talk over the real reply. + llm2 = Recorder() + llm2._said_working = False + await llm2._say_working(spoken=["Running that now."]) + quiet = not [f for f in llm2.pushed if isinstance(f, TTSSpeakFrame)] + print(f" {'PASS' if quiet else 'FAIL'} stays quiet when it already narrated") + + # Opted out. + llm3 = Recorder(working_phrase=None) + llm3._said_working = False + await llm3._say_working(spoken=[]) + quiet = not [f for f in llm3.pushed if isinstance(f, TTSSpeakFrame)] + print(f" {'PASS' if quiet else 'FAIL'} respects working_phrase=None") + +asyncio.run(main()) diff --git a/transcript_repair.py b/transcript_repair.py new file mode 100644 index 0000000..4d30041 --- /dev/null +++ b/transcript_repair.py @@ -0,0 +1,31 @@ +"""Rewrite transcripts before anything downstream reads them. + +Sits between the recogniser and the turn aggregator so the substitutions apply +whichever speech engine is in use, and so the corrected text is what reaches +Claude, the logs, and any future meeting-notes writer alike. +""" + +from loguru import logger + +from pipecat.frames.frames import Frame, TranscriptionFrame +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from vocabulary import Vocabulary + + +class TranscriptRepair(FrameProcessor): + """Apply the vocabulary's repair rules to every transcription.""" + + def __init__(self, vocabulary: Vocabulary, **kwargs): + super().__init__(**kwargs) + self._vocabulary = vocabulary + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, TranscriptionFrame): + repaired = self._vocabulary.repair(frame.text) + if repaired != frame.text: + logger.debug(f"Repaired transcript: {frame.text!r} -> {repaired!r}") + frame.text = repaired + + await self.push_frame(frame, direction) diff --git a/vocabulary.example.txt b/vocabulary.example.txt new file mode 100644 index 0000000..2ec53ad --- /dev/null +++ b/vocabulary.example.txt @@ -0,0 +1,48 @@ +# Words the speech recognizer should expect. +# +# One term per line; everything after a # is ignored. These are added to the +# terms discovered automatically from the project (filenames, class and function +# names, git branches and authors) and from what Claude has been saying. +# +# Keep them to one or two words each — Apple's guidance is a phrase you could +# say without pausing — and keep the list SHORT. Relevance beats coverage: +# 18 apt terms measured better than 1000 diluted ones. The total is capped at +# 100, with the terms in this file ranked first. +# +# Add the names, jargon and product names you actually say out loud. + +Metamate +Phabricator +fbsource +Scuba +Hack +Buck +Thrift +GraphQL +Adolfo Reyna +Marketplace + +# Kit for this project +pipecat +Kokoro +Moira +sounddevice +Silero +Whisper +MLX +Metal +pyobjc +Quartz +SFSpeechRecognizer +SpeechAnalyzer + +# Things I say about it out loud +push to talk +barge in +echo tail +sample rate +transport +self test +endpoint +hold key +voice activity diff --git a/vocabulary.py b/vocabulary.py new file mode 100644 index 0000000..066f064 --- /dev/null +++ b/vocabulary.py @@ -0,0 +1,272 @@ +"""Tell the recognizer which words to expect, and fix the ones it still gets wrong. + +Two mechanisms, deliberately separate, because they fail differently: + +- **Biasing.** `SFSpeechRecognitionRequest.contextualStrings` nudges the decoder + towards terms it would otherwise never produce. Measured at 23.5% -> 16.5% + word error rate on technical speech. It is a hint, so it fails softly. +- **Repair.** Deterministic substitutions applied after recognition, for + mistakes that recur identically ("the coral voice" is always Kokoro). Exact + and testable, but blind to context, so the rules must be specific. + +Relevance beats coverage, and by more than expected. Eighteen apt terms measured +16.5% WER; padding them out to 100 with names harvested from the project put it +back to 23.5%, i.e. no better than no biasing at all. Padding with a thousand +*random dictionary* words cost only 1.1 points, so the damage is not volume — +it is that filenames and identifiers ("bot", "plan", "hack") are ordinary words +the recognizer would happily produce anyway, and biasing towards them drags +real speech onto them. + +So auto-discovered terms are filtered to ones that are *not* ordinary English, +capped tightly, and always ranked behind the hand-written list. The curated file +is what carries the benefit; discovery is a small bonus with a real downside. +""" + +import re +import subprocess +from collections import OrderedDict +from pathlib import Path + +from loguru import logger + +# Apple documents a ceiling of 100, but measurement says stay well under it: +# quality decays fast once ordinary words get in. +MAX_TERMS = 40 + +# Discovered terms are the risky kind, so they get a small share of the budget. +MAX_DISCOVERED = 12 + +_SYSTEM_DICTIONARY = Path("/usr/share/dict/words") + +# Words that are never worth biasing towards and only crowd out real terms. +_STOPWORDS = { + "and", "are", "but", "for", "from", "has", "have", "into", "not", "our", + "out", "the", "that", "this", "was", "were", "will", "with", "you", + "your", "main", "init", "self", "test", "tests", "src", "lib", "util", + "utils", "readme", "true", "false", "none", "null", "class", "def", +} + +_IDENTIFIER = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]*$") +_CAMEL_BOUNDARY = re.compile(r"(?<=[a-z0-9])(?=[A-Z])") +# Things worth learning from Claude's replies: CamelCase, dotted paths, +# snake_case, and words carrying digits. Ordinary prose is skipped. +_TECHNICAL = re.compile(r"\b(?:[A-Za-z]+[._][A-Za-z0-9._-]+|[a-z]+[A-Z][A-Za-z]*|[A-Za-z]*\d[A-Za-z0-9]*)\b") + +_SKIP_DIRS = {".git", ".venv", "__pycache__", "node_modules", ".cache", "dist", "build"} + + +def _speakable(term: str) -> str: + """Turn an identifier into something a person could say. + + Apple's guidance is one or two words per phrase, spoken without a pause, so + `sounddevice_transport` is useless as-is but `sounddevice transport` is not. + """ + term = term.replace("_", " ").replace("-", " ").replace(".", " ") + term = _CAMEL_BOUNDARY.sub(" ", term) + return " ".join(term.split()) + + +class Vocabulary: + """Ranked terms for the recognizer, plus repair rules for its known mistakes. + + Terms are ordered by how likely they are to matter: words the user wrote + down, then the project around them, then whatever Claude has been talking + about lately. Only the first ``limit`` survive. + """ + + def __init__( + self, + *, + project_dir: Path | None = None, + vocabulary_file: Path | None = None, + corrections_file: Path | None = None, + limit: int = MAX_TERMS, + ): + self._limit = limit + self._user: list[str] = [] + self._project: list[str] = [] + self._observed: OrderedDict[str, None] = OrderedDict() + self._corrections: list[tuple[re.Pattern, str]] = [] + + if vocabulary_file and vocabulary_file.exists(): + self._user = _read_terms(vocabulary_file) + logger.debug(f"Vocabulary: {len(self._user)} terms from {vocabulary_file.name}") + if corrections_file and corrections_file.exists(): + self._corrections = _read_corrections(corrections_file) + logger.debug(f"Vocabulary: {len(self._corrections)} repair rules") + if project_dir: + self._project = _terms_from_project(project_dir) + + def add_terms(self, terms: list[str]): + """Add terms that rank alongside the hand-written ones. + + For names the user certainly says — project names out of the brain — + which are as authoritative as anything typed into vocabulary.txt. + """ + for term in terms: + speakable = _speakable(term) + if _worth_keeping(speakable) and speakable not in self._user: + self._user.append(speakable) + + def observe(self, text: str): + """Learn technical words from something Claude just said. + + What the assistant is discussing is a good predictor of what the user is + about to say back, so its replies feed the next recognition. + """ + for match in _TECHNICAL.findall(text or ""): + term = _speakable(match) + if _worth_keeping(term) and _is_distinctive(term): + self._observed.pop(term, None) # move to most-recent + self._observed[term] = None + # Keep the tail bounded; only the newest can reach the term list anyway. + while len(self._observed) > self._limit * 2: + self._observed.popitem(last=False) + + def terms(self) -> list[str]: + """The ranked, de-duplicated, capped term list. + + What Claude said a moment ago predicts the next utterance better than an + arbitrary filename does, so recent observations get a reserved share of + the budget rather than queueing behind the whole project. + """ + budget = max(0, self._limit - len(self._user)) + discovered = min(budget, MAX_DISCOVERED) + recent = list(reversed(self._observed))[: (discovered + 1) // 2] + project = self._project[: discovered - len(recent)] + ranked = [*self._user, *recent, *project] + + seen: dict[str, str] = {} + for term in ranked: + key = term.lower() + if key not in seen: + seen[key] = term + if len(seen) >= self._limit: + break + return list(seen.values()) + + def prompt_block(self, limit: int = 60) -> str: + """The stable terms, for Claude's system prompt. + + Only the written-down and project-derived terms go here: the prompt is + fixed for the session, so terms learned mid-conversation could not + appear anyway. + """ + stable = list(dict.fromkeys([*self._user, *self._project]))[:limit] + return ", ".join(stable) + + def repair(self, text: str) -> str: + """Apply the substitution rules to a transcript.""" + for pattern, replacement in self._corrections: + text = pattern.sub(replacement, text) + return text + + +def _worth_keeping(term: str) -> bool: + return ( + len(term) >= 3 + and term.lower() not in _STOPWORDS + and not term.isdigit() + and len(term) <= 40 + ) + + +def _load_dictionary() -> frozenset[str]: + try: + return frozenset(w.strip().lower() for w in _SYSTEM_DICTIONARY.read_text().splitlines()) + except OSError: + return frozenset() + + +_ORDINARY_WORDS = _load_dictionary() + + +def _is_distinctive(term: str) -> bool: + """Whether biasing towards this term could actually change an outcome. + + Only words the recognizer would not already produce are worth boosting. + Biasing towards ordinary English measurably drags correct speech onto the + wrong word, which is how a harvested term list erased the entire benefit. + """ + words = term.lower().split() + if len(words) > 1: + return True # multi-word phrases are specific enough to be safe + return words[0] not in _ORDINARY_WORDS + + +def _read_terms(path: Path) -> list[str]: + terms = [] + for line in path.read_text().splitlines(): + line = line.split("#", 1)[0].strip() + if line and _worth_keeping(line): + terms.append(line) + return terms + + +def _read_corrections(path: Path) -> list[tuple[re.Pattern, str]]: + rules = [] + for lineno, raw in enumerate(path.read_text().splitlines(), 1): + line = raw.split("#", 1)[0].strip() + if not line: + continue + if "=>" not in line: + logger.warning(f"{path.name}:{lineno}: expected 'heard => replacement', got {raw!r}") + continue + heard, replacement = (part.strip() for part in line.split("=>", 1)) + if not heard: + continue + # Word-bounded and case-insensitive so "coral voice" matches mid-sentence + # but "chorale" never does. + rules.append((re.compile(rf"\b{re.escape(heard)}\b", re.IGNORECASE), replacement)) + return rules + + +def _terms_from_project(project_dir: Path) -> list[str]: + """Names from the code the user is most likely to talk about.""" + terms: list[str] = [] + + for path in sorted(project_dir.rglob("*")): + if any(part in _SKIP_DIRS for part in path.parts): + continue + if path.is_file(): + term = _speakable(path.stem) + if _worth_keeping(term) and _is_distinctive(term): + terms.append(term) + + terms.extend(_git_terms(project_dir)) + + for path in sorted(project_dir.glob("*.py")): + try: + source = path.read_text(errors="ignore") + except OSError: + continue + for name in re.findall(r"^\s*(?:class|def)\s+([A-Za-z_][A-Za-z0-9_]*)", source, re.M): + term = _speakable(name) + if _worth_keeping(term) and _is_distinctive(term): + terms.append(term) + + return list(dict.fromkeys(terms)) + + +def _git_terms(project_dir: Path) -> list[str]: + """Branch and author names, which are spoken far more often than they're typed.""" + terms: list[str] = [] + commands = ( + ["git", "branch", "--format=%(refname:short)"], + ["git", "log", "-40", "--format=%an"], + ) + for command in commands: + try: + result = subprocess.run( + command, cwd=project_dir, capture_output=True, text=True, timeout=5 + ) + except (OSError, subprocess.SubprocessError): + continue + if result.returncode != 0: + continue + for line in result.stdout.splitlines(): + for piece in line.split(): + term = _speakable(piece) + if _worth_keeping(term) and _is_distinctive(term): + terms.append(term) + return terms