adolforeyna/VoiceAgent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit of current state

Browse Source
This commit is contained in:
Adolfo Reyna
2026-08-07 18:15:36 -04:00
commit 62d805e565
32 changed files with 4416 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bot.py
+583
View File
@@ -0,0 +1,583 @@
#!/usr/bin/env python3
"""Local voice conversation with Claude Code.
Everything but Claude itself runs on this machine: Apple's on-device dictation
model for speech-to-text, Kokoro for text-to-speech, Silero for voice activity
detection. Claude runs through the Claude Agent SDK, which drives the same
`claude` CLI — and the same auth — as a terminal session.
"""
import argparse
import asyncio
import re
import shutil
import sys
from pathlib import Path
from claude_agent_sdk import ClaudeAgentOptions, SandboxSettings
from loguru import logger
from brain import Brain
from claude_llm import ClaudeCodeLLM
from echo_guard import EchoGuardUserMuteStrategy
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import TTSSpeakFrame
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.worker import PipelineParams, PipelineWorker
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.aggregators.llm_response_universal import (
LLMContextAggregatorPair,
LLMUserAggregatorParams,
)
from pipecat.processors.audio.vad_processor import VADProcessor
from pipecat.services.kokoro.tts import KokoroTTSService
from pipecat.services.whisper.stt import MLXModel, WhisperSTTService, WhisperSTTServiceMLX
from pipecat.transcriptions.language import Language
from pipecat.turns.user_start.external_user_turn_start_strategy import (
ExternalUserTurnStartStrategy,
)
from pipecat.turns.user_stop.external_user_turn_stop_strategy import (
ExternalUserTurnStopStrategy,
)
from pipecat.turns.user_stop.speech_timeout_user_turn_stop_strategy import (
SpeechTimeoutUserTurnStopStrategy,
)
from pipecat.turns.user_turn_strategies import UserTurnStrategies
from pipecat.workers.runner import WorkerRunner
from global_hotkey import HOLD_KEYS
from push_to_talk import PushToTalk
from journal import Journal
from memory_tools import build_server
from transcript_repair import TranscriptRepair
from vocabulary import Vocabulary
from spoken_text import SpokenTextFilter
from sounddevice_transport import SoundDeviceTransport, SoundDeviceTransportParams, list_devices
# Every recognizer here expects 16 kHz; Kokoro synthesizes at 24 kHz. Matching
# both natively avoids a resample on the hot path in each direction.
STT_SAMPLE_RATE = 16000
TTS_SAMPLE_RATE = 24000
VOICE_STYLE = """
You are talking to the user out loud, over a microphone and speakers. Your
replies are spoken by a text-to-speech voice, so write them the way you would
say them:
- No markdown, bullet points, code blocks, emoji, or URLs. None of it survives
being read aloud.
- Keep answers to a few sentences unless asked to go deeper. This is a
conversation, not a document.
- Spell out things that only make sense visually. Say "line forty-two of
bot dot py" rather than pasting a path.
- If you need to run tools, say what you're doing in a short phrase first, so
the silence is explained.
- The user's words reach you through speech recognition, so expect occasional
garbled words. Ask rather than guess when it matters.
""".strip()
# Enforcement is by denial, not by allow-list. Measured: with
# permission_mode="bypassPermissions", passing allowed_tools does NOT restrict
# anything — Claude ran Bash while it was absent from that list, with no denial
# recorded. Only disallowed_tools blocks.
#
# The shell is on by default because the point of this assistant is asking about
# work out loud, and most of that lives behind `meta` — experiments, memory,
# tasks, calendar. Without a shell it can only apologise.
#
# Be clear-eyed about what the default denies: a shell can write files perfectly
# well, so withholding Edit and Write is a speed bump against casual edits, not
# a security boundary. --read-only is the real boundary.
FILE_EDIT_TOOLS = ["Write", "Edit", "NotebookEdit"]
SHELL_TOOLS = ["Bash", "BashOutput", "KillShell"]
# Left unset, the CLI picks whatever the interactive session would use, which
# here resolves to a model this account can't reach headlessly ("Access to Fable
# is currently restricted"). Naming one explicitly avoids that, and for a spoken
# conversation time-to-first-token matters more than the extra capability.
DEFAULT_CLAUDE_MODEL = "claude-sonnet-4-6"
# Where Claude works, and where the personality file lives. Deliberately not
# this repo: the assistant is for everyday use, not for editing itself.
DEFAULT_WORKSPACE = Path.home() / "Workspace"
# The CLI reads CLAUDE.md from the working directory on its own but ignores
# AGENTS.md, so that one is loaded here and appended to the system prompt.
PERSONALITY_FILE = "AGENTS.md"
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--list-devices", action="store_true", help="Print audio devices and exit."
)
parser.add_argument("--input-device", help="Microphone index or name substring.")
parser.add_argument("--output-device", help="Speaker index or name substring.")
parser.add_argument(
"--stt-engine",
choices=["auto", "analyzer", "apple", "mlx", "cpu"],
default="auto",
help=(
"analyzer is macOS 26's SpeechAnalyzer and measured best; apple is the "
"older dictation API; mlx and cpu run Whisper. auto tries them in that "
"order, falling back on whatever is available."
),
)
parser.add_argument(
"--analyzer-module",
choices=["transcriber", "dictation"],
default="transcriber",
help=(
"Which SpeechAnalyzer module: transcriber is macOS 26's new model; "
"dictation is the older one, which honours the vocabulary."
),
)
parser.add_argument(
"--whisper-model",
help="Whisper model, for the mlx and cpu engines. Downloaded on first use.",
)
parser.add_argument(
"--tts",
choices=["kokoro", "apple"],
default="kokoro",
help="kokoro is a local neural voice; apple uses the macOS system voices.",
)
parser.add_argument(
"--voice",
help="Kokoro voice id (default af_heart) or macOS voice name (default Moira).",
)
parser.add_argument(
"--voice-rate", type=int, help="Words per minute, macOS voices only."
)
parser.add_argument(
"--list-voices", action="store_true", help="Print macOS system voices and exit."
)
parser.add_argument(
"--claude-model",
default=DEFAULT_CLAUDE_MODEL,
help="Claude model. Voice wants fast first tokens more than raw capability.",
)
parser.add_argument(
"--voice-activity",
action="store_true",
help=(
"End turns by detecting silence instead of a keypress. Hands-free, but "
"adds about 0.8s per turn and lets the mic hear the speakers."
),
)
parser.add_argument(
"--hold-key",
default="fn",
choices=[*HOLD_KEYS, "none"],
help=(
"Modifier to hold while speaking, recognised in any app. "
"'none' falls back to a spacebar toggle in this terminal."
),
)
parser.add_argument(
"--ptt-key",
default=" ",
help="Key that toggles the microphone when not using a hold key.",
)
parser.add_argument(
"--cwd",
default=str(DEFAULT_WORKSPACE),
help=(
"Where Claude's tools point, and where AGENTS.md is read from. "
"Created if it doesn't exist."
),
)
parser.add_argument(
"--allow-writes",
action="store_true",
help="Also allow Edit and Write, on top of the shell it already has.",
)
parser.add_argument(
"--read-only",
action="store_true",
help="Deny the shell too, so it can only read and search. Overrides --allow-writes.",
)
parser.add_argument(
"--barge-in",
action="store_true",
help=(
"With --voice-activity, let your voice interrupt Claude mid-sentence. "
"Headphones only: on speakers the mic hears Claude and he interrupts himself."
),
)
parser.add_argument(
"--echo-tail",
type=float,
default=0.4,
help=(
"With --voice-activity, seconds to keep ignoring the mic after Claude "
"stops speaking."
),
)
parser.add_argument(
"--load-settings",
action="store_true",
help=(
"Load your ~/.claude settings, plugins and skills. Off by default because "
"they add seconds to startup and a voice agent needs few of them."
),
)
parser.add_argument(
"--greeting",
default="I'm listening.",
help="Spoken on startup. Pass an empty string to start silent.",
)
parser.add_argument(
"--vocabulary-file",
help="Terms to bias the recognizer towards. Defaults to vocabulary.txt here.",
)
parser.add_argument(
"--no-brain",
action="store_true",
help="Skip the Metamate personal brain: no long-term context, no remembering.",
)
parser.add_argument(
"--no-vocabulary",
action="store_true",
help="Turn off vocabulary biasing and transcript repair.",
)
parser.add_argument("--log-level", default="INFO")
return parser.parse_args()
def as_device(value: str | None) -> int | str | None:
"""Accept either a device index or a name substring."""
if value is None:
return None
return int(value) if value.isdigit() else value
def build_stt(args: argparse.Namespace, vocabulary):
engine = args.stt_engine
# Preferred. Scored the way the published benchmark scores — spelling digits
# out, so an engine is not punished for writing "400" — SpeechTranscriber
# reaches 9.4% with five repair rules and no vocabulary at all, matching the
# older model's best while needing far less hand-tuning, and beating it raw
# (16.5% against 20.0%). It also needs no transcript stitching on long
# speech. Requires the Swift helper built and approved, so fall back quietly.
if engine in ("auto", "analyzer"):
from speech_analyzer_stt import SpeechAnalyzerSTTService, probe
available, reason = probe()
if available:
model = (
"SpeechTranscriber, the new macOS 26 model"
if args.analyzer_module == "transcriber"
else "DictationTranscriber, the older model"
)
logger.info(f"Speech to text: {model} — {reason}")
return SpeechAnalyzerSTTService(
vocabulary=vocabulary, module=args.analyzer_module
)
if engine == "analyzer":
raise SystemExit(f"SpeechAnalyzer is unavailable: {reason}")
logger.info(f"SpeechAnalyzer unavailable ({reason}); using the dictation model.")
engine = "apple"
if engine in ("auto", "apple"):
from apple_stt import AppleSpeechSTTService, probe
available, reason = probe()
if available:
logger.info(f"Speech to text: Apple dictation model ({reason})")
return AppleSpeechSTTService(vocabulary=vocabulary)
if engine == "apple":
raise SystemExit(f"Apple speech recognition is unavailable: {reason}")
logger.warning(f"Apple speech recognition unavailable ({reason}); falling back to Whisper.")
engine = "cpu"
if engine == "mlx":
model = args.whisper_model or MLXModel.LARGE_V3_TURBO_Q4.value
logger.info(f"Speech to text: MLX Whisper ({model})")
return WhisperSTTServiceMLX(
settings=WhisperSTTServiceMLX.Settings(model=model, language=Language.EN)
)
# base.en transcribed the benchmark phrase as accurately as distil-medium.en
# while taking 0.3s instead of 2.2s, which matters a lot in a conversation.
model = args.whisper_model or "base.en"
logger.info(f"Speech to text: faster-whisper on CPU ({model})")
return WhisperSTTService(settings=WhisperSTTService.Settings(model=model, language=Language.EN))
def build_tts(args: argparse.Namespace):
if args.tts == "apple":
from apple_tts import AppleTTSService, find_voice
voice = args.voice or "Moira"
found = find_voice(voice)
logger.info(f"Text to speech: macOS voice {voice} ({found[1] if found else '?'})")
return AppleTTSService(
voice=voice, rate_wpm=args.voice_rate, text_filters=[SpokenTextFilter()]
)
if args.voice_rate:
logger.warning("--voice-rate only applies to --tts apple; ignoring it.")
voice = args.voice or "af_heart"
logger.info(f"Text to speech: Kokoro {voice}")
return KokoroTTSService(
settings=KokoroTTSService.Settings(voice=voice, language=Language.EN),
text_filters=[SpokenTextFilter()],
)
def build_turn_taking(args: argparse.Namespace):
"""Decide what opens and closes a turn.
Returns the processor that sits right after the transport, the turn
strategies for the aggregator, and any mute strategies.
"""
if args.voice_activity:
logger.info("Turn taking: voice activity detection")
return (
VADProcessor(vad_analyzer=SileroVADAnalyzer()),
# The default stop strategy loads a separate smart-turn model; plain
# VAD silence detection is enough and keeps startup fast.
UserTurnStrategies(stop=[SpeechTimeoutUserTurnStopStrategy()]),
# Without a keypress gating the mic, the speakers feed straight back
# into it and Claude answers himself.
[] if args.barge_in else [EchoGuardUserMuteStrategy(tail_secs=args.echo_tail)],
)
# Push-to-talk drives both ends of the turn itself, so no VAD, no smart-turn
# model, and no echo guard — audio outside a keypress is never transcribed.
return (
PushToTalk(
hold_key=None if args.hold_key == "none" else args.hold_key,
toggle_key=args.ptt_key,
),
UserTurnStrategies(
start=[ExternalUserTurnStartStrategy()],
stop=[ExternalUserTurnStopStrategy()],
),
[],
)
def build_vocabulary(args: argparse.Namespace, brain=None) -> Vocabulary | None:
if args.no_vocabulary:
logger.info("Vocabulary biasing disabled.")
return None
# These live in the workspace, not here: they are learned state that grows
# with use, and keeping them beside AGENTS.md means git tracks how they
# change. This repo only carries the starting templates.
here = Path(__file__).parent
workspace = Path(args.cwd)
vocabulary_file = (
Path(args.vocabulary_file) if args.vocabulary_file else workspace / "vocabulary.txt"
)
corrections_file = workspace / "corrections.txt"
for target, template in (
(vocabulary_file, here / "vocabulary.example.txt"),
(corrections_file, here / "corrections.example.txt"),
):
if not target.exists() and template.exists():
target.write_text(template.read_text())
logger.info(f"Created {target} from the template")
vocabulary = Vocabulary(
project_dir=workspace,
vocabulary_file=vocabulary_file,
corrections_file=corrections_file,
)
if brain and brain.projects:
# "CIP-Unified-Cooldown", "pSMSL-RT-Cache" — the words most likely to be
# spoken and least likely to be recognised.
vocabulary.add_terms(brain.projects)
terms = vocabulary.terms()
logger.info(f"Vocabulary: biasing towards {len(terms)} terms, e.g. {', '.join(terms[:6])}")
return vocabulary
def _denied_tools(args: argparse.Namespace) -> list[str]:
if getattr(args, "read_only", False):
return [*SHELL_TOOLS, *FILE_EDIT_TOOLS]
if args.allow_writes:
return []
return FILE_EDIT_TOOLS
def read_personality(cwd: str | None) -> str:
"""Load AGENTS.md from the working directory, if it's there.
HTML comments are stripped, so the file can carry notes to whoever edits it
without those notes reaching Claude as instructions.
"""
if not cwd:
return ""
path = Path(cwd) / PERSONALITY_FILE
if not path.exists():
return ""
text = re.sub(r"<!--.*?-->", "", path.read_text(), flags=re.DOTALL).strip()
if text:
logger.info(f"Personality: {path} ({len(text)} chars)")
return text
def build_claude_options(args: argparse.Namespace, vocabulary=None, brain=None) -> ClaudeAgentOptions:
# The SDK bundles its own stock Claude Code binary and prefers it over the
# one on PATH. That build knows nothing about this org's gateway or its
# managed apiKeyHelper, so every turn fails with "Invalid API key" unless we
# point it back at the installed CLI.
cli_path = shutil.which("claude")
if cli_path:
logger.info(f"Claude CLI: {cli_path}")
else:
logger.warning("No `claude` on PATH; falling back to the SDK's bundled CLI.")
# Naming the domain vocabulary lets Claude resolve a garbled transcript
# while answering it. That recovers most of what a separate correction pass
# would, at no latency cost, because he already reads every transcript.
# Personality first, then the voice rules, so the constraints of speaking
# aloud get the last word over anything the personality file asks for.
style = ""
if personality := read_personality(args.cwd):
style += personality + "\n\n"
style += VOICE_STYLE
if brain and (memory := brain.prompt_block()):
# Only the context goes in the prompt. When to write back is described
# by the memory skill in the workspace, which Claude picks up on its own.
style += "\n\n" + memory
if vocabulary and (terms := vocabulary.prompt_block()):
style += (
"\n\nSpeech recognition mangles unusual words. When a transcript is "
"close to one of these, assume that is what was said and carry on "
"without remarking on it:\n" + terms
)
memory_server = build_server(workspace=Path(args.cwd), brain=brain)
return ClaudeAgentOptions(
cli_path=cli_path,
mcp_servers={"memory": memory_server},
system_prompt={"type": "preset", "preset": "claude_code", "append": style},
disallowed_tools=_denied_tools(args),
permission_mode="bypassPermissions",
cwd=args.cwd,
model=args.claude_model,
# "project" makes the workspace's own .claude/skills discoverable;
# without it a skills folder there is silently ignored.
setting_sources=(
["user", "project", "local"] if args.load_settings else ["project"]
),
skills="all",
include_partial_messages=True, # Speak as tokens arrive instead of per message.
# The CLI's own sandbox uses sandbox-exec, which fails when this process
# is already running inside one.
sandbox=SandboxSettings(enabled=False),
)
async def main() -> int:
args = parse_args()
if args.list_devices:
print(list_devices())
return 0
if args.list_voices:
from apple_tts import available_voices
for name, language in sorted(available_voices(), key=lambda v: (v[1], v[0])):
print(f" {language:<8} {name}")
return 0
logger.remove()
logger.add(sys.stderr, level=args.log_level)
workspace = Path(args.cwd)
if not workspace.exists():
workspace.mkdir(parents=True)
logger.info(f"Created {workspace}")
logger.info(f"Workspace: {workspace}")
transport = SoundDeviceTransport(
SoundDeviceTransportParams(
audio_in_enabled=True,
audio_out_enabled=True,
audio_in_sample_rate=STT_SAMPLE_RATE,
audio_out_sample_rate=TTS_SAMPLE_RATE,
input_device=as_device(args.input_device),
output_device=as_device(args.output_device),
)
)
brain = None
if not args.no_brain:
brain = Brain()
brain.load()
vocabulary = build_vocabulary(args, brain)
stt = build_stt(args, vocabulary)
journal = Journal(workspace / "journal.jsonl")
def on_reply(text: str):
journal.record_reply(text)
if vocabulary:
vocabulary.observe(text)
llm = ClaudeCodeLLM(
options=build_claude_options(args, vocabulary, brain),
observer=on_reply,
)
tts = build_tts(args)
# Claude keeps its own history; this context exists so Pipecat can decide
# when a turn has ended.
context = LLMContext()
turn_source, turn_strategies, mute_strategies = build_turn_taking(args)
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
context,
user_params=LLMUserAggregatorParams(
user_turn_strategies=turn_strategies,
user_mute_strategies=mute_strategies,
),
)
pipeline = Pipeline(
[
transport.input(),
turn_source,
stt,
*([TranscriptRepair(vocabulary)] if vocabulary else []),
journal,
user_aggregator,
llm,
tts,
transport.output(),
assistant_aggregator,
]
)
worker = PipelineWorker(
pipeline,
params=PipelineParams(enable_metrics=True, enable_usage_metrics=True),
# Off, because "idle" is meaningless here. The timer only resets on
# speech frames, so it cannot tell an abandoned session from a user
# who hasn't pressed the key for a while, or from Claude spending two
# minutes inside a subagent — and on firing it cancels the worker *and*
# the runner, killing the conversation mid-answer. Waiting quietly is
# this program's normal state.
idle_timeout_secs=None,
)
if args.greeting:
await worker.queue_frames([TTSSpeakFrame(args.greeting)])
runner = WorkerRunner()
await runner.add_workers(worker)
await runner.run()
return 0
if __name__ == "__main__":
sys.exit(asyncio.run(main()))