Adolfo Reyna
733 lines
28 KiB
Python
733 lines
28 KiB
Python
#!/usr/bin/env python3
|
|
"""Local voice conversation with Claude Code.
|
|
|
|
Everything but Claude itself runs on this machine: Apple's on-device dictation
|
|
model for speech-to-text, Kokoro for text-to-speech, Silero for voice activity
|
|
detection. Claude runs through the Claude Agent SDK, which drives the same
|
|
`claude` CLI — and the same auth — as a terminal session.
|
|
"""
|
|
|
|
import argparse
|
|
import asyncio
|
|
import re
|
|
import shutil
|
|
import sys
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
from claude_agent_sdk import ClaudeAgentOptions, SandboxSettings
|
|
from loguru import logger
|
|
|
|
import env_setup
|
|
import web_server
|
|
from brain import Brain
|
|
from voice_manager import VoiceManager
|
|
from model_manager import ModelManager
|
|
from claude_llm import ClaudeCodeLLM
|
|
from echo_guard import EchoGuardUserMuteStrategy
|
|
from pipecat.audio.vad.silero import SileroVADAnalyzer
|
|
from pipecat.frames.frames import (
|
|
TTSSpeakFrame,
|
|
TranscriptionFrame,
|
|
UserStartedSpeakingFrame,
|
|
UserStoppedSpeakingFrame,
|
|
)
|
|
from pipecat.pipeline.pipeline import Pipeline
|
|
from pipecat.pipeline.worker import PipelineParams, PipelineWorker
|
|
from pipecat.processors.aggregators.llm_context import LLMContext
|
|
from pipecat.processors.aggregators.llm_response_universal import (
|
|
LLMContextAggregatorPair,
|
|
LLMUserAggregatorParams,
|
|
)
|
|
from pipecat.processors.audio.vad_processor import VADProcessor
|
|
from pipecat.services.kokoro.tts import KokoroTTSService
|
|
from pipecat.services.whisper.stt import MLXModel, WhisperSTTService, WhisperSTTServiceMLX
|
|
from pipecat.transcriptions.language import Language
|
|
from pipecat.turns.user_start.external_user_turn_start_strategy import (
|
|
ExternalUserTurnStartStrategy,
|
|
)
|
|
from pipecat.turns.user_stop.external_user_turn_stop_strategy import (
|
|
ExternalUserTurnStopStrategy,
|
|
)
|
|
from pipecat.turns.user_stop.speech_timeout_user_turn_stop_strategy import (
|
|
SpeechTimeoutUserTurnStopStrategy,
|
|
)
|
|
from pipecat.turns.user_turn_strategies import UserTurnStrategies
|
|
from pipecat.workers.runner import WorkerRunner
|
|
from global_hotkey import HOLD_KEYS
|
|
from push_to_talk import PushToTalk
|
|
from journal import Journal, recent_prompt
|
|
from memory_tools import build_server
|
|
from transcript_repair import TranscriptRepair
|
|
from vocabulary import Vocabulary
|
|
from spoken_text import SpokenTextFilter
|
|
from sounddevice_transport import SoundDeviceTransport, SoundDeviceTransportParams, list_devices
|
|
|
|
# Every recognizer here expects 16 kHz; Kokoro synthesizes at 24 kHz. Matching
|
|
# both natively avoids a resample on the hot path in each direction.
|
|
STT_SAMPLE_RATE = 16000
|
|
TTS_SAMPLE_RATE = 24000
|
|
|
|
VOICE_STYLE = """
|
|
You are talking to the user out loud, over a microphone and speakers. Your
|
|
replies are spoken by a text-to-speech voice, so write them the way you would
|
|
say them:
|
|
|
|
- No markdown, bullet points, code blocks, emoji, or URLs. None of it survives
|
|
being read aloud.
|
|
- Keep answers to a few sentences unless asked to go deeper. This is a
|
|
conversation, not a document.
|
|
- Spell out things that only make sense visually. Say "line forty-two of
|
|
bot dot py" rather than pasting a path.
|
|
- Use your available tools (listing directories, searching, reading files, shell execution) whenever the user asks about files, commands, CLI tools (such as Paseo), or workspace tasks.
|
|
- You can dynamically change your spoken voice mid-response! Use markdown tags like `[Voice:af_bella]` or `[Voice:am_michael]` inline to switch voices (e.g. `[Voice:af_bella] Hello from Bella! [Voice:am_michael] And hello from Michael!`). The active voice will persist until you change it again.
|
|
- You can change your default voice! If the user asks to list available voices or switch voice permanently, run `python bin/voice_tool.py list` or `python bin/voice_tool.py set <voice_name>` (voices: af_heart, af_bella, am_michael, am_fenrir, am_puck, bf_emma, bm_george, Moira, Daniel).
|
|
- You can change your AI model on the fly! If the user asks to list available models or change model, run `python bin/model_tool.py list` or `python bin/model_tool.py set <model_name>` (models: luna, gemma, deepseek, gpt-oss, sonnet, etc.).
|
|
- You can reset or start a fresh conversation session! If the user asks to start a fresh session, reset the conversation, or clear session context, run `python bin/session_tool.py reset`.
|
|
- You can switch Hermes agent profiles! If the user asks to list Hermes profiles or switch profile, run `python bin/profile_tool.py list` or `python bin/profile_tool.py set <profile_name>`.
|
|
- You can change the running microphone and speakers independently. For requests such as “use AirPods”, “switch to Mac speakers”, or “use the Mac default mic”, run `python bin/audio_tool.py list` then `python bin/audio_tool.py set input|output <device-name-or-index|default>`. Report the command result plainly; do not claim a device changed if the tool says it is unavailable.
|
|
- You can open files visually for the user in the Companion Web UI drawer! Run `python bin/web_tool.py show <filepath>`.
|
|
- You can open links or the Companion Web UI in the default browser! Run `python bin/web_tool.py open <url>`.
|
|
- Keep implementation details and tool activity silent in the spoken channel. The user can see technical progress in the logs or Companion Web UI; only speak the useful conversational response.
|
|
- Complete multi-step tool calls fully before speaking your final response summary.
|
|
- Be strictly truthful about your findings and never invent fake file contents.
|
|
- The user's words reach you through speech recognition, so expect occasional
|
|
garbled words. Ask rather than guess when it matters.
|
|
""".strip()
|
|
|
|
# Enforcement is by denial, not by allow-list. Measured: with
|
|
# permission_mode="bypassPermissions", passing allowed_tools does NOT restrict
|
|
# anything — Claude ran Bash while it was absent from that list, with no denial
|
|
# recorded. Only disallowed_tools blocks.
|
|
#
|
|
# The shell is on by default because the point of this assistant is asking about
|
|
# work out loud, and most of that lives behind `meta` — experiments, memory,
|
|
# tasks, calendar. Without a shell it can only apologise.
|
|
#
|
|
# Be clear-eyed about what the default denies: a shell can write files perfectly
|
|
# well, so withholding Edit and Write is a speed bump against casual edits, not
|
|
# a security boundary. --read-only is the real boundary.
|
|
FILE_EDIT_TOOLS = ["Write", "Edit", "NotebookEdit"]
|
|
SHELL_TOOLS = ["Bash", "BashOutput", "KillShell"]
|
|
|
|
# Left unset, the CLI picks whatever the interactive session would use, which
|
|
# here resolves to a model this account can't reach headlessly ("Access to Fable
|
|
# is currently restricted"). Naming one explicitly avoids that, and for a spoken
|
|
# conversation time-to-first-token matters more than the extra capability.
|
|
DEFAULT_CLAUDE_MODEL = "claude-sonnet-4-6"
|
|
|
|
# Default directory is now the app folder itself
|
|
DEFAULT_APP_DIR = Path(__file__).parent.resolve()
|
|
|
|
# Personality file path
|
|
PERSONALITY_FILE = "AGENTS.md"
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--list-devices", action="store_true", help="Print audio devices and exit."
|
|
)
|
|
parser.add_argument("--input-device", help="Microphone index or name substring.")
|
|
parser.add_argument("--output-device", help="Speaker index or name substring.")
|
|
parser.add_argument(
|
|
"--stt-engine",
|
|
choices=["auto", "analyzer", "apple", "mlx", "cpu"],
|
|
default="auto",
|
|
help=(
|
|
"analyzer is macOS 26's SpeechAnalyzer and measured best; apple is the "
|
|
"older dictation API; mlx and cpu run Whisper. auto tries them in that "
|
|
"order, falling back on whatever is available."
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--analyzer-module",
|
|
choices=["transcriber", "dictation"],
|
|
default="transcriber",
|
|
help=(
|
|
"Which SpeechAnalyzer module: transcriber is macOS 26's new model; "
|
|
"dictation is the older one, which honours the vocabulary."
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--whisper-model",
|
|
help="Whisper model, for the mlx and cpu engines. Downloaded on first use.",
|
|
)
|
|
parser.add_argument(
|
|
"--tts",
|
|
choices=["kokoro", "apple"],
|
|
default="kokoro",
|
|
help="kokoro is a local neural voice; apple uses the macOS system voices.",
|
|
)
|
|
parser.add_argument(
|
|
"--voice",
|
|
help="Kokoro voice id (default af_heart) or macOS voice name (default Moira).",
|
|
)
|
|
parser.add_argument(
|
|
"--voice-rate", type=int, help="Words per minute, macOS voices only."
|
|
)
|
|
parser.add_argument(
|
|
"--list-voices", action="store_true", help="Print macOS system voices and exit."
|
|
)
|
|
parser.add_argument(
|
|
"--llm-engine",
|
|
choices=["hermes", "ollama", "apple", "macos", "claude"],
|
|
default="hermes",
|
|
help="LLM engine to use: hermes for Hermes models/CLI/Gateway, apple/macos for local MLX, claude for Claude Code.",
|
|
)
|
|
parser.add_argument(
|
|
"--hermes-model",
|
|
default="hermes-3",
|
|
help="Hermes model (default hermes-3).",
|
|
)
|
|
parser.add_argument(
|
|
"--ollama-host",
|
|
help="Ollama API host URL (defaults to OLLAMA_HOST or http://localhost:11434).",
|
|
)
|
|
parser.add_argument(
|
|
"--mlx-model",
|
|
default="mlx-community/Qwen2.5-7B-Instruct-4bit",
|
|
help="MLX model repo or path for local macOS execution.",
|
|
)
|
|
parser.add_argument(
|
|
"--claude-model",
|
|
default=DEFAULT_CLAUDE_MODEL,
|
|
help="Claude model. Voice wants fast first tokens more than raw capability.",
|
|
)
|
|
parser.add_argument(
|
|
"--voice-activity",
|
|
action="store_true",
|
|
help=(
|
|
"End turns by detecting silence instead of a keypress. Hands-free, but "
|
|
"adds about 0.8s per turn and lets the mic hear the speakers."
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--hold-key",
|
|
default="fn",
|
|
choices=[*HOLD_KEYS, "none"],
|
|
help=(
|
|
"Modifier to hold while speaking, recognised in any app. "
|
|
"'none' falls back to a spacebar toggle in this terminal."
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--ptt-key",
|
|
default=" ",
|
|
help="Key that toggles the microphone when not using a hold key.",
|
|
)
|
|
parser.add_argument(
|
|
"--cwd",
|
|
default=str(DEFAULT_APP_DIR),
|
|
help=(
|
|
"Working directory for file tools and AGENTS.md."
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--allow-writes",
|
|
action="store_true",
|
|
help="Also allow Edit and Write, on top of the shell it already has.",
|
|
)
|
|
parser.add_argument(
|
|
"--read-only",
|
|
action="store_true",
|
|
help="Deny the shell too, so it can only read and search. Overrides --allow-writes.",
|
|
)
|
|
parser.add_argument(
|
|
"--barge-in",
|
|
action="store_true",
|
|
help=(
|
|
"With --voice-activity, let your voice interrupt Claude mid-sentence. "
|
|
"Headphones only: on speakers the mic hears Claude and he interrupts himself."
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--echo-tail",
|
|
type=float,
|
|
default=0.4,
|
|
help=(
|
|
"With --voice-activity, seconds to keep ignoring the mic after Claude "
|
|
"stops speaking."
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--load-settings",
|
|
action="store_true",
|
|
help=(
|
|
"Load your ~/.claude settings, plugins and skills. Off by default because "
|
|
"they add seconds to startup and a voice agent needs few of them."
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--greeting",
|
|
default="I'm listening.",
|
|
help="Spoken on startup. Pass an empty string to start silent.",
|
|
)
|
|
parser.add_argument(
|
|
"--vocabulary-file",
|
|
help="Terms to bias the recognizer towards. Defaults to vocabulary.txt here.",
|
|
)
|
|
parser.add_argument(
|
|
"--no-brain",
|
|
action="store_true",
|
|
help="Skip the Metamate personal brain: no long-term context, no remembering.",
|
|
)
|
|
parser.add_argument(
|
|
"--no-vocabulary",
|
|
action="store_true",
|
|
help="Turn off vocabulary biasing and transcript repair.",
|
|
)
|
|
parser.add_argument(
|
|
"--web-port",
|
|
type=int,
|
|
default=8888,
|
|
help="Port for the Companion Web Chat UI server (default 8888).",
|
|
)
|
|
parser.add_argument(
|
|
"--no-web",
|
|
action="store_true",
|
|
help="Disable the Companion Web Chat UI server.",
|
|
)
|
|
parser.add_argument(
|
|
"--dual-engine",
|
|
action="store_true",
|
|
default=False,
|
|
help="Enable dual-engine mode: instant macOS foundation model (<400ms) + deep Hermes reasoning.",
|
|
)
|
|
parser.add_argument(
|
|
"--no-dual-engine",
|
|
action="store_false",
|
|
dest="dual_engine",
|
|
help="Disable dual-engine mode and run single engine directly.",
|
|
)
|
|
parser.add_argument("--log-level", default="INFO")
|
|
return parser.parse_args()
|
|
|
|
|
|
def as_device(value: str | None) -> int | str | None:
|
|
"""Accept either a device index or a name substring."""
|
|
if value is None:
|
|
return None
|
|
return int(value) if value.isdigit() else value
|
|
|
|
|
|
def build_stt(args: argparse.Namespace, vocabulary):
|
|
engine = args.stt_engine
|
|
|
|
# Preferred. Scored the way the published benchmark scores — spelling digits
|
|
# out, so an engine is not punished for writing "400" — SpeechTranscriber
|
|
# reaches 9.4% with five repair rules and no vocabulary at all, matching the
|
|
# older model's best while needing far less hand-tuning, and beating it raw
|
|
# (16.5% against 20.0%). It also needs no transcript stitching on long
|
|
# speech. Requires the Swift helper built and approved, so fall back quietly.
|
|
if engine in ("auto", "analyzer"):
|
|
from speech_analyzer_stt import SpeechAnalyzerSTTService, probe
|
|
|
|
available, reason = probe()
|
|
if available:
|
|
model = (
|
|
"SpeechTranscriber, the new macOS 26 model"
|
|
if args.analyzer_module == "transcriber"
|
|
else "DictationTranscriber, the older model"
|
|
)
|
|
logger.info(f"Speech to text: {model} — {reason}")
|
|
return SpeechAnalyzerSTTService(
|
|
vocabulary=vocabulary, module=args.analyzer_module
|
|
)
|
|
if engine == "analyzer":
|
|
raise SystemExit(f"SpeechAnalyzer is unavailable: {reason}")
|
|
logger.info(f"SpeechAnalyzer unavailable ({reason}); using the dictation model.")
|
|
engine = "apple"
|
|
|
|
if engine in ("auto", "apple"):
|
|
from apple_stt import AppleSpeechSTTService, probe
|
|
|
|
available, reason = probe()
|
|
if available:
|
|
logger.info(f"Speech to text: Apple dictation model ({reason})")
|
|
return AppleSpeechSTTService(vocabulary=vocabulary)
|
|
if engine == "apple":
|
|
raise SystemExit(f"Apple speech recognition is unavailable: {reason}")
|
|
logger.warning(f"Apple speech recognition unavailable ({reason}); falling back to Whisper.")
|
|
engine = "cpu"
|
|
|
|
if engine == "mlx":
|
|
model = args.whisper_model or MLXModel.LARGE_V3_TURBO_Q4.value
|
|
logger.info(f"Speech to text: MLX Whisper ({model})")
|
|
return WhisperSTTServiceMLX(
|
|
settings=WhisperSTTServiceMLX.Settings(model=model, language=Language.EN)
|
|
)
|
|
|
|
# base.en transcribed the benchmark phrase as accurately as distil-medium.en
|
|
# while taking 0.3s instead of 2.2s, which matters a lot in a conversation.
|
|
model = args.whisper_model or "base.en"
|
|
logger.info(f"Speech to text: faster-whisper on CPU ({model})")
|
|
return WhisperSTTService(settings=WhisperSTTService.Settings(model=model, language=Language.EN))
|
|
|
|
|
|
def build_tts(args: argparse.Namespace, voice_manager=None):
|
|
if args.tts == "apple":
|
|
from apple_tts import AppleTTSService, find_voice
|
|
|
|
voice = args.voice or "Moira"
|
|
found = find_voice(voice)
|
|
logger.info(f"Text to speech: macOS voice {voice} ({found[1] if found else '?'})")
|
|
return AppleTTSService(
|
|
voice=voice, rate_wpm=args.voice_rate, text_filters=[SpokenTextFilter()]
|
|
)
|
|
|
|
if args.voice_rate:
|
|
logger.warning("--voice-rate only applies to --tts apple; ignoring it.")
|
|
voice = (voice_manager.load_saved_voice() if voice_manager else None) or args.voice or "af_heart"
|
|
logger.info(f"Text to speech: Kokoro {voice}")
|
|
tts = KokoroTTSService(
|
|
settings=KokoroTTSService.Settings(voice=voice, language=Language.EN),
|
|
text_filters=[SpokenTextFilter(voice_manager=voice_manager)],
|
|
)
|
|
if voice_manager:
|
|
voice_manager.set_tts_processor(tts)
|
|
return tts
|
|
|
|
|
|
def build_turn_taking(args: argparse.Namespace):
|
|
"""Decide what opens and closes a turn.
|
|
|
|
Returns the processor that sits right after the transport, the turn
|
|
strategies for the aggregator, and any mute strategies.
|
|
"""
|
|
if args.voice_activity:
|
|
logger.info("Turn taking: voice activity detection")
|
|
return (
|
|
VADProcessor(vad_analyzer=SileroVADAnalyzer()),
|
|
# The default stop strategy loads a separate smart-turn model; plain
|
|
# VAD silence detection is enough and keeps startup fast.
|
|
UserTurnStrategies(stop=[SpeechTimeoutUserTurnStopStrategy()]),
|
|
# Without a keypress gating the mic, the speakers feed straight back
|
|
# into it and Claude answers himself.
|
|
[] if args.barge_in else [EchoGuardUserMuteStrategy(tail_secs=args.echo_tail)],
|
|
)
|
|
|
|
# Push-to-talk drives both ends of the turn itself, so no VAD, no smart-turn
|
|
# model, and no echo guard — audio outside a keypress is never transcribed.
|
|
return (
|
|
PushToTalk(
|
|
hold_key=None if args.hold_key == "none" else args.hold_key,
|
|
toggle_key=args.ptt_key,
|
|
),
|
|
UserTurnStrategies(
|
|
start=[ExternalUserTurnStartStrategy()],
|
|
stop=[ExternalUserTurnStopStrategy()],
|
|
),
|
|
[],
|
|
)
|
|
|
|
|
|
def build_vocabulary(args: argparse.Namespace, brain=None) -> Vocabulary | None:
|
|
if args.no_vocabulary:
|
|
logger.info("Vocabulary biasing disabled.")
|
|
return None
|
|
|
|
# Vocabulary and correction files live in the app folder
|
|
app_dir = Path(__file__).parent
|
|
vocabulary_file = (
|
|
Path(args.vocabulary_file) if args.vocabulary_file else app_dir / "vocabulary.txt"
|
|
)
|
|
corrections_file = app_dir / "corrections.txt"
|
|
|
|
for target, template in (
|
|
(vocabulary_file, app_dir / "vocabulary.example.txt"),
|
|
(corrections_file, app_dir / "corrections.example.txt"),
|
|
):
|
|
if not target.exists() and template.exists():
|
|
target.write_text(template.read_text())
|
|
logger.info(f"Created {target} from template")
|
|
|
|
vocabulary = Vocabulary(
|
|
project_dir=Path(args.cwd),
|
|
vocabulary_file=vocabulary_file,
|
|
corrections_file=corrections_file,
|
|
)
|
|
if brain and brain.projects:
|
|
vocabulary.add_terms(brain.projects)
|
|
terms = vocabulary.terms()
|
|
logger.info(f"Vocabulary: biasing towards {len(terms)} terms, e.g. {', '.join(terms[:6])}")
|
|
return vocabulary
|
|
|
|
|
|
def _denied_tools(args: argparse.Namespace) -> list[str]:
|
|
if getattr(args, "read_only", False):
|
|
return [*SHELL_TOOLS, *FILE_EDIT_TOOLS]
|
|
if args.allow_writes:
|
|
return []
|
|
return FILE_EDIT_TOOLS
|
|
|
|
|
|
def read_personality(cwd: str | None) -> str:
|
|
"""Load AGENTS.md from the working directory, if it's there.
|
|
|
|
HTML comments are stripped, so the file can carry notes to whoever edits it
|
|
without those notes reaching Claude as instructions.
|
|
"""
|
|
if not cwd:
|
|
return ""
|
|
path = Path(cwd) / PERSONALITY_FILE
|
|
if not path.exists():
|
|
return ""
|
|
text = re.sub(r"<!--.*?-->", "", path.read_text(), flags=re.DOTALL).strip()
|
|
if text:
|
|
logger.info(f"Personality: {path} ({len(text)} chars)")
|
|
return text
|
|
|
|
|
|
def build_claude_options(
|
|
args: argparse.Namespace, vocabulary=None, brain=None, journal_context: str = ""
|
|
) -> ClaudeAgentOptions:
|
|
cli_path = shutil.which("claude")
|
|
if cli_path:
|
|
logger.info(f"Claude CLI: {cli_path}")
|
|
else:
|
|
logger.warning("No `claude` on PATH; falling back to the SDK's bundled CLI.")
|
|
|
|
style = ""
|
|
if personality := read_personality(args.cwd):
|
|
style += personality + "\n\n"
|
|
style += VOICE_STYLE
|
|
if brain and (memory := brain.prompt_block()):
|
|
style += "\n\n" + memory
|
|
if journal_context:
|
|
style += "\n\n" + journal_context
|
|
if vocabulary and (terms := vocabulary.prompt_block()):
|
|
style += (
|
|
"\n\nSpeech recognition mangles unusual words. When a transcript is "
|
|
"close to one of these, assume that is what was said and carry on "
|
|
"without remarking on it:\n" + terms
|
|
)
|
|
|
|
memory_server = build_server(workspace=Path(args.cwd), brain=brain)
|
|
|
|
return ClaudeAgentOptions(
|
|
cli_path=cli_path,
|
|
mcp_servers={"memory": memory_server},
|
|
system_prompt={"type": "preset", "preset": "claude_code", "append": style},
|
|
disallowed_tools=_denied_tools(args),
|
|
permission_mode="bypassPermissions",
|
|
cwd=args.cwd,
|
|
model=args.claude_model,
|
|
setting_sources=(
|
|
["user", "project", "local"] if args.load_settings else ["project"]
|
|
),
|
|
skills="all",
|
|
include_partial_messages=True,
|
|
sandbox=SandboxSettings(enabled=False),
|
|
)
|
|
|
|
|
|
def build_llm(
|
|
args: argparse.Namespace,
|
|
vocabulary=None,
|
|
brain=None,
|
|
observer=None,
|
|
model_manager=None,
|
|
journal_context: str = "",
|
|
):
|
|
model = (model_manager.load_saved_model() if model_manager else None) or getattr(args, "hermes_model", "hermes-3")
|
|
if args.llm_engine in ("hermes", "ollama"):
|
|
from hermes_llm import HermesLLM, probe_hermes
|
|
from dual_engine import DualEngineProcessor
|
|
|
|
available, reason = probe_hermes(model)
|
|
logger.info(f"LLM: Hermes ({reason})")
|
|
|
|
deep_llm = HermesLLM(
|
|
model=model,
|
|
cwd=args.cwd,
|
|
session_name="Voice Agent",
|
|
observer=observer,
|
|
)
|
|
|
|
if getattr(args, "dual_engine", False):
|
|
from apple_llm import MacOSLLM, probe_apple_llm
|
|
fast_available, fast_reason = probe_apple_llm()
|
|
if fast_available:
|
|
logger.info(f"Dual-Engine: Pairing Hermes with fast macOS Foundation Model ({fast_reason})")
|
|
fast_llm = MacOSLLM(model=args.mlx_model, system_prompt="You are a fast voice assistant.")
|
|
return DualEngineProcessor(fast_llm=fast_llm, deep_llm=deep_llm, observer=observer)
|
|
|
|
return deep_llm
|
|
|
|
if args.llm_engine in ("apple", "macos"):
|
|
from apple_llm import MacOSLLM, probe_apple_llm
|
|
|
|
available, reason = probe_apple_llm()
|
|
logger.info(f"LLM: macOS native model ({reason})")
|
|
personality = read_personality(args.cwd) or "You are a helpful macOS voice assistant."
|
|
system_prompt = personality + "\n\n" + VOICE_STYLE
|
|
if brain and (memory := brain.prompt_block()):
|
|
system_prompt += "\n\n" + memory
|
|
if journal_context:
|
|
system_prompt += "\n\n" + journal_context
|
|
return MacOSLLM(model=args.mlx_model, system_prompt=system_prompt, observer=observer)
|
|
|
|
logger.info("LLM: Claude Code")
|
|
return ClaudeCodeLLM(
|
|
options=build_claude_options(args, vocabulary, brain, journal_context),
|
|
observer=observer,
|
|
)
|
|
|
|
|
|
async def main() -> int:
|
|
args = parse_args()
|
|
|
|
if args.list_devices:
|
|
print(list_devices())
|
|
return 0
|
|
|
|
if args.list_voices:
|
|
from apple_tts import available_voices
|
|
|
|
for name, language in sorted(available_voices(), key=lambda v: (v[1], v[0])):
|
|
print(f" {language:<8} {name}")
|
|
return 0
|
|
|
|
logger.remove()
|
|
logger.add(sys.stderr, level=args.log_level)
|
|
|
|
workspace = Path(args.cwd)
|
|
if not workspace.exists():
|
|
workspace.mkdir(parents=True)
|
|
logger.info(f"Created {workspace}")
|
|
logger.info(f"Workspace: {workspace}")
|
|
|
|
async def on_audio_device_event(snapshot) -> None:
|
|
"""Keep native route changes observable without touching conversation state."""
|
|
logger.info(
|
|
"Audio device event "
|
|
f"generation={snapshot.generation} reason={snapshot.reason.value} "
|
|
f"input={snapshot.default_input_uid!r} output={snapshot.default_output_uid!r}"
|
|
)
|
|
if not getattr(args, "no_web", False):
|
|
web_server.broadcast_event("audio_device", {
|
|
"generation": snapshot.generation,
|
|
"reason": snapshot.reason.value,
|
|
"input_uid": snapshot.default_input_uid,
|
|
"output_uid": snapshot.default_output_uid,
|
|
})
|
|
|
|
transport = SoundDeviceTransport(
|
|
SoundDeviceTransportParams(
|
|
audio_in_enabled=True,
|
|
audio_out_enabled=True,
|
|
audio_in_sample_rate=STT_SAMPLE_RATE,
|
|
audio_out_sample_rate=TTS_SAMPLE_RATE,
|
|
input_device=as_device(args.input_device),
|
|
output_device=as_device(args.output_device),
|
|
),
|
|
device_event_sink=on_audio_device_event,
|
|
)
|
|
|
|
brain = None
|
|
if not args.no_brain:
|
|
brain = Brain()
|
|
brain.load()
|
|
|
|
vocabulary = build_vocabulary(args, brain)
|
|
stt = build_stt(args, vocabulary)
|
|
journal = Journal(workspace / "journal.jsonl")
|
|
journal_context = recent_prompt(workspace / "journal.jsonl")
|
|
if journal_context:
|
|
logger.info("Journal: loaded the 10 most recent entries")
|
|
|
|
def on_reply(text: str):
|
|
journal.record_reply(text)
|
|
if vocabulary:
|
|
vocabulary.observe(text)
|
|
|
|
voice_manager = VoiceManager(workspace)
|
|
model_manager = ModelManager(workspace)
|
|
if not getattr(args, "no_web", False):
|
|
await web_server.start_server(workspace, port=getattr(args, "web_port", 8888))
|
|
web_server.set_managers(workspace, model_manager, voice_manager, audio_controller=transport)
|
|
|
|
llm = build_llm(
|
|
args,
|
|
vocabulary,
|
|
brain,
|
|
observer=on_reply,
|
|
model_manager=model_manager,
|
|
journal_context=journal_context,
|
|
)
|
|
model_manager.set_llm_processor(llm)
|
|
tts = build_tts(args, voice_manager=voice_manager)
|
|
|
|
# Claude keeps its own history; this context exists so Pipecat can decide
|
|
# when a turn has ended.
|
|
context = LLMContext()
|
|
turn_source, turn_strategies, mute_strategies = build_turn_taking(args)
|
|
|
|
user_aggregator, assistant_aggregator = LLMContextAggregatorPair(
|
|
context,
|
|
user_params=LLMUserAggregatorParams(
|
|
user_turn_strategies=turn_strategies,
|
|
user_mute_strategies=mute_strategies,
|
|
),
|
|
)
|
|
|
|
pipeline = Pipeline(
|
|
[
|
|
transport.input(),
|
|
turn_source,
|
|
stt,
|
|
*([TranscriptRepair(vocabulary)] if vocabulary else []),
|
|
journal,
|
|
user_aggregator,
|
|
llm,
|
|
tts,
|
|
transport.output(),
|
|
assistant_aggregator,
|
|
]
|
|
)
|
|
|
|
worker = PipelineWorker(
|
|
pipeline,
|
|
params=PipelineParams(enable_metrics=True, enable_usage_metrics=True),
|
|
# Off, because "idle" is meaningless here. The timer only resets on
|
|
# speech frames, so it cannot tell an abandoned session from a user
|
|
# who hasn't pressed the key for a while, or from Claude spending two
|
|
# minutes inside a subagent — and on firing it cancels the worker *and*
|
|
# the runner, killing the conversation mid-answer. Waiting quietly is
|
|
# this program's normal state.
|
|
idle_timeout_secs=None,
|
|
)
|
|
|
|
async def on_web_input(text: str):
|
|
logger.info(f"Web typed input: {text}")
|
|
web_server.broadcast_event("heard", {"text": text})
|
|
if hasattr(llm, "start_turn_direct"):
|
|
llm.start_turn_direct(text)
|
|
else:
|
|
frames = [
|
|
UserStartedSpeakingFrame(),
|
|
TranscriptionFrame(
|
|
text=text,
|
|
user_id="user",
|
|
timestamp=datetime.now().isoformat(timespec="seconds")
|
|
),
|
|
UserStoppedSpeakingFrame(),
|
|
]
|
|
await worker.queue_frames(frames)
|
|
|
|
if not getattr(args, "no_web", False):
|
|
web_server.set_managers(workspace, model_manager, voice_manager, input_callback=on_web_input, audio_controller=transport)
|
|
|
|
if args.greeting:
|
|
await worker.queue_frames([TTSSpeakFrame(args.greeting)])
|
|
|
|
runner = WorkerRunner()
|
|
await runner.add_workers(worker)
|
|
await runner.run()
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(asyncio.run(main()))
|