adolforeyna/VoiceAgent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace OpenCode harness with Hermes, remove workspace dependency, and move vocabulary/corrections to app folder

Browse Source
This commit is contained in:
Adolfo Reyna
2026-08-09 21:35:52 -04:00
parent 6e8a578e86
commit 2f181ff4f1
25 changed files with 2395 additions and 436 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bot.py
+110 -79
View File
@@ -12,17 +12,26 @@ import asyncio
import re
import shutil
import sys
from datetime import datetime
from pathlib import Path
from claude_agent_sdk import ClaudeAgentOptions, SandboxSettings
from loguru import logger
import env_setup
import web_server
from brain import Brain
from voice_manager import VoiceManager
from model_manager import ModelManager
from claude_llm import ClaudeCodeLLM
from echo_guard import EchoGuardUserMuteStrategy
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.frames.frames import TTSSpeakFrame
from pipecat.frames.frames import (
TTSSpeakFrame,
TranscriptionFrame,
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
)
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.worker import PipelineParams, PipelineWorker
from pipecat.processors.aggregators.llm_context import LLMContext
@@ -47,7 +56,7 @@ from pipecat.turns.user_turn_strategies import UserTurnStrategies
from pipecat.workers.runner import WorkerRunner
from global_hotkey import HOLD_KEYS
from push_to_talk import PushToTalk
from journal import Journal
from journal import Journal, recent_prompt
from memory_tools import build_server
from transcript_repair import TranscriptRepair
from vocabulary import Vocabulary
@@ -70,9 +79,13 @@ say them:
conversation, not a document.
- Spell out things that only make sense visually. Say "line forty-two of
bot dot py" rather than pasting a path.
- Use your available tools (listing directories, searching, reading files) whenever the user asks about files or workspace tasks.
- Use your available tools (listing directories, searching, reading files, shell execution) whenever the user asks about files, commands, CLI tools (such as Paseo), or workspace tasks.
- You can change your own voice! If the user asks to list available voices or switch voice, run `python bin/voice_tool.py list` or `python bin/voice_tool.py set <voice_name>` (voices: af_heart, af_bella, am_michael, am_fenrir, am_puck, bf_emma, bm_george, Moira, Daniel).
- Complete multi-step tool calls fully before speaking your response. Do not stop halfway to ask if you should continue.
- You can change your AI model on the fly! If the user asks to list available models or change model, run `python bin/model_tool.py list` or `python bin/model_tool.py set <model_name>` (models: luna, gemma, deepseek, gpt-oss, sonnet, etc.).
- You can open files visually for the user in the Companion Web UI drawer! Run `python bin/web_tool.py show <filepath>`.
- You can open links or the Companion Web UI in the default browser! Run `python bin/web_tool.py open <url>`.
- Speak your intent out loud BEFORE calling tools! Give brief guidance on what you are attempting (e.g. "Switching your voice to af_heart now...", "Checking Paseo CLI agents...", "Inspecting bot dot py..."). Speak a short guiding sentence first, then run your tools.
- Complete multi-step tool calls fully before speaking your final response summary.
- Be strictly truthful about your findings and never invent fake file contents.
- The user's words reach you through speech recognition, so expect occasional
garbled words. Ask rather than guess when it matters.
@@ -99,12 +112,10 @@ SHELL_TOOLS = ["Bash", "BashOutput", "KillShell"]
# conversation time-to-first-token matters more than the extra capability.
DEFAULT_CLAUDE_MODEL = "claude-sonnet-4-6"
# Where Claude works, and where the personality file lives. Deliberately not
# this repo: the assistant is for everyday use, not for editing itself.
DEFAULT_WORKSPACE = Path.home() / "Workspace"
# Default directory is now the app folder itself
DEFAULT_APP_DIR = Path(__file__).parent.resolve()
# The CLI reads CLAUDE.md from the working directory on its own but ignores
# AGENTS.md, so that one is loaded here and appended to the system prompt.
# Personality file path
PERSONALITY_FILE = "AGENTS.md"
@@ -156,14 +167,14 @@ def parse_args() -> argparse.Namespace:
)
parser.add_argument(
"--llm-engine",
choices=["opencode", "ollama", "apple", "macos", "claude"],
default="opencode",
help="LLM engine to use: opencode/ollama for OpenCode Cloud models (gemma4:31b), apple/macos for local MLX, claude for Claude Code.",
choices=["hermes", "ollama", "apple", "macos", "claude"],
default="hermes",
help="LLM engine to use: hermes for Hermes models/CLI/Gateway, apple/macos for local MLX, claude for Claude Code.",
)
parser.add_argument(
"--opencode-model",
default="ollama-cloud/gemma4:31b",
help="OpenCode Cloud model (default ollama-cloud/gemma4:31b).",
"--hermes-model",
default="hermes-3",
help="Hermes model (default hermes-3).",
)
parser.add_argument(
"--ollama-host",
@@ -203,10 +214,9 @@ def parse_args() -> argparse.Namespace:
)
parser.add_argument(
"--cwd",
default=str(DEFAULT_WORKSPACE),
default=str(DEFAULT_APP_DIR),
help=(
"Where Claude's tools point, and where AGENTS.md is read from. "
"Created if it doesn't exist."
"Working directory for file tools and AGENTS.md."
),
)
parser.add_argument(
@@ -263,6 +273,17 @@ def parse_args() -> argparse.Namespace:
action="store_true",
help="Turn off vocabulary biasing and transcript repair.",
)
parser.add_argument(
"--web-port",
type=int,
default=8888,
help="Port for the Companion Web Chat UI server (default 8888).",
)
parser.add_argument(
"--no-web",
action="store_true",
help="Disable the Companion Web Chat UI server.",
)
parser.add_argument("--log-level", default="INFO")
return parser.parse_args()
@@ -390,49 +411,27 @@ def build_vocabulary(args: argparse.Namespace, brain=None) -> Vocabulary | None:
logger.info("Vocabulary biasing disabled.")
return None
# These live in the workspace, not here: they are learned state that grows
# with use, and keeping them beside AGENTS.md means git tracks how they
# change. This repo only carries the starting templates.
here = Path(__file__).parent
workspace = Path(args.cwd)
# Vocabulary and correction files live in the app folder
app_dir = Path(__file__).parent
vocabulary_file = (
Path(args.vocabulary_file) if args.vocabulary_file else workspace / "vocabulary.txt"
Path(args.vocabulary_file) if args.vocabulary_file else app_dir / "vocabulary.txt"
)
corrections_file = workspace / "corrections.txt"
corrections_file = app_dir / "corrections.txt"
for target, template in (
(vocabulary_file, here / "vocabulary.example.txt"),
(corrections_file, here / "corrections.example.txt"),
(workspace / PERSONALITY_FILE, here / "AGENTS.md.example"),
(vocabulary_file, app_dir / "vocabulary.example.txt"),
(corrections_file, app_dir / "corrections.example.txt"),
):
if not target.exists() and template.exists():
target.write_text(template.read_text())
logger.info(f"Created {target} from the template")
# Ensure bin/voice_tool.py is available in the workspace for OpenCode and Claude
ws_bin = workspace / "bin"
ws_bin.mkdir(exist_ok=True)
voice_script = ws_bin / "voice_tool.py"
source_script = here / "bin" / "voice_tool.py"
if source_script.exists() and not voice_script.exists():
try:
voice_script.symlink_to(source_script.resolve())
logger.info(f"Created symlink {voice_script} -> {source_script}")
except Exception:
try:
voice_script.write_text(source_script.read_text())
logger.info(f"Copied {source_script} -> {voice_script}")
except Exception as e:
logger.warning(f"Could not install voice_tool.py into {ws_bin}: {e}")
logger.info(f"Created {target} from template")
vocabulary = Vocabulary(
project_dir=workspace,
project_dir=Path(args.cwd),
vocabulary_file=vocabulary_file,
corrections_file=corrections_file,
)
if brain and brain.projects:
# "CIP-Unified-Cooldown", "pSMSL-RT-Cache" — the words most likely to be
# spoken and least likely to be recognised.
vocabulary.add_terms(brain.projects)
terms = vocabulary.terms()
logger.info(f"Vocabulary: biasing towards {len(terms)} terms, e.g. {', '.join(terms[:6])}")
@@ -464,30 +463,23 @@ def read_personality(cwd: str | None) -> str:
return text
def build_claude_options(args: argparse.Namespace, vocabulary=None, brain=None) -> ClaudeAgentOptions:
# The SDK bundles its own stock Claude Code binary and prefers it over the
# one on PATH. That build knows nothing about this org's gateway or its
# managed apiKeyHelper, so every turn fails with "Invalid API key" unless we
# point it back at the installed CLI.
def build_claude_options(
args: argparse.Namespace, vocabulary=None, brain=None, journal_context: str = ""
) -> ClaudeAgentOptions:
cli_path = shutil.which("claude")
if cli_path:
logger.info(f"Claude CLI: {cli_path}")
else:
logger.warning("No `claude` on PATH; falling back to the SDK's bundled CLI.")
# Naming the domain vocabulary lets Claude resolve a garbled transcript
# while answering it. That recovers most of what a separate correction pass
# would, at no latency cost, because he already reads every transcript.
# Personality first, then the voice rules, so the constraints of speaking
# aloud get the last word over anything the personality file asks for.
style = ""
if personality := read_personality(args.cwd):
style += personality + "\n\n"
style += VOICE_STYLE
if brain and (memory := brain.prompt_block()):
# Only the context goes in the prompt. When to write back is described
# by the memory skill in the workspace, which Claude picks up on its own.
style += "\n\n" + memory
if journal_context:
style += "\n\n" + journal_context
if vocabulary and (terms := vocabulary.prompt_block()):
style += (
"\n\nSpeech recognition mangles unusual words. When a transcript is "
@@ -505,33 +497,34 @@ def build_claude_options(args: argparse.Namespace, vocabulary=None, brain=None)
permission_mode="bypassPermissions",
cwd=args.cwd,
model=args.claude_model,
# "project" makes the workspace's own .claude/skills discoverable;
# without it a skills folder there is silently ignored.
setting_sources=(
["user", "project", "local"] if args.load_settings else ["project"]
),
skills="all",
include_partial_messages=True, # Speak as tokens arrive instead of per message.
# The CLI's own sandbox uses sandbox-exec, which fails when this process
# is already running inside one.
include_partial_messages=True,
sandbox=SandboxSettings(enabled=False),
)
def build_llm(args: argparse.Namespace, vocabulary=None, brain=None, observer=None):
if args.llm_engine in ("opencode", "ollama"):
from opencode_llm import OpenCodeLLM, probe_opencode
def build_llm(
args: argparse.Namespace,
vocabulary=None,
brain=None,
observer=None,
model_manager=None,
journal_context: str = "",
):
model = (model_manager.load_saved_model() if model_manager else None) or getattr(args, "hermes_model", "hermes-3")
if args.llm_engine in ("hermes", "ollama"):
from hermes_llm import HermesLLM, probe_hermes
available, reason = probe_opencode(args.opencode_model)
logger.info(f"LLM: OpenCode Cloud ({reason})")
personality = read_personality(args.cwd) or "You are a helpful voice assistant."
system_prompt = personality + "\n\n" + VOICE_STYLE
if brain and (memory := brain.prompt_block()):
system_prompt += "\n\n" + memory
return OpenCodeLLM(
model=args.opencode_model,
available, reason = probe_hermes(model)
logger.info(f"LLM: Hermes ({reason})")
# Hermes handles persona, personality, and memory natively.
return HermesLLM(
model=model,
cwd=args.cwd,
system_prompt=system_prompt,
session_name="Voice Agent",
observer=observer,
)
@@ -544,11 +537,13 @@ def build_llm(args: argparse.Namespace, vocabulary=None, brain=None, observer=No
system_prompt = personality + "\n\n" + VOICE_STYLE
if brain and (memory := brain.prompt_block()):
system_prompt += "\n\n" + memory
if journal_context:
system_prompt += "\n\n" + journal_context
return MacOSLLM(model=args.mlx_model, system_prompt=system_prompt, observer=observer)
logger.info("LLM: Claude Code")
return ClaudeCodeLLM(
options=build_claude_options(args, vocabulary, brain),
options=build_claude_options(args, vocabulary, brain, journal_context),
observer=observer,
)
@@ -595,6 +590,9 @@ async def main() -> int:
vocabulary = build_vocabulary(args, brain)
stt = build_stt(args, vocabulary)
journal = Journal(workspace / "journal.jsonl")
journal_context = recent_prompt(workspace / "journal.jsonl")
if journal_context:
logger.info("Journal: loaded the 10 most recent entries")
def on_reply(text: str):
journal.record_reply(text)
@@ -602,7 +600,20 @@ async def main() -> int:
vocabulary.observe(text)
voice_manager = VoiceManager(workspace)
llm = build_llm(args, vocabulary, brain, observer=on_reply)
model_manager = ModelManager(workspace)
if not getattr(args, "no_web", False):
await web_server.start_server(workspace, port=getattr(args, "web_port", 8888))
web_server.set_managers(workspace, model_manager, voice_manager)
llm = build_llm(
args,
vocabulary,
brain,
observer=on_reply,
model_manager=model_manager,
journal_context=journal_context,
)
model_manager.set_llm_processor(llm)
tts = build_tts(args, voice_manager=voice_manager)
# Claude keeps its own history; this context exists so Pipecat can decide
@@ -645,6 +656,26 @@ async def main() -> int:
idle_timeout_secs=None,
)
async def on_web_input(text: str):
logger.info(f"Web typed input: {text}")
web_server.broadcast_event("heard", {"text": text})
if hasattr(llm, "start_turn_direct"):
llm.start_turn_direct(text)
else:
frames = [
UserStartedSpeakingFrame(),
TranscriptionFrame(
text=text,
user_id="user",
timestamp=datetime.now().isoformat(timespec="seconds")
),
UserStoppedSpeakingFrame(),
]
await worker.queue_frames(frames)
if not getattr(args, "no_web", False):
web_server.set_managers(workspace, model_manager, voice_manager, input_callback=on_web_input)
if args.greeting:
await worker.queue_frames([TTSSpeakFrame(args.greeting)])