Compare commits

...

10 Commits

7 changed files with 392 additions and 36 deletions
+4 -2
View File
@@ -92,8 +92,10 @@ def _clean_spoken_text(text: str) -> str:
"""Clean text for speech output and truncate fake turn generations."""
if not text:
return ""
# Truncate if model hallucinates fake turn markers
for marker in ("User:", "Human:", "Assistant:", "\nUser", "\nHuman", "\nAssistant"):
# Strip leading or inline role headers (e.g. "Assistant:")
text = re.sub(r"(?i)\b(Assistant|assistant|Bot|bot):\s*", "", text)
# Truncate if model hallucinates fake subsequent user turns
for marker in ("\nUser:", "\nHuman:", "\nUser", "\nHuman"):
if marker in text:
text = text.split(marker)[0]
# Remove markdown code blocks
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/env python3
"""CLI helper to list and set voices for OpenCode and Claude tools."""
import sys
from pathlib import Path
# Add VoiceAgent1 project root to sys.path
project_root = Path("/Users/adolforeyna/Projects/VoiceAgent1")
if str(project_root) not in sys.path:
sys.path.insert(0, str(project_root))
from voice_manager import KOKORO_VOICES, MACOS_VOICES, VoiceManager
def main():
workspace = Path.home() / "Workspace"
vm = VoiceManager(workspace)
if len(sys.argv) < 2 or sys.argv[1] in ("list", "ls", "--list"):
print("Available Voices:\n")
print("Kokoro Voices (On-Device Neural Speech):")
for k, v in KOKORO_VOICES.items():
active = " (ACTIVE)" if k == vm._active_voice else ""
print(f" - {k}: {v}{active}")
print("\nmacOS System Voices:")
for k, v in MACOS_VOICES.items():
active = " (ACTIVE)" if k == vm._active_voice else ""
print(f" - {k}: {v}{active}")
return
action = sys.argv[1]
if action in ("set", "change") and len(sys.argv) >= 3:
target_voice = sys.argv[2]
ok, msg = vm.apply_voice(target_voice)
print(msg)
else:
# Treat single argument as target voice
target_voice = sys.argv[1]
ok, msg = vm.apply_voice(target_voice)
print(msg)
if __name__ == "__main__":
main()
+29 -7
View File
@@ -18,6 +18,7 @@ from claude_agent_sdk import ClaudeAgentOptions, SandboxSettings
from loguru import logger
from brain import Brain
from voice_manager import VoiceManager
from claude_llm import ClaudeCodeLLM
from echo_guard import EchoGuardUserMuteStrategy
from pipecat.audio.vad.silero import SileroVADAnalyzer
@@ -69,7 +70,9 @@ say them:
conversation, not a document.
- Spell out things that only make sense visually. Say "line forty-two of
bot dot py" rather than pasting a path.
- Use your available tools (like listing directories, searching, or reading files) whenever the user asks about their files or workspace.
- Use your available tools (listing directories, searching, reading files) whenever the user asks about files or workspace tasks.
- You can change your own voice! If the user asks to list available voices or switch voice, run `python bin/voice_tool.py list` or `python bin/voice_tool.py set <voice_name>` (voices: af_heart, af_bella, am_michael, am_fenrir, am_puck, bf_emma, bm_george, Moira, Daniel).
- Complete multi-step tool calls fully before speaking your response. Do not stop halfway to ask if you should continue.
- Be strictly truthful about your findings and never invent fake file contents.
- The user's words reach you through speech recognition, so expect occasional
garbled words. Ask rather than guess when it matters.
@@ -325,7 +328,7 @@ def build_stt(args: argparse.Namespace, vocabulary):
return WhisperSTTService(settings=WhisperSTTService.Settings(model=model, language=Language.EN))
def build_tts(args: argparse.Namespace):
def build_tts(args: argparse.Namespace, voice_manager=None):
if args.tts == "apple":
from apple_tts import AppleTTSService, find_voice
@@ -338,12 +341,15 @@ def build_tts(args: argparse.Namespace):
if args.voice_rate:
logger.warning("--voice-rate only applies to --tts apple; ignoring it.")
voice = args.voice or "af_heart"
voice = (voice_manager.load_saved_voice() if voice_manager else None) or args.voice or "af_heart"
logger.info(f"Text to speech: Kokoro {voice}")
return KokoroTTSService(
tts = KokoroTTSService(
settings=KokoroTTSService.Settings(voice=voice, language=Language.EN),
text_filters=[SpokenTextFilter()],
text_filters=[SpokenTextFilter(voice_manager=voice_manager)],
)
if voice_manager:
voice_manager.set_tts_processor(tts)
return tts
def build_turn_taking(args: argparse.Namespace):
@@ -403,6 +409,22 @@ def build_vocabulary(args: argparse.Namespace, brain=None) -> Vocabulary | None:
target.write_text(template.read_text())
logger.info(f"Created {target} from the template")
# Ensure bin/voice_tool.py is available in the workspace for OpenCode and Claude
ws_bin = workspace / "bin"
ws_bin.mkdir(exist_ok=True)
voice_script = ws_bin / "voice_tool.py"
source_script = here / "bin" / "voice_tool.py"
if source_script.exists() and not voice_script.exists():
try:
voice_script.symlink_to(source_script.resolve())
logger.info(f"Created symlink {voice_script} -> {source_script}")
except Exception:
try:
voice_script.write_text(source_script.read_text())
logger.info(f"Copied {source_script} -> {voice_script}")
except Exception as e:
logger.warning(f"Could not install voice_tool.py into {ws_bin}: {e}")
vocabulary = Vocabulary(
project_dir=workspace,
vocabulary_file=vocabulary_file,
@@ -579,9 +601,9 @@ async def main() -> int:
if vocabulary:
vocabulary.observe(text)
voice_manager = VoiceManager(workspace)
llm = build_llm(args, vocabulary, brain, observer=on_reply)
tts = build_tts(args)
tts = build_tts(args, voice_manager=voice_manager)
# Claude keeps its own history; this context exists so Pipecat can decide
# when a turn has ended.
+4 -2
View File
@@ -42,8 +42,10 @@ def _clean_spoken_text(text: str) -> str:
"""Clean text for speech output and truncate fake turn generations."""
if not text:
return ""
# Truncate if model hallucinates fake turn markers
for marker in ("User:", "Human:", "Assistant:", "\nUser", "\nHuman", "\nAssistant"):
# Strip leading or inline role headers (e.g. "Assistant:")
text = re.sub(r"(?i)\b(Assistant|assistant|Bot|bot):\s*", "", text)
# Truncate if model hallucinates fake subsequent user turns
for marker in ("\nUser:", "\nHuman:", "\nUser", "\nHuman"):
if marker in text:
text = text.split(marker)[0]
# Remove markdown code blocks
+154 -25
View File
@@ -1,7 +1,8 @@
"""A Pipecat processor that puts OpenCode (with ollama-cloud/gemma4:31b) in the LLM slot.
Drives the installed OpenCode CLI (`opencode run -m ollama-cloud/gemma4:31b`) to stream
cloud responses to text-to-speech downstream with zero local GPU overhead.
Supports both:
1. OpenCode Server Daemon (`opencode serve --port 4096`) for zero-latency, persistent in-memory sessions.
2. OpenCode CLI (`opencode run --continue`) for direct process invocation.
"""
import asyncio
@@ -10,6 +11,7 @@ import os
import re
import shutil
from pathlib import Path
import aiohttp
from loguru import logger
from pipecat.frames.frames import (
@@ -37,13 +39,17 @@ _NOISE_TRANSCRIPTS = {
"[silence]",
}
_server_proc: asyncio.subprocess.Process | None = None
def _clean_spoken_text(text: str) -> str:
"""Clean text for speech output and truncate fake turn generations."""
if not text:
return ""
# Truncate if model hallucinates fake turn markers
for marker in ("User:", "Human:", "Assistant:", "\nUser", "\nHuman", "\nAssistant"):
# Strip leading or inline role headers (e.g. "Assistant:")
text = re.sub(r"(?i)\b(Assistant|assistant|Bot|bot):\s*", "", text)
# Truncate if model hallucinates fake subsequent user turns
for marker in ("\nUser:", "\nHuman:", "\nUser", "\nHuman"):
if marker in text:
text = text.split(marker)[0]
# Remove markdown code blocks
@@ -65,40 +71,82 @@ def find_opencode_cli() -> str | None:
)
async def ensure_opencode_server(port: int = 4096) -> tuple[bool, str]:
"""Ensure opencode serve daemon is running on port."""
global _server_proc
url = f"http://localhost:{port}/session"
try:
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=2.0)) as session:
async with session.get(url) as resp:
if resp.status == 200:
return True, f"OpenCode server active on http://localhost:{port}"
except Exception:
pass
cli = find_opencode_cli()
if not cli:
return False, "OpenCode CLI binary not found"
logger.info(f"Starting OpenCode server daemon on port {port}...")
try:
_server_proc = await asyncio.create_subprocess_exec(
cli,
"serve",
"--port", str(port),
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.DEVNULL,
)
await asyncio.sleep(1.2)
return True, f"Started OpenCode server daemon on http://localhost:{port}"
except Exception as e:
return False, f"Failed to start OpenCode server daemon: {e}"
def probe_opencode(model: str = "ollama-cloud/gemma4:31b") -> tuple[bool, str]:
cli = find_opencode_cli()
if not cli:
return False, "OpenCode CLI binary not found"
return True, f"OpenCode CLI available ({cli}) with model {model}"
return True, f"OpenCode available ({cli}) with model {model}"
class OpenCodeLLM(FrameProcessor):
"""Runs user turns through the OpenCode CLI driving OpenCode Cloud models."""
"""Runs user turns through OpenCode Server Daemon or CLI."""
def __init__(
self,
*,
model: str = "ollama-cloud/gemma4:31b",
cwd: str | Path | None = None,
port: int = 4096,
system_prompt: str | None = None,
observer=None,
use_server: bool = False,
**kwargs,
):
super().__init__(**kwargs)
self._model = model
self._cwd = Path(cwd or Path.home() / "Workspace").expanduser().resolve()
self._port = port
self._system_prompt = system_prompt or "You are a helpful spoken voice assistant. Keep answers brief and conversational."
self._on_reply = observer
self._turn_task: asyncio.Task | None = None
self._history: list[dict[str, str]] = []
self._cli_path = find_opencode_cli() or "opencode"
self._use_server = use_server
self._server_session_id: str | None = None
self._has_session = False
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, StartFrame):
await self.push_frame(frame, direction)
logger.info(f"OpenCode LLM engine ready: CLI={self._cli_path}, model={self._model}")
if self._use_server:
ok, reason = await ensure_opencode_server(self._port)
logger.info(f"OpenCode Server engine: {reason}")
else:
logger.info(f"OpenCode CLI engine ready: CLI={self._cli_path}, model={self._model}")
elif isinstance(frame, (EndFrame, CancelFrame)):
await self._cancel_turn()
await self.push_frame(frame, direction)
@@ -143,7 +191,6 @@ class OpenCodeLLM(FrameProcessor):
async def _run_turn(self, utterance: str):
self._history.append({"role": "user", "content": utterance})
# Format prompt with system instructions and recent conversation history
recent_history = self._history[-6:]
conv_text = "\n".join(
f"{'User' if m['role']=='user' else 'Assistant'}: {m['content']}"
@@ -154,19 +201,107 @@ class OpenCodeLLM(FrameProcessor):
await self.push_frame(LLMFullResponseStartFrame())
chunks: list[str] = []
if self._use_server:
await self._run_turn_server(prompt_str, chunks)
else:
await self._run_turn_cli(prompt_str, chunks)
full_reply = _clean_spoken_text(" ".join(chunks))
if full_reply:
self._history.append({"role": "assistant", "content": full_reply})
logger.info(f"OpenCode LLM ({self._model}): {full_reply}")
if self._on_reply:
self._on_reply(full_reply)
async def _run_turn_server(self, prompt_str: str, chunks: list[str]):
"""Run turn via OpenCode Server Daemon HTTP API."""
try:
ok, _ = await ensure_opencode_server(self._port)
if not ok:
raise RuntimeError("OpenCode server daemon unavailable")
async with aiohttp.ClientSession() as session:
if not self._server_session_id:
create_url = f"http://localhost:{self._port}/session"
async with session.post(create_url, json={"directory": str(self._cwd)}) as res:
if res.status == 200:
data = await res.json()
self._server_session_id = data.get("id")
logger.info(f"OpenCode server session created: {self._server_session_id}")
if not self._server_session_id:
raise RuntimeError("Failed to create OpenCode server session")
msg_url = f"http://localhost:{self._port}/session/{self._server_session_id}/message"
model_id = self._model.split("/")[-1] if "/" in self._model else self._model
provider_id = self._model.split("/")[0] if "/" in self._model else "ollama-cloud"
payload = {
"model": {"providerID": provider_id, "modelID": model_id},
"parts": [{"type": "text", "text": prompt_str}],
}
async with session.post(msg_url, json=payload) as resp:
if resp.status == 200:
data = await resp.json()
parts = data.get("parts", []) if isinstance(data, dict) else []
for p in parts:
if isinstance(p, dict):
p_type = p.get("type")
if p_type == "text" and "text" in p:
text_chunk = _clean_spoken_text(p["text"])
if text_chunk:
chunks.append(text_chunk)
await self.push_frame(LLMTextFrame(text_chunk))
elif p_type not in ("step-start", "step-finish"):
logger.info(f"OpenCode Tool: {p_type} -> {json.dumps(p)[:120]}")
if not chunks and isinstance(data, dict):
if "delta" in data:
text_chunk = _clean_spoken_text(data["delta"])
if text_chunk:
chunks.append(text_chunk)
await self.push_frame(LLMTextFrame(text_chunk))
elif "text" in data:
text_chunk = _clean_spoken_text(data["text"])
if text_chunk:
chunks.append(text_chunk)
await self.push_frame(LLMTextFrame(text_chunk))
else:
err_text = await resp.text()
logger.error(f"OpenCode server HTTP {resp.status}: {err_text}")
except asyncio.CancelledError:
logger.info("OpenCode server turn cancelled mid-response.")
raise
except Exception as e:
logger.warning(f"OpenCode server error ({e}), falling back to CLI...")
await self._run_turn_cli(prompt_str, chunks)
finally:
await self.push_frame(LLMFullResponseEndFrame())
async def _run_turn_cli(self, prompt_str: str, chunks: list[str]):
"""Fallback turn via OpenCode CLI."""
cmd = [
self._cli_path,
"run",
"-m", self._model,
"--dir", str(self._cwd),
"--auto",
]
if self._has_session:
cmd.append("--continue")
cmd.append(prompt_str)
proc = None
try:
proc = await asyncio.create_subprocess_exec(
self._cli_path,
"run",
"-m", self._model,
"--dir", str(self._cwd),
"--auto",
prompt_str,
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
stdin=asyncio.subprocess.DEVNULL,
cwd=str(self._cwd),
)
self._has_session = True
async def _read_stderr(stream):
while True:
@@ -187,11 +322,14 @@ class OpenCodeLLM(FrameProcessor):
cleaned = _clean_spoken_text(text_line)
if cleaned:
chunks.append(cleaned)
await self.push_frame(LLMTextFrame(cleaned))
await proc.wait()
await stderr_task
full_text = _clean_spoken_text(" ".join(chunks))
if full_text:
await self.push_frame(LLMTextFrame(full_text))
except asyncio.CancelledError:
logger.info("OpenCode turn cancelled mid-response.")
if proc and proc.returncode is None:
@@ -201,16 +339,7 @@ class OpenCodeLLM(FrameProcessor):
pass
raise
except Exception as e:
logger.error(f"OpenCode LLM error: {e}")
logger.error(f"OpenCode LLM CLI error: {e}")
err_msg = "Sorry, I ran into an error generating a response."
chunks.append(err_msg)
await self.push_frame(LLMTextFrame(err_msg))
finally:
await self.push_frame(LLMFullResponseEndFrame())
full_reply = _clean_spoken_text(" ".join(chunks))
if full_reply:
self._history.append({"role": "assistant", "content": full_reply})
logger.info(f"OpenCode LLM ({self._model}): {full_reply}")
if self._on_reply:
self._on_reply(full_reply)
+6
View File
@@ -34,7 +34,13 @@ _EXTRA_SPACE = re.compile(r"[ \t]{2,}")
class SpokenTextFilter(MarkdownTextFilter):
"""Markdown filtering, plus the leftovers that matter when read aloud."""
def __init__(self, voice_manager=None, **kwargs):
super().__init__(**kwargs)
self._voice_manager = voice_manager
async def filter(self, text: str) -> str:
if self._voice_manager:
self._voice_manager.sync_voice()
text = _TIMES.sub(" times ", text)
text = await super().filter(text)
text = _STRIKETHROUGH.sub(r"\1", text)
+153
View File
@@ -0,0 +1,153 @@
"""Voice Manager for listing and dynamically changing TTS voices at runtime.
Supports both Kokoro TTS voices and macOS System voices, persisting user voice preferences to disk.
"""
import json
import logging
from pathlib import Path
try:
from loguru import logger
except ImportError:
logger = logging.getLogger("voice_manager")
# Supported Kokoro TTS voices categorized by style
KOKORO_VOICES = {
"af_heart": "American Female - Warm & Natural (Default)",
"af_bella": "American Female - Clear & Expressive",
"af_sarah": "American Female - Soft & Smooth",
"af_nicole": "American Female - Relaxed",
"af_sky": "American Female - Bright",
"am_michael": "American Male - Friendly & Crisp",
"am_adam": "American Male - Natural",
"am_fenrir": "American Male - Deep",
"am_puck": "American Male - Energetic",
"bf_emma": "British Female - Professional",
"bf_isabella": "British Female - Smooth",
"bm_george": "British Male - Warm",
"bm_fable": "British Male - Expressive",
}
MACOS_VOICES = {
"Moira": "Irish Female",
"Daniel": "UK Male",
"Samantha": "US Female",
"Karen": "Australian Female",
"Alex": "US Male",
}
VOICE_ALIASES = {
"daniel": "bm_george", # British Male
"moira": "bf_emma", # British Female
"samantha": "af_bella", # US Female
"alex": "am_michael", # US Male
"karen": "bf_isabella", # AU/UK Female
"michael": "am_michael",
"bella": "af_bella",
"heart": "af_heart",
"fenrir": "am_fenrir",
"adam": "am_adam",
}
class VoiceManager:
"""Manages active voice settings and dynamic voice switching."""
def __init__(self, workspace_dir: Path, tts_processor=None):
self._workspace_dir = Path(workspace_dir)
self._tts_processor = tts_processor
self._config_file = self._workspace_dir / "voice_settings.json"
self._active_voice = "af_heart"
self.load_saved_voice()
def set_tts_processor(self, tts_processor):
self._tts_processor = tts_processor
if self._active_voice:
self.apply_voice(self._active_voice)
def load_saved_voice(self) -> str:
if self._config_file.exists():
try:
data = json.loads(self._config_file.read_text())
if "voice" in data:
self._active_voice = data["voice"]
logger.info(f"Loaded saved voice preference: {self._active_voice}")
except Exception as e:
logger.warning(f"Could not load saved voice settings: {e}")
return self._active_voice
def sync_voice(self) -> str:
"""Check if voice_settings.json was updated on disk and update TTS live."""
current_disk_voice = self.load_saved_voice()
if self._tts_processor and current_disk_voice:
if hasattr(self._tts_processor, "_settings"):
current_tts_voice = getattr(self._tts_processor._settings, "voice", None)
if current_tts_voice != current_disk_voice:
self._tts_processor._settings.voice = current_disk_voice
logger.info(f"Live TTS voice synced to: {current_disk_voice}")
elif hasattr(self._tts_processor, "set_voice"):
self._tts_processor.set_voice(current_disk_voice)
return current_disk_voice
def save_voice(self, voice_name: str):
try:
self._config_file.write_text(json.dumps({"voice": voice_name}, indent=2))
except Exception as e:
logger.warning(f"Could not save voice setting: {e}")
def list_voices(self) -> str:
kokoro_lines = [f"- {v}: {desc}" for v, desc in KOKORO_VOICES.items()]
macos_lines = [f"- {v}: {desc}" for v, desc in MACOS_VOICES.items()]
return (
"Available Voices:\n\n"
"Kokoro Voices:\n" + "\n".join(kokoro_lines) + "\n\n"
"macOS System Voices:\n" + "\n".join(macos_lines)
)
def apply_voice(self, voice_name: str) -> tuple[bool, str]:
voice_name = voice_name.strip()
matched_voice = None
# Check voice aliases first (e.g. Daniel -> bm_george)
clean_name = voice_name.lower()
if clean_name in VOICE_ALIASES:
matched_voice = VOICE_ALIASES[clean_name]
# Exact match or fuzzy match
if not matched_voice:
for v in KOKORO_VOICES:
if clean_name == v.lower():
matched_voice = v
break
if not matched_voice:
# Partial match search
for v in KOKORO_VOICES:
if clean_name in v.lower():
matched_voice = v
break
if not matched_voice:
available = ", ".join(list(KOKORO_VOICES.keys()))
return False, f"Voice '{voice_name}' not found. Available voices: {available}"
self._active_voice = matched_voice
self.save_voice(matched_voice)
if self._tts_processor:
try:
# Update Kokoro TTS setting if active
if hasattr(self._tts_processor, "_settings"):
self._tts_processor._settings.voice = matched_voice
logger.info(f"Dynamic voice updated to: {matched_voice}")
return True, f"Voice changed to {matched_voice}."
elif hasattr(self._tts_processor, "set_voice"):
self._tts_processor.set_voice(matched_voice)
logger.info(f"Dynamic voice updated to: {matched_voice}")
return True, f"Voice changed to {matched_voice}."
except Exception as e:
logger.error(f"Failed to apply voice to TTS processor: {e}")
return False, f"Could not change voice: {e}"
return True, f"Voice set to {matched_voice}."