adolforeyna/VoiceAgent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: real-time reasoning speech, dynamic voice tags, and unified web UI bubbles

Browse Source
This commit is contained in:
Adolfo Reyna
2026-08-12 10:17:47 -04:00
parent b5034b4b16
commit 404a7071fb
8 changed files with 734 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hermes_llm.py
+330 -43
View File
@@ -12,6 +12,8 @@ import re
import shutil
from pathlib import Path
import aiohttp
from typing import Callable, Optional
import time
from loguru import logger
import env_setup
@@ -26,6 +28,8 @@ from pipecat.frames.frames import (
LLMFullResponseStartFrame,
LLMTextFrame,
StartFrame,
TextFrame,
TTSSpeakFrame,
)
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
@@ -81,6 +85,38 @@ def _clean_spoken_text(text: str) -> str:
return " ".join(lines).strip()
class StreamParser:
"""Parses streaming tokens, separating <think>...</think> reasoning blocks from spoken text."""
def __init__(self):
self.in_think = False
def feed(self, chunk: str) -> tuple[str, str]:
thinking = ""
spoken = ""
buf = chunk
while buf:
if not self.in_think:
if "<think>" in buf:
parts = buf.split("<think>", 1)
spoken += parts[0]
self.in_think = True
buf = parts[1]
else:
spoken += buf
buf = ""
else:
if "</think>" in buf:
parts = buf.split("</think>", 1)
thinking += parts[0]
self.in_think = False
buf = parts[1]
else:
thinking += buf
buf = ""
return thinking, spoken
def find_hermes_cli() -> str | None:
candidates = [
shutil.which("hermes"),
@@ -95,20 +131,37 @@ def find_hermes_cli() -> str | None:
return shutil.which("hermes")
async def check_hermes_server_active(port: int = 8642) -> tuple[bool, str]:
"""Check if Hermes gateway server daemon is responding to health requests."""
url = f"http://localhost:{port}/api/health"
def get_hermes_api_key() -> str:
env_file = Path.home() / ".hermes" / ".env"
if env_file.exists():
try:
with open(env_file, "r") as f:
for line in f:
if line.startswith("API_SERVER_KEY="):
return line.split("=", 1)[1].strip().strip('"').strip("'")
except Exception:
pass
return os.environ.get("API_SERVER_KEY", "")
async def check_hermes_server_active(port: int = 9119) -> tuple[bool, str]:
"""Check if Hermes OpenAI gateway endpoint (v1/models) is responding with authorization."""
url_models = f"http://127.0.0.1:{port}/v1/models"
key = get_hermes_api_key()
headers = {"Authorization": f"Bearer {key}"} if key else {}
try:
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=1.5)) as session:
async with session.get(url) as resp:
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=1.0)) as session:
async with session.get(url_models, headers=headers) as resp:
if resp.status == 200:
return True, f"Hermes server active on http://localhost:{port}"
return True, f"Hermes Gateway API active on http://127.0.0.1:{port}/v1"
elif resp.status == 401:
return False, "Hermes Gateway API requires API_SERVER_KEY"
except Exception:
pass
return False, "Hermes server daemon not active"
return False, "Hermes Gateway API server not active"
async def ensure_hermes_server(port: int = 8642) -> tuple[bool, str]:
async def ensure_hermes_server(port: int = 9119) -> tuple[bool, str]:
"""Ensure Hermes gateway server daemon or CLI binary is available."""
active, msg = await check_hermes_server_active(port)
if active:
@@ -127,8 +180,6 @@ def probe_hermes(model: str | None = None) -> tuple[bool, str]:
m_str = f" with model {model}" if model else ""
return True, f"Hermes available ({cli}){m_str}"
return False, "Hermes CLI binary not found (install hermes or ensure it is in PATH)"
class HermesLLM(FrameProcessor):
"""Runs user turns through Hermes CLI or Gateway API using persistent session tracking."""
@@ -137,10 +188,12 @@ class HermesLLM(FrameProcessor):
*,
model: str | None = None,
cwd: str | Path | None = None,
port: int = 8642,
port: int = 9119,
session_name: str = "Voice Agent",
observer=None,
on_tool_event: Optional[Callable[[str], None]] = None,
use_server: bool = False,
keep_open: bool = True,
**kwargs,
):
super().__init__(**kwargs)
@@ -149,13 +202,19 @@ class HermesLLM(FrameProcessor):
self._port = port
self._session_name = session_name
self._on_reply = observer
self._on_tool_event = on_tool_event
self._turn_task: asyncio.Task | None = None
self._history: list[dict[str, str]] = []
self._cli_path = find_hermes_cli() or "hermes"
self._use_server = use_server
self._keep_open = keep_open
self._session_renamed = False
self._http_session: aiohttp.ClientSession | None = None
self._proc: asyncio.subprocess.Process | None = None
self._proc_lock = asyncio.Lock()
self._stderr_task: asyncio.Task | None = None
# Keep the conversation lineage with the workspace. A single global
# session file can make two voice-agent workspaces resume each other's
# Hermes conversations.
@@ -172,6 +231,8 @@ class HermesLLM(FrameProcessor):
self._session_id = None
self._session_renamed = False
self._history.clear()
if self._proc:
asyncio.create_task(self._stop_persistent_proc())
try:
if self._session_state_file.exists():
self._session_state_file.unlink()
@@ -186,6 +247,8 @@ class HermesLLM(FrameProcessor):
self._session_id = disk_sid
self._session_renamed = False
self._history.clear()
if self._proc:
asyncio.create_task(self._stop_persistent_proc())
async def _get_http_session(self) -> aiohttp.ClientSession:
if self._http_session is None or self._http_session.closed:
@@ -250,6 +313,53 @@ class HermesLLM(FrameProcessor):
except Exception as e:
logger.debug(f"Could not rename Hermes session: {e}")
async def _ensure_persistent_proc(self):
async with self._proc_lock:
ok, _ = await check_hermes_server_active(self._port)
if ok:
return
cmd = [self._cli_path, "serve", "--port", str(self._port), "--skip-build"]
try:
env = {**os.environ, "PYTHONUNBUFFERED": "1", "FORCE_COLOR": "0", "NO_COLOR": "1"}
self._proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=str(self._cwd),
env=env,
)
logger.info(f"Auto-started Hermes server (`hermes serve --port {self._port} --skip-build`, PID: {self._proc.pid})")
for _ in range(50):
ready, _ = await check_hermes_server_active(self._port)
if ready:
logger.info(f"Hermes server active and ready on port {self._port}.")
break
await asyncio.sleep(0.1)
except Exception as e:
logger.warning(f"Could not auto-start Hermes server daemon (`hermes serve --skip-build`): {e}")
async def _stop_persistent_proc(self):
async with self._proc_lock:
proc = self._proc
self._proc = None
if self._stderr_task and not self._stderr_task.done():
self._stderr_task.cancel()
self._stderr_task = None
if proc and proc.returncode is None:
try:
if proc.stdin and not proc.stdin.is_closing():
proc.stdin.close()
proc.terminate()
await asyncio.wait_for(proc.wait(), timeout=1.5)
except Exception:
try:
proc.kill()
except Exception:
pass
logger.info("Persistent Hermes process terminated cleanly.")
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
@@ -257,8 +367,11 @@ class HermesLLM(FrameProcessor):
await self.push_frame(frame, direction)
available, reason = probe_hermes(self._model)
logger.info(f"Hermes LLM engine initialized: {reason}")
if self._keep_open:
asyncio.create_task(self._ensure_persistent_proc())
elif isinstance(frame, (EndFrame, CancelFrame)):
await self._cancel_turn()
await self._stop_persistent_proc()
await self._close_http_session()
await self.push_frame(frame, direction)
elif isinstance(frame, InterruptionFrame):
@@ -325,76 +438,237 @@ class HermesLLM(FrameProcessor):
if new_model != self._model:
logger.info(f"Hermes LLM switching active model: {self._model} -> {new_model}")
self._model = new_model
if self._proc:
asyncio.create_task(self._stop_persistent_proc())
except Exception as e:
logger.debug(f"Could not read model settings: {e}")
async def _run_turn(self, utterance: str):
async def _run_turn(self, utterance: str, suppress_output: bool = False):
self._sync_disk_model()
self._sync_disk_session()
self._history.append({"role": "user", "content": utterance})
await self.push_frame(LLMFullResponseStartFrame())
if not suppress_output:
await self.push_frame(LLMFullResponseStartFrame())
chunks: list[str] = []
if self._use_server:
if self._keep_open:
await self._ensure_persistent_proc()
t0 = time.perf_counter()
server_ok, _ = await check_hermes_server_active(self._port)
mode_str = f"API ({self._port})" if (server_ok or self._use_server) else "CLI"
try:
import web_server
web_server.broadcast_event("hermes_status", {
"mode": mode_str,
"is_api": server_ok or self._use_server,
"port": self._port,
})
except Exception:
pass
if server_ok or self._use_server:
await self._run_turn_server(utterance, chunks)
else:
await self._run_turn_cli(utterance, chunks)
# The transport-specific runners may fall back from one to the other;
# emit exactly one response terminator for the whole turn.
await self.push_frame(LLMFullResponseEndFrame())
t1 = time.perf_counter()
total_ms = int((t1 - t0) * 1000)
if not suppress_output:
await self.push_frame(LLMFullResponseEndFrame())
logger.info(f"⏱ [PROFILING] Hermes LLM ({mode_str}): Turn completed in {total_ms}ms ({total_ms/1000:.2f}s)")
try:
import web_server
web_server.broadcast_event("profiling", {
"mode": mode_str,
"total_ms": total_ms,
"model": self._model or "default",
})
except Exception:
pass
full_reply = _clean_spoken_text(" ".join(chunks))
if full_reply:
self._history.append({"role": "assistant", "content": full_reply})
logger.info(f"Hermes LLM ({self._model or 'default'}): {full_reply}")
if self._on_reply:
self._on_reply(full_reply)
if not suppress_output:
try:
import web_server
web_server.broadcast_event("reply", {"text": full_reply})
except Exception:
pass
if self._on_reply:
self._on_reply(full_reply)
async def _run_turn_server(self, utterance: str, chunks: list[str]):
"""Run turn via Hermes Server / Gateway HTTP API if available."""
"""Run turn via Hermes OpenAI-compatible Gateway API using SSE streaming (stream: true)."""
try:
ok, _ = await check_hermes_server_active(self._port)
if not ok:
raise RuntimeError("Hermes server daemon unavailable")
raise RuntimeError("Hermes Gateway API unavailable")
key = get_hermes_api_key()
headers = {"Authorization": f"Bearer {key}", "Content-Type": "application/json"} if key else {"Content-Type": "application/json"}
url = f"http://127.0.0.1:{self._port}/v1/chat/completions"
messages = [dict(m) for m in self._history[-10:]]
if not messages or messages[-1].get("content") != utterance:
messages.append({"role": "user", "content": utterance})
session_id = self._session_id or "voice-agent"
url = f"http://localhost:{self._port}/api/sessions/{session_id}/chat"
payload = {
"message": utterance,
"model": "hermes-agent" if not self._model or self._model.lower() in ("default", "none", "") else self._model,
"messages": messages,
"stream": True,
}
if self._model and self._model.lower() not in ("default", "none", ""):
payload["model"] = self._model
session = await self._get_http_session()
async with session.post(url, json=payload) as resp:
if resp.status == 200:
data = await resp.json()
text_val = data.get("reply") or data.get("text") or data.get("content", "")
cleaned = _clean_spoken_text(str(text_val))
if cleaned:
chunks.append(cleaned)
await self.push_frame(LLMTextFrame(cleaned))
async with session.post(url, json=payload, headers=headers) as resp:
if resp.status == 200:
sentence_buffer = ""
reasoning_buffer = ""
parser = StreamParser()
async for raw_line in resp.content:
line = raw_line.decode("utf-8").strip()
if not line or line.startswith(":"):
continue
if line == "data: [DONE]":
break
if line.startswith("data: "):
try:
data = json.loads(line[6:])
choices = data.get("choices", [])
if choices:
delta = choices[0].get("delta", {})
# Handle thinking/reasoning deltas immediately
reasoning = delta.get("reasoning") or delta.get("thought")
if reasoning:
reasoning_buffer += str(reasoning)
words = reasoning_buffer.strip().split()
if "\n" in reasoning_buffer or any(p in reasoning_buffer for p in (".", "!", "?")) or len(words) >= 4:
reasoning_phrase = reasoning_buffer.strip()
reasoning_buffer = ""
cleaned_reasoning = _clean_spoken_text(reasoning_phrase)
if cleaned_reasoning:
if not cleaned_reasoning.endswith((".", "!", "?")):
cleaned_reasoning += "."
chunks.append(cleaned_reasoning)
# Push TTSSpeakFrame so Kokoro TTS synthesizes & plays audio IMMEDIATELY
await self.push_frame(TTSSpeakFrame(cleaned_reasoning))
try:
import web_server
web_server.broadcast_event("thinking", {"text": str(reasoning)})
except Exception:
pass
# Handle tool call deltas
tool_calls = delta.get("tool_calls")
if tool_calls:
for tc in tool_calls:
fn = tc.get("function", {})
tool_name = fn.get("name", "tool")
tool_args = fn.get("arguments", "")
if self._on_tool_event:
self._on_tool_event(f"Executing {tool_name}")
try:
import web_server
web_server.broadcast_event("tool", {"name": tool_name, "detail": tool_args})
except Exception:
pass
content = delta.get("content", "")
if content:
think_text, spoken_text = parser.feed(content)
if think_text:
reasoning_buffer += think_text
words = reasoning_buffer.strip().split()
if "\n" in reasoning_buffer or any(p in reasoning_buffer for p in (".", "!", "?")) or len(words) >= 4:
reasoning_phrase = reasoning_buffer.strip()
reasoning_buffer = ""
cleaned_reasoning = _clean_spoken_text(reasoning_phrase)
if cleaned_reasoning:
if not cleaned_reasoning.endswith((".", "!", "?")):
cleaned_reasoning += "."
chunks.append(cleaned_reasoning)
# Push TTSSpeakFrame so Kokoro TTS synthesizes & plays audio IMMEDIATELY
await self.push_frame(TTSSpeakFrame(cleaned_reasoning))
try:
import web_server
web_server.broadcast_event("thinking", {"text": think_text})
except Exception:
pass
if spoken_text:
sentence_buffer += spoken_text
while any(p in sentence_buffer for p in (".", "!", "?", "\n")):
idxs = [sentence_buffer.find(p) for p in (".", "!", "?", "\n") if sentence_buffer.find(p) != -1]
split_idx = min(idxs) + 1
sentence = sentence_buffer[:split_idx].strip()
sentence_buffer = sentence_buffer[split_idx:]
cleaned_sent = _clean_spoken_text(sentence)
if cleaned_sent:
if not cleaned_sent.endswith((".", "!", "?")):
cleaned_sent += "."
chunks.append(cleaned_sent)
await self.push_frame(LLMTextFrame(cleaned_sent))
try:
import web_server
web_server.broadcast_event("partial_reply", {"text": cleaned_sent})
except Exception:
pass
except json.JSONDecodeError:
pass
if reasoning_buffer.strip():
cleaned_r_rem = _clean_spoken_text(reasoning_buffer)
if cleaned_r_rem:
if not cleaned_r_rem.endswith((".", "!", "?")):
cleaned_r_rem += "."
chunks.append(cleaned_r_rem)
await self.push_frame(TTSSpeakFrame(cleaned_r_rem))
if sentence_buffer.strip():
cleaned_rem = _clean_spoken_text(sentence_buffer)
if cleaned_rem:
if not cleaned_rem.endswith((".", "!", "?")):
cleaned_rem += "."
chunks.append(cleaned_rem)
await self.push_frame(LLMTextFrame(cleaned_rem))
try:
import web_server
web_server.broadcast_event("partial_reply", {"text": cleaned})
web_server.broadcast_event("partial_reply", {"text": cleaned_rem})
except Exception:
pass
else:
err_text = await resp.text()
logger.error(f"Hermes server HTTP {resp.status}: {err_text}")
raise RuntimeError(f"HTTP {resp.status}")
else:
err_text = await resp.text()
logger.error(f"Hermes Gateway API HTTP {resp.status}: {err_text}")
raise RuntimeError(f"HTTP {resp.status}")
except asyncio.CancelledError:
logger.info("Hermes server turn cancelled mid-response.")
logger.info("Hermes Gateway turn cancelled mid-response.")
raise
except Exception as e:
logger.warning(f"Hermes server error ({e}), falling back to CLI...")
logger.warning(f"Hermes Gateway error ({e}), falling back to CLI...")
try:
import web_server
web_server.broadcast_event("hermes_status", {
"mode": "CLI",
"is_api": False,
"port": self._port,
})
except Exception:
pass
await self._run_turn_cli(utterance, chunks)
async def _run_turn_cli(self, utterance: str, chunks: list[str]):
"""Run turn via Hermes CLI using persistent session tracking."""
cmd = [self._cli_path, "chat", "-q", utterance, "-Q", "--source", "voice"]
cmd = [self._cli_path, "chat", "-q", utterance, "-Q", "--source", "voice", "--reasoning", "none"]
if self._session_id:
cmd.extend(["-r", self._session_id])
if self._model and self._model.lower() not in ("default", "none", ""):
@@ -423,7 +697,14 @@ class HermesLLM(FrameProcessor):
# Extract session_id emitted on stderr
self._remember_session_id(cleaned)
if cleaned and cleaned not in ("[0m", "0m", "]") and not cleaned.startswith(">"):
if (
cleaned
and cleaned not in ("[0m", "0m", "]")
and not cleaned.startswith(">")
and not cleaned.startswith("↻")
and not "Resumed session" in cleaned
and not cleaned.lower().startswith("session_id:")
):
logger.info(f"Hermes Tool: {cleaned}")
try:
import web_server
@@ -431,6 +712,12 @@ class HermesLLM(FrameProcessor):
except Exception:
pass
if self._on_tool_event:
try:
self._on_tool_event(cleaned)
except Exception as exc:
logger.debug(f"on_tool_event error: {exc}")
# Keep tool progress visible in logs and the Companion Web UI, but
# do not send implementation details through the spoken channel.