Adolfo Reyna
279 lines
10 KiB
Python
279 lines
10 KiB
Python
"""A Pipecat processor that puts Claude Code in the LLM slot of a voice pipeline.
|
|
|
|
This drives the Claude Agent SDK rather than the raw Anthropic API, so the
|
|
conversation runs through the same `claude` CLI (and the same auth) that a
|
|
terminal session uses, and Claude keeps its tools.
|
|
|
|
The SDK owns conversation history, so the Pipecat context is used only for turn
|
|
detection: this reads the newest user message off each `LLMContextFrame` and
|
|
emits the `LLMFullResponseStartFrame` / `LLMTextFrame` /
|
|
`LLMFullResponseEndFrame` sequence the TTS service downstream expects.
|
|
"""
|
|
|
|
import asyncio
|
|
|
|
from claude_agent_sdk import (
|
|
AssistantMessage,
|
|
ClaudeAgentOptions,
|
|
ClaudeSDKClient,
|
|
ResultMessage,
|
|
StreamEvent,
|
|
TextBlock,
|
|
ToolUseBlock,
|
|
)
|
|
from loguru import logger
|
|
|
|
from pipecat.frames.frames import (
|
|
CancelFrame,
|
|
EndFrame,
|
|
Frame,
|
|
InterruptionFrame,
|
|
LLMContextFrame,
|
|
LLMFullResponseEndFrame,
|
|
LLMFullResponseStartFrame,
|
|
LLMTextFrame,
|
|
StartFrame,
|
|
TTSSpeakFrame,
|
|
)
|
|
from pipecat.processors.aggregators.llm_context import LLMContext
|
|
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
|
|
|
# Whisper hallucinates these on silence or breath noise. Treating them as speech
|
|
# makes the agent answer questions nobody asked.
|
|
_NOISE_TRANSCRIPTS = {
|
|
"",
|
|
".",
|
|
"thank you.",
|
|
"thanks for watching!",
|
|
"you",
|
|
"bye.",
|
|
"okay.",
|
|
"[blank_audio]",
|
|
"[silence]",
|
|
}
|
|
|
|
|
|
class ClaudeCodeLLM(FrameProcessor):
|
|
"""Runs each user utterance through a persistent Claude Code session."""
|
|
|
|
def __init__(
|
|
self, *, options: ClaudeAgentOptions, observer=None, working_phrase: str | None = "One moment.", **kwargs
|
|
):
|
|
super().__init__(**kwargs)
|
|
# Said once per turn if a tool runs before any answer has begun. Silence
|
|
# while Claude works reads as a crash: in one recorded session four of
|
|
# nine turns went unanswered because tool-heavy turns took over a minute
|
|
# with no sound, and speaking again to check cancels the turn in flight.
|
|
self._working_phrase = working_phrase
|
|
self._said_working = False
|
|
self._options = options
|
|
self._client: ClaudeSDKClient | None = None
|
|
self._turn_task: asyncio.Task | None = None
|
|
# Called with each completed reply. What Claude is talking about
|
|
# predicts what the user will say next, so this feeds the recognizer.
|
|
#
|
|
# Not named _observer: FrameProcessor owns that attribute and reassigns
|
|
# it during pipeline setup, so it would be silently replaced by
|
|
# pipecat's own observer and then called with the wrong signature.
|
|
self._on_reply = observer
|
|
|
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
|
await super().process_frame(frame, direction)
|
|
|
|
if isinstance(frame, StartFrame):
|
|
await self.push_frame(frame, direction)
|
|
await self._connect()
|
|
elif isinstance(frame, (EndFrame, CancelFrame)):
|
|
await self._cancel_turn()
|
|
await self._disconnect()
|
|
await self.push_frame(frame, direction)
|
|
elif isinstance(frame, InterruptionFrame):
|
|
await self._cancel_turn()
|
|
await self.push_frame(frame, direction)
|
|
elif isinstance(frame, LLMContextFrame):
|
|
await self._maybe_start_turn(_latest_user_text(frame.context))
|
|
else:
|
|
await self.push_frame(frame, direction)
|
|
|
|
def start_turn_direct(self, text: str):
|
|
utterance = text.strip()
|
|
if not utterance:
|
|
return
|
|
self.create_task(self._maybe_start_turn(utterance))
|
|
|
|
async def _connect(self):
|
|
if self._client:
|
|
return
|
|
logger.info("Starting Claude Code session...")
|
|
client = ClaudeSDKClient(options=self._options)
|
|
try:
|
|
await client.connect()
|
|
except Exception as e:
|
|
self._client = None
|
|
logger.error(
|
|
f"Could not start the Claude CLI: {e}\n"
|
|
"Exit code -9 means the CLI was killed rather than failing on its own. "
|
|
"On a managed Mac that is usually either the binary still awaiting "
|
|
"approval, or its sandbox failing to apply because this process is "
|
|
"already inside one — in which case run `./talk` from a normal terminal."
|
|
)
|
|
return
|
|
self._client = client
|
|
logger.info("Claude Code session ready.")
|
|
|
|
async def _disconnect(self):
|
|
if not self._client:
|
|
return
|
|
await self._client.disconnect()
|
|
self._client = None
|
|
|
|
async def _maybe_start_turn(self, text: str):
|
|
utterance = text.strip()
|
|
if utterance.lower() in _NOISE_TRANSCRIPTS or len(utterance) < 2:
|
|
logger.debug(f"Ignoring noise transcript: {utterance!r}")
|
|
return
|
|
if not self._client:
|
|
logger.warning("Transcript arrived before the Claude session was ready; dropping it.")
|
|
return
|
|
|
|
# A new utterance supersedes whatever Claude was in the middle of saying.
|
|
await self._cancel_turn()
|
|
|
|
logger.info(f"You: {utterance}")
|
|
self._turn_task = self.create_task(self._run_turn(utterance))
|
|
|
|
async def _cancel_turn(self):
|
|
if not self._turn_task:
|
|
return
|
|
task, self._turn_task = self._turn_task, None
|
|
if self._client:
|
|
try:
|
|
await self._client.interrupt()
|
|
except Exception as e:
|
|
logger.debug(f"Interrupt failed (session may be idle): {e}")
|
|
await self.cancel_task(task)
|
|
|
|
async def _say_working(self, spoken: list[str]):
|
|
"""Break the silence before a slow tool call, once per turn."""
|
|
if self._said_working or spoken or not self._working_phrase:
|
|
return
|
|
self._said_working = True
|
|
await self.push_frame(TTSSpeakFrame(self._working_phrase))
|
|
|
|
async def _run_turn(self, utterance: str):
|
|
await self.push_frame(LLMFullResponseStartFrame())
|
|
self._said_working = False
|
|
spoken: list[str] = []
|
|
try:
|
|
await self._client.query(utterance)
|
|
async for message in self._client.receive_response():
|
|
# Subagent chatter carries a parent tool id; only speak the main thread.
|
|
if getattr(message, "parent_tool_use_id", None):
|
|
continue
|
|
|
|
if isinstance(message, StreamEvent):
|
|
text = _text_delta(message)
|
|
if text:
|
|
spoken.append(text)
|
|
await self.push_frame(LLMTextFrame(text))
|
|
elif isinstance(message, AssistantMessage):
|
|
for block in message.content:
|
|
if isinstance(block, ToolUseBlock):
|
|
logger.info(f" [tool] {block.name}")
|
|
try:
|
|
import web_server
|
|
web_server.broadcast_event("tool", {"name": block.name, "detail": str(getattr(block, "input", ""))})
|
|
except Exception:
|
|
pass
|
|
await self._say_working(spoken)
|
|
elif isinstance(block, TextBlock) and not self._streams_partials():
|
|
spoken.append(block.text)
|
|
await self.push_frame(LLMTextFrame(block.text))
|
|
elif isinstance(message, ResultMessage):
|
|
if message.is_error:
|
|
_log_result_error(message)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as e:
|
|
logger.exception(f"Claude turn failed: {e}")
|
|
finally:
|
|
if spoken:
|
|
reply = "".join(spoken)
|
|
logger.info(f"Claude: {reply}")
|
|
if self._on_reply:
|
|
# Isolated deliberately: this frame ends the turn for the
|
|
# TTS and the aggregator, so nothing optional may prevent
|
|
# it being pushed.
|
|
try:
|
|
self._on_reply(reply)
|
|
except Exception as e:
|
|
logger.warning(f"Reply observer failed: {e}")
|
|
await self.push_frame(LLMFullResponseEndFrame())
|
|
|
|
def _streams_partials(self) -> bool:
|
|
return self._options.include_partial_messages
|
|
|
|
|
|
def _log_result_error(message: ResultMessage):
|
|
"""Report a failed turn using whichever field actually says something.
|
|
|
|
``result`` is often None on a failure — it holds the reply text, and a turn
|
|
that failed has none — so logging it alone produced "Claude returned an
|
|
error: None" and told us nothing. The diagnosis lives in the other fields.
|
|
"""
|
|
details = {
|
|
field: value
|
|
for field in (
|
|
"subtype",
|
|
"stop_reason",
|
|
"terminal_reason",
|
|
"api_error_status",
|
|
"errors",
|
|
"permission_denials",
|
|
"result",
|
|
)
|
|
if (value := getattr(message, field, None))
|
|
}
|
|
|
|
# Cutting Claude off mid-answer is a normal part of talking, and the CLI
|
|
# reports it the same way it reports a genuine failure. Don't cry wolf.
|
|
if details.get("terminal_reason") in (
|
|
"interrupted",
|
|
"cancelled",
|
|
# What the CLI reports when interrupt() lands mid-stream, which is
|
|
# simply what talking over Claude looks like from its side.
|
|
"aborted_streaming",
|
|
):
|
|
logger.debug(f"Turn interrupted: {details}")
|
|
return
|
|
|
|
logger.error(f"Claude turn failed: {details or 'no detail reported'}")
|
|
|
|
|
|
def _latest_user_text(context: LLMContext) -> str:
|
|
"""Return the text of the most recent user message in the context."""
|
|
for message in reversed(context.get_messages()):
|
|
if not isinstance(message, dict) or message.get("role") != "user":
|
|
continue
|
|
content = message.get("content")
|
|
if isinstance(content, str):
|
|
return content
|
|
if isinstance(content, list):
|
|
return " ".join(
|
|
part.get("text", "")
|
|
for part in content
|
|
if isinstance(part, dict) and part.get("type") == "text"
|
|
)
|
|
return ""
|
|
|
|
|
|
def _text_delta(event: StreamEvent) -> str | None:
|
|
"""Pull assistant text out of a raw stream event, ignoring thinking and tool input."""
|
|
raw = event.event or {}
|
|
if raw.get("type") != "content_block_delta":
|
|
return None
|
|
delta = raw.get("delta") or {}
|
|
if delta.get("type") != "text_delta":
|
|
return None
|
|
return delta.get("text") or None
|