Initial commit of current state
This commit is contained in:
Adolfo Reyna
+267
@@ -0,0 +1,267 @@
|
||||
"""A Pipecat processor that puts Claude Code in the LLM slot of a voice pipeline.
|
||||
|
||||
This drives the Claude Agent SDK rather than the raw Anthropic API, so the
|
||||
conversation runs through the same `claude` CLI (and the same auth) that a
|
||||
terminal session uses, and Claude keeps its tools.
|
||||
|
||||
The SDK owns conversation history, so the Pipecat context is used only for turn
|
||||
detection: this reads the newest user message off each `LLMContextFrame` and
|
||||
emits the `LLMFullResponseStartFrame` / `LLMTextFrame` /
|
||||
`LLMFullResponseEndFrame` sequence the TTS service downstream expects.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
from claude_agent_sdk import (
|
||||
AssistantMessage,
|
||||
ClaudeAgentOptions,
|
||||
ClaudeSDKClient,
|
||||
ResultMessage,
|
||||
StreamEvent,
|
||||
TextBlock,
|
||||
ToolUseBlock,
|
||||
)
|
||||
from loguru import logger
|
||||
|
||||
from pipecat.frames.frames import (
|
||||
CancelFrame,
|
||||
EndFrame,
|
||||
Frame,
|
||||
InterruptionFrame,
|
||||
LLMContextFrame,
|
||||
LLMFullResponseEndFrame,
|
||||
LLMFullResponseStartFrame,
|
||||
LLMTextFrame,
|
||||
StartFrame,
|
||||
TTSSpeakFrame,
|
||||
)
|
||||
from pipecat.processors.aggregators.llm_context import LLMContext
|
||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||
|
||||
# Whisper hallucinates these on silence or breath noise. Treating them as speech
|
||||
# makes the agent answer questions nobody asked.
|
||||
_NOISE_TRANSCRIPTS = {
|
||||
"",
|
||||
".",
|
||||
"thank you.",
|
||||
"thanks for watching!",
|
||||
"you",
|
||||
"bye.",
|
||||
"okay.",
|
||||
"[blank_audio]",
|
||||
"[silence]",
|
||||
}
|
||||
|
||||
|
||||
class ClaudeCodeLLM(FrameProcessor):
|
||||
"""Runs each user utterance through a persistent Claude Code session."""
|
||||
|
||||
def __init__(
|
||||
self, *, options: ClaudeAgentOptions, observer=None, working_phrase: str | None = "One moment.", **kwargs
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
# Said once per turn if a tool runs before any answer has begun. Silence
|
||||
# while Claude works reads as a crash: in one recorded session four of
|
||||
# nine turns went unanswered because tool-heavy turns took over a minute
|
||||
# with no sound, and speaking again to check cancels the turn in flight.
|
||||
self._working_phrase = working_phrase
|
||||
self._said_working = False
|
||||
self._options = options
|
||||
self._client: ClaudeSDKClient | None = None
|
||||
self._turn_task: asyncio.Task | None = None
|
||||
# Called with each completed reply. What Claude is talking about
|
||||
# predicts what the user will say next, so this feeds the recognizer.
|
||||
#
|
||||
# Not named _observer: FrameProcessor owns that attribute and reassigns
|
||||
# it during pipeline setup, so it would be silently replaced by
|
||||
# pipecat's own observer and then called with the wrong signature.
|
||||
self._on_reply = observer
|
||||
|
||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||
await super().process_frame(frame, direction)
|
||||
|
||||
if isinstance(frame, StartFrame):
|
||||
await self.push_frame(frame, direction)
|
||||
await self._connect()
|
||||
elif isinstance(frame, (EndFrame, CancelFrame)):
|
||||
await self._cancel_turn()
|
||||
await self._disconnect()
|
||||
await self.push_frame(frame, direction)
|
||||
elif isinstance(frame, InterruptionFrame):
|
||||
await self._cancel_turn()
|
||||
await self.push_frame(frame, direction)
|
||||
elif isinstance(frame, LLMContextFrame):
|
||||
await self._maybe_start_turn(_latest_user_text(frame.context))
|
||||
else:
|
||||
await self.push_frame(frame, direction)
|
||||
|
||||
async def _connect(self):
|
||||
if self._client:
|
||||
return
|
||||
logger.info("Starting Claude Code session...")
|
||||
client = ClaudeSDKClient(options=self._options)
|
||||
try:
|
||||
await client.connect()
|
||||
except Exception as e:
|
||||
self._client = None
|
||||
logger.error(
|
||||
f"Could not start the Claude CLI: {e}\n"
|
||||
"Exit code -9 means the CLI was killed rather than failing on its own. "
|
||||
"On a managed Mac that is usually either the binary still awaiting "
|
||||
"approval, or its sandbox failing to apply because this process is "
|
||||
"already inside one — in which case run `./talk` from a normal terminal."
|
||||
)
|
||||
return
|
||||
self._client = client
|
||||
logger.info("Claude Code session ready.")
|
||||
|
||||
async def _disconnect(self):
|
||||
if not self._client:
|
||||
return
|
||||
await self._client.disconnect()
|
||||
self._client = None
|
||||
|
||||
async def _maybe_start_turn(self, text: str):
|
||||
utterance = text.strip()
|
||||
if utterance.lower() in _NOISE_TRANSCRIPTS or len(utterance) < 2:
|
||||
logger.debug(f"Ignoring noise transcript: {utterance!r}")
|
||||
return
|
||||
if not self._client:
|
||||
logger.warning("Transcript arrived before the Claude session was ready; dropping it.")
|
||||
return
|
||||
|
||||
# A new utterance supersedes whatever Claude was in the middle of saying.
|
||||
await self._cancel_turn()
|
||||
|
||||
logger.info(f"You: {utterance}")
|
||||
self._turn_task = self.create_task(self._run_turn(utterance))
|
||||
|
||||
async def _cancel_turn(self):
|
||||
if not self._turn_task:
|
||||
return
|
||||
task, self._turn_task = self._turn_task, None
|
||||
if self._client:
|
||||
try:
|
||||
await self._client.interrupt()
|
||||
except Exception as e:
|
||||
logger.debug(f"Interrupt failed (session may be idle): {e}")
|
||||
await self.cancel_task(task)
|
||||
|
||||
async def _say_working(self, spoken: list[str]):
|
||||
"""Break the silence before a slow tool call, once per turn."""
|
||||
if self._said_working or spoken or not self._working_phrase:
|
||||
return
|
||||
self._said_working = True
|
||||
await self.push_frame(TTSSpeakFrame(self._working_phrase))
|
||||
|
||||
async def _run_turn(self, utterance: str):
|
||||
await self.push_frame(LLMFullResponseStartFrame())
|
||||
self._said_working = False
|
||||
spoken: list[str] = []
|
||||
try:
|
||||
await self._client.query(utterance)
|
||||
async for message in self._client.receive_response():
|
||||
# Subagent chatter carries a parent tool id; only speak the main thread.
|
||||
if getattr(message, "parent_tool_use_id", None):
|
||||
continue
|
||||
|
||||
if isinstance(message, StreamEvent):
|
||||
text = _text_delta(message)
|
||||
if text:
|
||||
spoken.append(text)
|
||||
await self.push_frame(LLMTextFrame(text))
|
||||
elif isinstance(message, AssistantMessage):
|
||||
for block in message.content:
|
||||
if isinstance(block, ToolUseBlock):
|
||||
logger.info(f" [tool] {block.name}")
|
||||
await self._say_working(spoken)
|
||||
elif isinstance(block, TextBlock) and not self._streams_partials():
|
||||
spoken.append(block.text)
|
||||
await self.push_frame(LLMTextFrame(block.text))
|
||||
elif isinstance(message, ResultMessage):
|
||||
if message.is_error:
|
||||
_log_result_error(message)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception(f"Claude turn failed: {e}")
|
||||
finally:
|
||||
if spoken:
|
||||
reply = "".join(spoken)
|
||||
logger.info(f"Claude: {reply}")
|
||||
if self._on_reply:
|
||||
# Isolated deliberately: this frame ends the turn for the
|
||||
# TTS and the aggregator, so nothing optional may prevent
|
||||
# it being pushed.
|
||||
try:
|
||||
self._on_reply(reply)
|
||||
except Exception as e:
|
||||
logger.warning(f"Reply observer failed: {e}")
|
||||
await self.push_frame(LLMFullResponseEndFrame())
|
||||
|
||||
def _streams_partials(self) -> bool:
|
||||
return self._options.include_partial_messages
|
||||
|
||||
|
||||
def _log_result_error(message: ResultMessage):
|
||||
"""Report a failed turn using whichever field actually says something.
|
||||
|
||||
``result`` is often None on a failure — it holds the reply text, and a turn
|
||||
that failed has none — so logging it alone produced "Claude returned an
|
||||
error: None" and told us nothing. The diagnosis lives in the other fields.
|
||||
"""
|
||||
details = {
|
||||
field: value
|
||||
for field in (
|
||||
"subtype",
|
||||
"stop_reason",
|
||||
"terminal_reason",
|
||||
"api_error_status",
|
||||
"errors",
|
||||
"permission_denials",
|
||||
"result",
|
||||
)
|
||||
if (value := getattr(message, field, None))
|
||||
}
|
||||
|
||||
# Cutting Claude off mid-answer is a normal part of talking, and the CLI
|
||||
# reports it the same way it reports a genuine failure. Don't cry wolf.
|
||||
if details.get("terminal_reason") in (
|
||||
"interrupted",
|
||||
"cancelled",
|
||||
# What the CLI reports when interrupt() lands mid-stream, which is
|
||||
# simply what talking over Claude looks like from its side.
|
||||
"aborted_streaming",
|
||||
):
|
||||
logger.debug(f"Turn interrupted: {details}")
|
||||
return
|
||||
|
||||
logger.error(f"Claude turn failed: {details or 'no detail reported'}")
|
||||
|
||||
|
||||
def _latest_user_text(context: LLMContext) -> str:
|
||||
"""Return the text of the most recent user message in the context."""
|
||||
for message in reversed(context.get_messages()):
|
||||
if not isinstance(message, dict) or message.get("role") != "user":
|
||||
continue
|
||||
content = message.get("content")
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
return " ".join(
|
||||
part.get("text", "")
|
||||
for part in content
|
||||
if isinstance(part, dict) and part.get("type") == "text"
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
def _text_delta(event: StreamEvent) -> str | None:
|
||||
"""Pull assistant text out of a raw stream event, ignoring thinking and tool input."""
|
||||
raw = event.event or {}
|
||||
if raw.get("type") != "content_block_delta":
|
||||
return None
|
||||
delta = raw.get("delta") or {}
|
||||
if delta.get("type") != "text_delta":
|
||||
return None
|
||||
return delta.get("text") or None
|
||||
Reference in New Issue
Block a user