"""A Pipecat processor that puts Claude Code in the LLM slot of a voice pipeline. This drives the Claude Agent SDK rather than the raw Anthropic API, so the conversation runs through the same `claude` CLI (and the same auth) that a terminal session uses, and Claude keeps its tools. The SDK owns conversation history, so the Pipecat context is used only for turn detection: this reads the newest user message off each `LLMContextFrame` and emits the `LLMFullResponseStartFrame` / `LLMTextFrame` / `LLMFullResponseEndFrame` sequence the TTS service downstream expects. """ import asyncio from claude_agent_sdk import ( AssistantMessage, ClaudeAgentOptions, ClaudeSDKClient, ResultMessage, StreamEvent, TextBlock, ToolUseBlock, ) from loguru import logger from pipecat.frames.frames import ( CancelFrame, EndFrame, Frame, InterruptionFrame, LLMContextFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMTextFrame, StartFrame, TTSSpeakFrame, ) from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor # Whisper hallucinates these on silence or breath noise. Treating them as speech # makes the agent answer questions nobody asked. _NOISE_TRANSCRIPTS = { "", ".", "thank you.", "thanks for watching!", "you", "bye.", "okay.", "[blank_audio]", "[silence]", } class ClaudeCodeLLM(FrameProcessor): """Runs each user utterance through a persistent Claude Code session.""" def __init__( self, *, options: ClaudeAgentOptions, observer=None, working_phrase: str | None = "One moment.", **kwargs ): super().__init__(**kwargs) # Said once per turn if a tool runs before any answer has begun. Silence # while Claude works reads as a crash: in one recorded session four of # nine turns went unanswered because tool-heavy turns took over a minute # with no sound, and speaking again to check cancels the turn in flight. self._working_phrase = working_phrase self._said_working = False self._options = options self._client: ClaudeSDKClient | None = None self._turn_task: asyncio.Task | None = None # Called with each completed reply. What Claude is talking about # predicts what the user will say next, so this feeds the recognizer. # # Not named _observer: FrameProcessor owns that attribute and reassigns # it during pipeline setup, so it would be silently replaced by # pipecat's own observer and then called with the wrong signature. self._on_reply = observer async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) if isinstance(frame, StartFrame): await self.push_frame(frame, direction) await self._connect() elif isinstance(frame, (EndFrame, CancelFrame)): await self._cancel_turn() await self._disconnect() await self.push_frame(frame, direction) elif isinstance(frame, InterruptionFrame): await self._cancel_turn() await self.push_frame(frame, direction) elif isinstance(frame, LLMContextFrame): await self._maybe_start_turn(_latest_user_text(frame.context)) else: await self.push_frame(frame, direction) def start_turn_direct(self, text: str): utterance = text.strip() if not utterance: return self.create_task(self._maybe_start_turn(utterance)) async def _connect(self): if self._client: return logger.info("Starting Claude Code session...") client = ClaudeSDKClient(options=self._options) try: await client.connect() except Exception as e: self._client = None logger.error( f"Could not start the Claude CLI: {e}\n" "Exit code -9 means the CLI was killed rather than failing on its own. " "On a managed Mac that is usually either the binary still awaiting " "approval, or its sandbox failing to apply because this process is " "already inside one — in which case run `./talk` from a normal terminal." ) return self._client = client logger.info("Claude Code session ready.") async def _disconnect(self): if not self._client: return await self._client.disconnect() self._client = None async def _maybe_start_turn(self, text: str): utterance = text.strip() if utterance.lower() in _NOISE_TRANSCRIPTS or len(utterance) < 2: logger.debug(f"Ignoring noise transcript: {utterance!r}") return if not self._client: logger.warning("Transcript arrived before the Claude session was ready; dropping it.") return # A new utterance supersedes whatever Claude was in the middle of saying. await self._cancel_turn() logger.info(f"You: {utterance}") self._turn_task = self.create_task(self._run_turn(utterance)) async def _cancel_turn(self): if not self._turn_task: return task, self._turn_task = self._turn_task, None if self._client: try: await self._client.interrupt() except Exception as e: logger.debug(f"Interrupt failed (session may be idle): {e}") await self.cancel_task(task) async def _say_working(self, spoken: list[str]): """Break the silence before a slow tool call, once per turn.""" if self._said_working or spoken or not self._working_phrase: return self._said_working = True await self.push_frame(TTSSpeakFrame(self._working_phrase)) async def _run_turn(self, utterance: str): await self.push_frame(LLMFullResponseStartFrame()) self._said_working = False spoken: list[str] = [] try: await self._client.query(utterance) async for message in self._client.receive_response(): # Subagent chatter carries a parent tool id; only speak the main thread. if getattr(message, "parent_tool_use_id", None): continue if isinstance(message, StreamEvent): text = _text_delta(message) if text: spoken.append(text) await self.push_frame(LLMTextFrame(text)) elif isinstance(message, AssistantMessage): for block in message.content: if isinstance(block, ToolUseBlock): logger.info(f" [tool] {block.name}") try: import web_server web_server.broadcast_event("tool", {"name": block.name, "detail": str(getattr(block, "input", ""))}) except Exception: pass await self._say_working(spoken) elif isinstance(block, TextBlock) and not self._streams_partials(): spoken.append(block.text) await self.push_frame(LLMTextFrame(block.text)) elif isinstance(message, ResultMessage): if message.is_error: _log_result_error(message) except asyncio.CancelledError: raise except Exception as e: logger.exception(f"Claude turn failed: {e}") finally: if spoken: reply = "".join(spoken) logger.info(f"Claude: {reply}") if self._on_reply: # Isolated deliberately: this frame ends the turn for the # TTS and the aggregator, so nothing optional may prevent # it being pushed. try: self._on_reply(reply) except Exception as e: logger.warning(f"Reply observer failed: {e}") await self.push_frame(LLMFullResponseEndFrame()) def _streams_partials(self) -> bool: return self._options.include_partial_messages def _log_result_error(message: ResultMessage): """Report a failed turn using whichever field actually says something. ``result`` is often None on a failure — it holds the reply text, and a turn that failed has none — so logging it alone produced "Claude returned an error: None" and told us nothing. The diagnosis lives in the other fields. """ details = { field: value for field in ( "subtype", "stop_reason", "terminal_reason", "api_error_status", "errors", "permission_denials", "result", ) if (value := getattr(message, field, None)) } # Cutting Claude off mid-answer is a normal part of talking, and the CLI # reports it the same way it reports a genuine failure. Don't cry wolf. if details.get("terminal_reason") in ( "interrupted", "cancelled", # What the CLI reports when interrupt() lands mid-stream, which is # simply what talking over Claude looks like from its side. "aborted_streaming", ): logger.debug(f"Turn interrupted: {details}") return logger.error(f"Claude turn failed: {details or 'no detail reported'}") def _latest_user_text(context: LLMContext) -> str: """Return the text of the most recent user message in the context.""" for message in reversed(context.get_messages()): if not isinstance(message, dict) or message.get("role") != "user": continue content = message.get("content") if isinstance(content, str): return content if isinstance(content, list): return " ".join( part.get("text", "") for part in content if isinstance(part, dict) and part.get("type") == "text" ) return "" def _text_delta(event: StreamEvent) -> str | None: """Pull assistant text out of a raw stream event, ignoring thinking and tool input.""" raw = event.event or {} if raw.get("type") != "content_block_delta": return None delta = raw.get("delta") or {} if delta.get("type") != "text_delta": return None return delta.get("text") or None