From 4773b9377925df049d75f037e7328ffb7c38e5d7 Mon Sep 17 00:00:00 2001 From: Adolfo Reyna Date: Fri, 7 Aug 2026 18:57:11 -0400 Subject: [PATCH] Add OpenCode / Ollama LLM integration with gemma4:31b default --- README.md | 5 +- bot.py | 33 +++++++- ollama_llm.py | 213 ++++++++++++++++++++++++++++++++++++++++++++++++++ selftest.py | 10 ++- 4 files changed, 254 insertions(+), 7 deletions(-) create mode 100644 ollama_llm.py diff --git a/README.md b/README.md index 8fc9234..d29917a 100644 --- a/README.md +++ b/README.md @@ -34,8 +34,9 @@ when the hold key is working. Useful flags: ```bash -./talk --llm-engine apple # macOS on-device LLM model (default) -./talk --mlx-model mlx-community/gemma-2-2b-it-4bit # specify any local MLX model +./talk --llm-engine ollama # Ollama / OpenCode LLM engine (default) +./talk --ollama-model gemma4:31b # specify model (e.g. gemma4:31b) +./talk --llm-engine apple # local Apple Silicon MLX model ./talk --llm-engine claude # Claude Code CLI engine ./talk --list-devices # see microphones and speakers ./talk --list-voices # see macOS system voices diff --git a/bot.py b/bot.py index 41d311e..1f220fe 100644 --- a/bot.py +++ b/bot.py @@ -153,14 +153,23 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument( "--llm-engine", - choices=["apple", "macos", "claude"], - default="apple", - help="LLM engine to use: apple/macos for native on-device macOS LLM, claude for Claude Code.", + choices=["ollama", "opencode", "apple", "macos", "claude"], + default="ollama", + help="LLM engine to use: ollama/opencode for Ollama/Cloud models, apple/macos for local MLX, claude for Claude Code.", + ) + parser.add_argument( + "--ollama-model", + default="gemma4:31b", + help="Ollama / OpenCode model name (default gemma4:31b).", + ) + parser.add_argument( + "--ollama-host", + help="Ollama API host URL (defaults to OLLAMA_HOST or http://localhost:11434).", ) parser.add_argument( "--mlx-model", default="mlx-community/Qwen2.5-7B-Instruct-4bit", - help="MLX model repo or path for local macOS execution (e.g. mlx-community/Qwen2.5-7B-Instruct-4bit).", + help="MLX model repo or path for local macOS execution.", ) parser.add_argument( "--claude-model", @@ -488,6 +497,22 @@ def build_claude_options(args: argparse.Namespace, vocabulary=None, brain=None) def build_llm(args: argparse.Namespace, vocabulary=None, brain=None, observer=None): + if args.llm_engine in ("ollama", "opencode"): + from ollama_llm import OllamaLLM, probe_ollama, get_default_ollama_host + + host = args.ollama_host or get_default_ollama_host() + logger.info(f"LLM: Ollama / OpenCode ({args.ollama_model} @ {host})") + personality = read_personality(args.cwd) or "You are a helpful voice assistant." + system_prompt = personality + "\n\n" + VOICE_STYLE + if brain and (memory := brain.prompt_block()): + system_prompt += "\n\n" + memory + return OllamaLLM( + model=args.ollama_model, + host=host, + system_prompt=system_prompt, + observer=observer, + ) + if args.llm_engine in ("apple", "macos"): from apple_llm import MacOSLLM, probe_apple_llm diff --git a/ollama_llm.py b/ollama_llm.py new file mode 100644 index 0000000..e33f758 --- /dev/null +++ b/ollama_llm.py @@ -0,0 +1,213 @@ +"""A Pipecat processor that puts Ollama / OpenCode / Ollama Cloud in the LLM slot. + +Drives Ollama API (local or remote/cloud) with streaming responses so text-to-speech +downstream starts speaking immediately as tokens arrive. +""" + +import asyncio +import json +import os +import re +from pathlib import Path +import aiohttp +from loguru import logger + +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + Frame, + InterruptionFrame, + LLMContextFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMTextFrame, + StartFrame, +) +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor + +_NOISE_TRANSCRIPTS = { + "", + ".", + "thank you.", + "thanks for watching!", + "you", + "bye.", + "okay.", + "[blank_audio]", + "[silence]", +} + + +def _clean_spoken_text(text: str) -> str: + """Clean text for speech output and truncate fake turn generations.""" + if not text: + return "" + # Truncate if model hallucinates fake turn markers + for marker in ("User:", "Human:", "Assistant:", "\nUser", "\nHuman", "\nAssistant"): + if marker in text: + text = text.split(marker)[0] + # Remove markdown code blocks + text = re.sub(r"```[\s\S]*?```", "", text) + # Remove inline code ticks + text = re.sub(r"`[^`]*`", "", text) + # Remove markdown syntax characters + text = re.sub(r"[\#\*\_\~]", "", text) + # Flatten newlines into clear speech + lines = [line.strip() for line in text.splitlines() if line.strip()] + return " ".join(lines).strip() + + +def get_default_ollama_host() -> str: + host = os.environ.get("OLLAMA_HOST") or os.environ.get("OLLAMA_URL") or "http://localhost:11434" + if not host.startswith("http://") and not host.startswith("https://"): + host = f"http://{host}" + return host.rstrip("/") + + +async def probe_ollama(host: str | None = None, model: str = "gemma4:31b") -> tuple[bool, str]: + """Check if Ollama server and requested model are reachable.""" + host = host or get_default_ollama_host() + url = f"{host}/api/tags" + try: + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=4.0)) as session: + async with session.get(url) as resp: + if resp.status == 200: + data = await resp.json() + models = [m.get("name", "") for m in data.get("models", [])] + match = any(m.startswith(model.split(":")[0]) for m in models) + if match or not models: + return True, f"Ollama host {host} active with model {model}" + return True, f"Ollama host {host} active (models: {', '.join(models[:4])})" + return False, f"Ollama returned HTTP {resp.status}" + except Exception as e: + return False, f"Cannot connect to Ollama at {host}: {e}" + + +class OllamaLLM(FrameProcessor): + """Runs user turns through Ollama / OpenCode API.""" + + def __init__( + self, + *, + model: str = "gemma4:31b", + host: str | None = None, + system_prompt: str | None = None, + observer=None, + **kwargs, + ): + super().__init__(**kwargs) + self._model = model + self._host = host or get_default_ollama_host() + self._system_prompt = system_prompt or "You are a helpful spoken voice assistant. Keep answers brief and conversational." + self._on_reply = observer + self._turn_task: asyncio.Task | None = None + self._history: list[dict[str, str]] = [] + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, StartFrame): + await self.push_frame(frame, direction) + logger.info(f"Ollama LLM engine initialized: host={self._host}, model={self._model}") + elif isinstance(frame, (EndFrame, CancelFrame)): + await self._cancel_turn() + await self.push_frame(frame, direction) + elif isinstance(frame, InterruptionFrame): + await self._cancel_turn() + await self.push_frame(frame, direction) + elif isinstance(frame, LLMContextFrame): + text = self._latest_user_text(frame.context) + await self._maybe_start_turn(text) + else: + await self.push_frame(frame, direction) + + def _latest_user_text(self, context) -> str: + if not context or not hasattr(context, "messages"): + return "" + for msg in reversed(context.messages): + if isinstance(msg, dict) and msg.get("role") == "user": + content = msg.get("content", "") + if isinstance(content, str): + return content + elif isinstance(content, list): + text_parts = [c.get("text", "") for c in content if isinstance(c, dict)] + return " ".join(text_parts) + return "" + + async def _maybe_start_turn(self, text: str): + utterance = text.strip() + if utterance.lower() in _NOISE_TRANSCRIPTS or len(utterance) < 2: + logger.debug(f"Ignoring noise transcript: {utterance!r}") + return + + await self._cancel_turn() + logger.info(f"You: {utterance}") + self._turn_task = self.create_task(self._run_turn(utterance)) + + async def _cancel_turn(self): + if not self._turn_task: + return + task, self._turn_task = self._turn_task, None + await self.cancel_task(task) + + async def _run_turn(self, utterance: str): + self._history.append({"role": "user", "content": utterance}) + recent_history = self._history[-8:] + messages = [{"role": "system", "content": self._system_prompt}] + recent_history + + await self.push_frame(LLMFullResponseStartFrame()) + chunks: list[str] = [] + + url = f"{self._host}/api/chat" + payload = { + "model": self._model, + "messages": messages, + "stream": True, + "options": { + "temperature": 0.7, + }, + } + + try: + async with aiohttp.ClientSession() as session: + async with session.post(url, json=payload) as resp: + if resp.status != 200: + err_text = await resp.text() + logger.error(f"Ollama API HTTP {resp.status}: {err_text}") + err_msg = f"Sorry, Ollama API returned error {resp.status}." + chunks.append(err_msg) + await self.push_frame(LLMTextFrame(err_msg)) + else: + async for line in resp.content: + line_str = line.decode("utf-8").strip() + if not line_str: + continue + try: + data = json.loads(line_str) + msg = data.get("message", {}) + chunk = msg.get("content", "") + if chunk: + chunks.append(chunk) + cleaned_chunk = _clean_spoken_text(chunk) + if cleaned_chunk: + await self.push_frame(LLMTextFrame(cleaned_chunk)) + except json.JSONDecodeError: + pass + except asyncio.CancelledError: + logger.info("Ollama turn cancelled mid-response.") + raise + except Exception as e: + logger.error(f"Ollama LLM connection error: {e}") + err_msg = "Sorry, I ran into an error connecting to Ollama." + chunks.append(err_msg) + await self.push_frame(LLMTextFrame(err_msg)) + finally: + await self.push_frame(LLMFullResponseEndFrame()) + + full_reply = "".join(chunks).strip() + cleaned_reply = _clean_spoken_text(full_reply) + if cleaned_reply: + self._history.append({"role": "assistant", "content": cleaned_reply}) + logger.info(f"Ollama LLM ({self._model}): {cleaned_reply}") + if self._on_reply: + self._on_reply(cleaned_reply) diff --git a/selftest.py b/selftest.py index 5dee9f7..66112cf 100755 --- a/selftest.py +++ b/selftest.py @@ -171,11 +171,18 @@ def check_mlx_whisper(synthesized): record("mlx whisper (optional)", matches(heard), f"heard {heard!r}") +async def check_ollama_llm(): + from ollama_llm import probe_ollama + + available, reason = await probe_ollama(model="gemma4:31b") + record("ollama llm (optional)", available, reason) + + def check_macos_llm(): from apple_llm import probe_apple_llm available, reason = probe_apple_llm() - record("macos llm", available, reason) + record("macos llm (optional)", available, reason) async def check_claude(): @@ -231,6 +238,7 @@ async def main(): check_apple_stt(synthesized) check_mlx_whisper(synthesized) check_macos_llm() + await check_ollama_llm() await check_claude() required_failures = [