From 598d55e3839ffd923d19a97f69603a5fb33eda59 Mon Sep 17 00:00:00 2001 From: Adolfo Reyna Date: Fri, 7 Aug 2026 18:23:38 -0400 Subject: [PATCH] Add macOS LLM model engine support and environment setup --- .gitignore | 8 ++ README.md | 2 + apple_llm.py | 229 ++++++++++++++++++++++++++++++++++++++++++ bot.py | 28 +++++- selftest.py | 14 ++- swift/LLMHelper.swift | 91 +++++++++++++++++ swift/build.sh | 17 +++- 7 files changed, 378 insertions(+), 11 deletions(-) create mode 100644 .gitignore create mode 100644 apple_llm.py create mode 100644 swift/LLMHelper.swift diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6fe90bf --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.venv/ +__pycache__/ +*.pyc +scratch/ +swift/speech-helper +swift/llm-helper +swift/test_foundation +swift/test_foundation.swift diff --git a/README.md b/README.md index 7da6b24..f080932 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ when the hold key is working. Useful flags: ```bash +./talk --llm-engine apple # macOS on-device LLM model (default) +./talk --llm-engine claude # Claude Code CLI engine ./talk --list-devices # see microphones and speakers ./talk --list-voices # see macOS system voices ./talk --input-device "Adolfo i16" # pick a device by index or name substring diff --git a/apple_llm.py b/apple_llm.py new file mode 100644 index 0000000..e02342f --- /dev/null +++ b/apple_llm.py @@ -0,0 +1,229 @@ +"""A Pipecat processor that puts the native macOS / Apple Silicon LLM model in the LLM slot. + +Supports: +1. Apple Intelligence / FoundationModels (via swift/llm-helper) +2. MLX local on-device LLM on Apple Silicon (via mlx-lm) + +Streams response text frames (LLMFullResponseStartFrame, LLMTextFrame, +LLMFullResponseEndFrame) so text-to-speech downstream responds with low latency. +""" + +import asyncio +import json +import os +import shutil +import subprocess +from pathlib import Path +from loguru import logger + +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + Frame, + InterruptionFrame, + LLMContextFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMTextFrame, + StartFrame, +) +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor + +_NOISE_TRANSCRIPTS = { + "", + ".", + "thank you.", + "thanks for watching!", + "you", + "bye.", + "okay.", + "[blank_audio]", + "[silence]", +} + +HERE = Path(__file__).parent +LLM_HELPER_PATH = HERE / "swift" / "llm-helper" + + +def probe_apple_llm() -> tuple[bool, str]: + """Check if native macOS / Apple Silicon LLM options are available.""" + if LLM_HELPER_PATH.exists() and os.access(LLM_HELPER_PATH, os.X_OK): + try: + res = subprocess.run( + [str(LLM_HELPER_PATH), "--check"], + capture_output=True, + text=True, + timeout=5.0, + ) + if res.returncode == 0: + try: + payload = json.loads(res.stdout.strip()) + if payload.get("available"): + return True, payload.get("reason", "macOS FoundationModels available") + except json.JSONDecodeError: + pass + except Exception: + pass + + try: + import mlx_lm + return True, "Apple Silicon MLX local LLM engine available" + except ImportError: + return False, "Neither swift/llm-helper nor mlx-lm is available" + + +def _latest_user_text(context) -> str: + if not context or not hasattr(context, "messages"): + return "" + for msg in reversed(context.messages): + if isinstance(msg, dict) and msg.get("role") == "user": + content = msg.get("content", "") + if isinstance(content, str): + return content + elif isinstance(content, list): + text_parts = [c.get("text", "") for c in content if isinstance(c, dict)] + return " ".join(text_parts) + return "" + + +class MacOSLLM(FrameProcessor): + """Runs user turns through macOS native LLM (Apple Intelligence or MLX).""" + + def __init__( + self, + *, + model: str = "mlx-community/Qwen2.5-0.5B-Instruct-4bit", + system_prompt: str | None = None, + observer=None, + **kwargs, + ): + super().__init__(**kwargs) + self._model_name = model + self._system_prompt = system_prompt or "You are a helpful voice assistant. Keep answers brief and spoken naturally." + self._on_reply = observer + self._turn_task: asyncio.Task | None = None + self._history: list[dict[str, str]] = [] + self._mlx_model = None + self._mlx_tokenizer = None + self._use_swift = False + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, StartFrame): + await self.push_frame(frame, direction) + await self._connect() + elif isinstance(frame, (EndFrame, CancelFrame)): + await self._cancel_turn() + await self.push_frame(frame, direction) + elif isinstance(frame, InterruptionFrame): + await self._cancel_turn() + await self.push_frame(frame, direction) + elif isinstance(frame, LLMContextFrame): + text = _latest_user_text(frame.context) + await self._maybe_start_turn(text) + else: + await self.push_frame(frame, direction) + + async def _connect(self): + available, reason = probe_apple_llm() + logger.info(f"macOS LLM engine: {reason}") + if LLM_HELPER_PATH.exists() and "FoundationModels available" in reason: + self._use_swift = True + logger.info("Using Swift FoundationModels engine.") + else: + self._use_swift = False + logger.info(f"Loading MLX model {self._model_name} on Apple Silicon...") + loop = asyncio.get_running_loop() + def _load(): + from mlx_lm import load + return load(self._model_name) + self._mlx_model, self._mlx_tokenizer = await loop.run_in_executor(None, _load) + logger.info("MLX model loaded.") + + async def _maybe_start_turn(self, text: str): + utterance = text.strip() + if utterance.lower() in _NOISE_TRANSCRIPTS or len(utterance) < 2: + logger.debug(f"Ignoring noise transcript: {utterance!r}") + return + + await self._cancel_turn() + logger.info(f"You: {utterance}") + self._turn_task = self.create_task(self._run_turn(utterance)) + + async def _cancel_turn(self): + if not self._turn_task: + return + task, self._turn_task = self._turn_task, None + await self.cancel_task(task) + + async def _run_turn(self, utterance: str): + self._history.append({"role": "user", "content": utterance}) + + await self.push_frame(LLMFullResponseStartFrame()) + chunks: list[str] = [] + + try: + if self._use_swift: + proc = await asyncio.create_subprocess_exec( + str(LLM_HELPER_PATH), + "--prompt", utterance, + "--system", self._system_prompt, + "--stream", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + while True: + line = await proc.stdout.readline() + if not line: + break + try: + data = json.loads(line.decode().strip()) + if "delta" in data: + chunk = data["delta"] + chunks.append(chunk) + await self.push_frame(LLMTextFrame(chunk)) + elif "text" in data and not chunks: + chunk = data["text"] + chunks.append(chunk) + await self.push_frame(LLMTextFrame(chunk)) + except json.JSONDecodeError: + pass + await proc.wait() + else: + loop = asyncio.get_running_loop() + def _gen(): + from mlx_lm import generate + messages = [{"role": "system", "content": self._system_prompt}] + self._history + prompt = self._mlx_tokenizer.apply_chat_template( + messages, add_generation_prompt=True, tokenize=False + ) + return generate( + self._mlx_model, + self._mlx_tokenizer, + prompt=prompt, + max_tokens=256, + verbose=False, + ) + + response_text = await loop.run_in_executor(None, _gen) + chunks.append(response_text) + await self.push_frame(LLMTextFrame(response_text)) + + except asyncio.CancelledError: + logger.info("Turn cancelled mid-response.") + raise + except Exception as e: + logger.error(f"macOS LLM error: {e}") + err_msg = "Sorry, I ran into an error generating a response." + chunks.append(err_msg) + await self.push_frame(LLMTextFrame(err_msg)) + finally: + await self.push_frame(LLMFullResponseEndFrame()) + + full_reply = "".join(chunks).strip() + if full_reply: + self._history.append({"role": "assistant", "content": full_reply}) + logger.info(f"macOS LLM: {full_reply}") + if self._on_reply: + self._on_reply(full_reply) diff --git a/bot.py b/bot.py index 03cd4db..a21e12a 100644 --- a/bot.py +++ b/bot.py @@ -151,6 +151,12 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--list-voices", action="store_true", help="Print macOS system voices and exit." ) + parser.add_argument( + "--llm-engine", + choices=["apple", "macos", "claude"], + default="apple", + help="LLM engine to use: apple/macos for native on-device macOS LLM, claude for Claude Code.", + ) parser.add_argument( "--claude-model", default=DEFAULT_CLAUDE_MODEL, @@ -475,6 +481,23 @@ def build_claude_options(args: argparse.Namespace, vocabulary=None, brain=None) ) +def build_llm(args: argparse.Namespace, vocabulary=None, brain=None, observer=None): + if args.llm_engine in ("apple", "macos"): + from apple_llm import MacOSLLM, probe_apple_llm + + available, reason = probe_apple_llm() + logger.info(f"LLM: macOS native model ({reason})") + personality = read_personality(args.cwd) or "You are a helpful macOS voice assistant." + system_prompt = personality + "\n\n" + VOICE_STYLE + return MacOSLLM(system_prompt=system_prompt, observer=observer) + + logger.info("LLM: Claude Code") + return ClaudeCodeLLM( + options=build_claude_options(args, vocabulary, brain), + observer=observer, + ) + + async def main() -> int: args = parse_args() @@ -523,10 +546,7 @@ async def main() -> int: if vocabulary: vocabulary.observe(text) - llm = ClaudeCodeLLM( - options=build_claude_options(args, vocabulary, brain), - observer=on_reply, - ) + llm = build_llm(args, vocabulary, brain, observer=on_reply) tts = build_tts(args) diff --git a/selftest.py b/selftest.py index 3076903..5dee9f7 100755 --- a/selftest.py +++ b/selftest.py @@ -171,6 +171,13 @@ def check_mlx_whisper(synthesized): record("mlx whisper (optional)", matches(heard), f"heard {heard!r}") +def check_macos_llm(): + from apple_llm import probe_apple_llm + + available, reason = probe_apple_llm() + record("macos llm", available, reason) + + async def check_claude(): from claude_agent_sdk import ClaudeSDKClient, ResultMessage, StreamEvent @@ -199,16 +206,16 @@ async def check_claude(): if text: spoken.append(text) elif isinstance(message, ResultMessage) and message.is_error: - record("claude session", False, f"error: {message.result}") + record("claude session (optional)", False, f"error: {message.result}") return except Exception as e: hint = "" if "-9" in str(e): hint = " — the CLI was killed applying its own sandbox; run this from a normal terminal" - record("claude session", False, f"{e}{hint}") + record("claude session (optional)", False, f"{e}{hint}") return record( - "claude session", + "claude session (optional)", True, f"replied {''.join(spoken).strip()!r} using {DEFAULT_CLAUDE_MODEL}", ) @@ -223,6 +230,7 @@ async def main(): synthesized = check_kokoro() check_apple_stt(synthesized) check_mlx_whisper(synthesized) + check_macos_llm() await check_claude() required_failures = [ diff --git a/swift/LLMHelper.swift b/swift/LLMHelper.swift new file mode 100644 index 0000000..490749c --- /dev/null +++ b/swift/LLMHelper.swift @@ -0,0 +1,91 @@ +// LLMHelper.swift - Interface with macOS native on-device LLM (FoundationModels / Apple Intelligence) +// +// Usage: +// llm-helper --check +// llm-helper --prompt "Hello" [--system "System prompt"] + +import Foundation +import FoundationModels + +struct Options { + var prompt: String = "" + var systemPrompt: String? = nil + var check: Bool = false +} + +func parseArguments() -> Options { + var options = Options() + var arguments = Array(CommandLine.arguments.dropFirst()) + while let argument = arguments.first { + arguments.removeFirst() + switch argument { + case "--prompt": + if !arguments.isEmpty { options.prompt = arguments.removeFirst() } + case "--system": + if !arguments.isEmpty { options.systemPrompt = arguments.removeFirst() } + case "--check": + options.check = true + default: + if options.prompt.isEmpty && !argument.hasPrefix("--") { + options.prompt = argument + } + } + } + return options +} + +func emit(_ payload: [String: Any]) { + if let data = try? JSONSerialization.data(withJSONObject: payload, options: [.sortedKeys]), + let str = String(data: data, encoding: .utf8) { + print(str) + fflush(stdout) + } +} + +func fail(_ message: String) -> Never { + emit(["error": message]) + exit(1) +} + +@main +struct LLMHelperApp { + static func main() async { + let options = parseArguments() + + if options.check { + let model = SystemLanguageModel.default + let isAvailable = model.isAvailable + emit([ + "available": isAvailable, + "model": "macOS SystemLanguageModel", + "reason": isAvailable ? "macOS Apple Intelligence model available" : "SystemLanguageModel unavailable on this hardware/OS configuration" + ]) + exit(0) + } + + guard !options.prompt.isEmpty else { + fail("no prompt provided") + } + + do { + let model = SystemLanguageModel.default + guard model.isAvailable else { + fail("macOS SystemLanguageModel is not available on this machine") + } + + let instructions = options.systemPrompt ?? "" + let session: LanguageModelSession + if !instructions.isEmpty { + session = LanguageModelSession(model: model, instructions: instructions) + } else { + session = LanguageModelSession(model: model) + } + + let response = try await session.respond(to: options.prompt) + let resultText = String(describing: response) + emit(["done": true, "text": resultText]) + } catch { + fail("LanguageModel error: \(error.localizedDescription)") + } + } +} diff --git a/swift/build.sh b/swift/build.sh index e4ca582..09ece11 100755 --- a/swift/build.sh +++ b/swift/build.sh @@ -13,14 +13,23 @@ swiftc -O -parse-as-library SpeechHelper.swift -o speech-helper echo "built $HERE/speech-helper" if ./speech-helper --check >/dev/null 2>&1; then - echo "it runs — ./speech-helper --check for details" + echo "speech-helper runs — ./speech-helper --check for details" else status=$? if [ "$status" -eq 137 ]; then - echo "built, but killed on launch (137): request binary approval for" + echo "built speech-helper, but killed on launch (137): request binary approval for" echo " $HERE/speech-helper" - echo "then wait a few minutes for it to sync." else - echo "built, but exited $status — run ./speech-helper --check to see why" + echo "built speech-helper, but exited $status — run ./speech-helper --check to see why" fi fi + +if swiftc -O LLMHelper.swift -o llm-helper 2>/dev/null; then + echo "built $HERE/llm-helper" + if ./llm-helper --check >/dev/null 2>&1; then + echo "llm-helper runs — ./llm-helper --check for details" + fi +else + echo "could not build llm-helper with FoundationModels; fallback to Python MLX/PyObjC bridge will be available" +fi +