Add macOS LLM model engine support and environment setup

.gitignore

@@ -0,0 +1,8 @@
+.venv/
+__pycache__/
+*.pyc
+scratch/
+swift/speech-helper
+swift/llm-helper
+swift/test_foundation
+swift/test_foundation.swift

README.md

@@ -34,6 +34,8 @@ when the hold key is working.
 Useful flags:
 
 ```bash
+./talk --llm-engine apple             # macOS on-device LLM model (default)
+./talk --llm-engine claude            # Claude Code CLI engine
 ./talk --list-devices                # see microphones and speakers
 ./talk --list-voices                 # see macOS system voices
 ./talk --input-device "Adolfo i16"   # pick a device by index or name substring

apple_llm.py

@@ -0,0 +1,229 @@
+"""A Pipecat processor that puts the native macOS / Apple Silicon LLM model in the LLM slot.
+
+Supports:
+1. Apple Intelligence / FoundationModels (via swift/llm-helper)
+2. MLX local on-device LLM on Apple Silicon (via mlx-lm)
+
+Streams response text frames (LLMFullResponseStartFrame, LLMTextFrame,
+LLMFullResponseEndFrame) so text-to-speech downstream responds with low latency.
+"""
+
+import asyncio
+import json
+import os
+import shutil
+import subprocess
+from pathlib import Path
+from loguru import logger
+
+from pipecat.frames.frames import (
+    CancelFrame,
+    EndFrame,
+    Frame,
+    InterruptionFrame,
+    LLMContextFrame,
+    LLMFullResponseEndFrame,
+    LLMFullResponseStartFrame,
+    LLMTextFrame,
+    StartFrame,
+)
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+
+_NOISE_TRANSCRIPTS = {
+    "",
+    ".",
+    "thank you.",
+    "thanks for watching!",
+    "you",
+    "bye.",
+    "okay.",
+    "[blank_audio]",
+    "[silence]",
+}
+
+HERE = Path(__file__).parent
+LLM_HELPER_PATH = HERE / "swift" / "llm-helper"
+
+
+def probe_apple_llm() -> tuple[bool, str]:
+    """Check if native macOS / Apple Silicon LLM options are available."""
+    if LLM_HELPER_PATH.exists() and os.access(LLM_HELPER_PATH, os.X_OK):
+        try:
+            res = subprocess.run(
+                [str(LLM_HELPER_PATH), "--check"],
+                capture_output=True,
+                text=True,
+                timeout=5.0,
+            )
+            if res.returncode == 0:
+                try:
+                    payload = json.loads(res.stdout.strip())
+                    if payload.get("available"):
+                        return True, payload.get("reason", "macOS FoundationModels available")
+                except json.JSONDecodeError:
+                    pass
+        except Exception:
+            pass
+
+    try:
+        import mlx_lm
+        return True, "Apple Silicon MLX local LLM engine available"
+    except ImportError:
+        return False, "Neither swift/llm-helper nor mlx-lm is available"
+
+
+def _latest_user_text(context) -> str:
+    if not context or not hasattr(context, "messages"):
+        return ""
+    for msg in reversed(context.messages):
+        if isinstance(msg, dict) and msg.get("role") == "user":
+            content = msg.get("content", "")
+            if isinstance(content, str):
+                return content
+            elif isinstance(content, list):
+                text_parts = [c.get("text", "") for c in content if isinstance(c, dict)]
+                return " ".join(text_parts)
+    return ""
+
+
+class MacOSLLM(FrameProcessor):
+    """Runs user turns through macOS native LLM (Apple Intelligence or MLX)."""
+
+    def __init__(
+        self,
+        *,
+        model: str = "mlx-community/Qwen2.5-0.5B-Instruct-4bit",
+        system_prompt: str | None = None,
+        observer=None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self._model_name = model
+        self._system_prompt = system_prompt or "You are a helpful voice assistant. Keep answers brief and spoken naturally."
+        self._on_reply = observer
+        self._turn_task: asyncio.Task | None = None
+        self._history: list[dict[str, str]] = []
+        self._mlx_model = None
+        self._mlx_tokenizer = None
+        self._use_swift = False
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+
+        if isinstance(frame, StartFrame):
+            await self.push_frame(frame, direction)
+            await self._connect()
+        elif isinstance(frame, (EndFrame, CancelFrame)):
+            await self._cancel_turn()
+            await self.push_frame(frame, direction)
+        elif isinstance(frame, InterruptionFrame):
+            await self._cancel_turn()
+            await self.push_frame(frame, direction)
+        elif isinstance(frame, LLMContextFrame):
+            text = _latest_user_text(frame.context)
+            await self._maybe_start_turn(text)
+        else:
+            await self.push_frame(frame, direction)
+
+    async def _connect(self):
+        available, reason = probe_apple_llm()
+        logger.info(f"macOS LLM engine: {reason}")
+        if LLM_HELPER_PATH.exists() and "FoundationModels available" in reason:
+            self._use_swift = True
+            logger.info("Using Swift FoundationModels engine.")
+        else:
+            self._use_swift = False
+            logger.info(f"Loading MLX model {self._model_name} on Apple Silicon...")
+            loop = asyncio.get_running_loop()
+            def _load():
+                from mlx_lm import load
+                return load(self._model_name)
+            self._mlx_model, self._mlx_tokenizer = await loop.run_in_executor(None, _load)
+            logger.info("MLX model loaded.")
+
+    async def _maybe_start_turn(self, text: str):
+        utterance = text.strip()
+        if utterance.lower() in _NOISE_TRANSCRIPTS or len(utterance) < 2:
+            logger.debug(f"Ignoring noise transcript: {utterance!r}")
+            return
+
+        await self._cancel_turn()
+        logger.info(f"You: {utterance}")
+        self._turn_task = self.create_task(self._run_turn(utterance))
+
+    async def _cancel_turn(self):
+        if not self._turn_task:
+            return
+        task, self._turn_task = self._turn_task, None
+        await self.cancel_task(task)
+
+    async def _run_turn(self, utterance: str):
+        self._history.append({"role": "user", "content": utterance})
+
+        await self.push_frame(LLMFullResponseStartFrame())
+        chunks: list[str] = []
+
+        try:
+            if self._use_swift:
+                proc = await asyncio.create_subprocess_exec(
+                    str(LLM_HELPER_PATH),
+                    "--prompt", utterance,
+                    "--system", self._system_prompt,
+                    "--stream",
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.PIPE,
+                )
+                while True:
+                    line = await proc.stdout.readline()
+                    if not line:
+                        break
+                    try:
+                        data = json.loads(line.decode().strip())
+                        if "delta" in data:
+                            chunk = data["delta"]
+                            chunks.append(chunk)
+                            await self.push_frame(LLMTextFrame(chunk))
+                        elif "text" in data and not chunks:
+                            chunk = data["text"]
+                            chunks.append(chunk)
+                            await self.push_frame(LLMTextFrame(chunk))
+                    except json.JSONDecodeError:
+                        pass
+                await proc.wait()
+            else:
+                loop = asyncio.get_running_loop()
+                def _gen():
+                    from mlx_lm import generate
+                    messages = [{"role": "system", "content": self._system_prompt}] + self._history
+                    prompt = self._mlx_tokenizer.apply_chat_template(
+                        messages, add_generation_prompt=True, tokenize=False
+                    )
+                    return generate(
+                        self._mlx_model,
+                        self._mlx_tokenizer,
+                        prompt=prompt,
+                        max_tokens=256,
+                        verbose=False,
+                    )
+
+                response_text = await loop.run_in_executor(None, _gen)
+                chunks.append(response_text)
+                await self.push_frame(LLMTextFrame(response_text))
+
+        except asyncio.CancelledError:
+            logger.info("Turn cancelled mid-response.")
+            raise
+        except Exception as e:
+            logger.error(f"macOS LLM error: {e}")
+            err_msg = "Sorry, I ran into an error generating a response."
+            chunks.append(err_msg)
+            await self.push_frame(LLMTextFrame(err_msg))
+        finally:
+            await self.push_frame(LLMFullResponseEndFrame())
+
+        full_reply = "".join(chunks).strip()
+        if full_reply:
+            self._history.append({"role": "assistant", "content": full_reply})
+            logger.info(f"macOS LLM: {full_reply}")
+            if self._on_reply:
+                self._on_reply(full_reply)

bot.py

@@ -151,6 +151,12 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "--list-voices", action="store_true", help="Print macOS system voices and exit."
     )
+    parser.add_argument(
+        "--llm-engine",
+        choices=["apple", "macos", "claude"],
+        default="apple",
+        help="LLM engine to use: apple/macos for native on-device macOS LLM, claude for Claude Code.",
+    )
     parser.add_argument(
         "--claude-model",
         default=DEFAULT_CLAUDE_MODEL,
@@ -475,6 +481,23 @@ def build_claude_options(args: argparse.Namespace, vocabulary=None, brain=None)
     )
 
 
+def build_llm(args: argparse.Namespace, vocabulary=None, brain=None, observer=None):
+    if args.llm_engine in ("apple", "macos"):
+        from apple_llm import MacOSLLM, probe_apple_llm
+
+        available, reason = probe_apple_llm()
+        logger.info(f"LLM: macOS native model ({reason})")
+        personality = read_personality(args.cwd) or "You are a helpful macOS voice assistant."
+        system_prompt = personality + "\n\n" + VOICE_STYLE
+        return MacOSLLM(system_prompt=system_prompt, observer=observer)
+
+    logger.info("LLM: Claude Code")
+    return ClaudeCodeLLM(
+        options=build_claude_options(args, vocabulary, brain),
+        observer=observer,
+    )
+
+
 async def main() -> int:
     args = parse_args()
 
@@ -523,10 +546,7 @@ async def main() -> int:
         if vocabulary:
             vocabulary.observe(text)
 
-    llm = ClaudeCodeLLM(
+    llm = build_llm(args, vocabulary, brain, observer=on_reply)
-        options=build_claude_options(args, vocabulary, brain),
-        observer=on_reply,
-    )
 
     tts = build_tts(args)
 

selftest.py

@@ -171,6 +171,13 @@ def check_mlx_whisper(synthesized):
     record("mlx whisper (optional)", matches(heard), f"heard {heard!r}")
 
 
+def check_macos_llm():
+    from apple_llm import probe_apple_llm
+
+    available, reason = probe_apple_llm()
+    record("macos llm", available, reason)
+
+
 async def check_claude():
     from claude_agent_sdk import ClaudeSDKClient, ResultMessage, StreamEvent
 
@@ -199,16 +206,16 @@ async def check_claude():
                     if text:
                         spoken.append(text)
                 elif isinstance(message, ResultMessage) and message.is_error:
-                    record("claude session", False, f"error: {message.result}")
+                    record("claude session (optional)", False, f"error: {message.result}")
                     return
     except Exception as e:
         hint = ""
         if "-9" in str(e):
             hint = " — the CLI was killed applying its own sandbox; run this from a normal terminal"
-        record("claude session", False, f"{e}{hint}")
+        record("claude session (optional)", False, f"{e}{hint}")
         return
     record(
-        "claude session",
+        "claude session (optional)",
         True,
         f"replied {''.join(spoken).strip()!r} using {DEFAULT_CLAUDE_MODEL}",
     )
@@ -223,6 +230,7 @@ async def main():
     synthesized = check_kokoro()
     check_apple_stt(synthesized)
     check_mlx_whisper(synthesized)
+    check_macos_llm()
     await check_claude()
 
     required_failures = [

swift/LLMHelper.swift

@@ -0,0 +1,91 @@
+// LLMHelper.swift - Interface with macOS native on-device LLM (FoundationModels / Apple Intelligence)
+//
+// Usage:
+//   llm-helper --check
+//   llm-helper --prompt "Hello" [--system "System prompt"]
+
+import Foundation
+import FoundationModels
+
+struct Options {
+    var prompt: String = ""
+    var systemPrompt: String? = nil
+    var check: Bool = false
+}
+
+func parseArguments() -> Options {
+    var options = Options()
+    var arguments = Array(CommandLine.arguments.dropFirst())
+    while let argument = arguments.first {
+        arguments.removeFirst()
+        switch argument {
+        case "--prompt":
+            if !arguments.isEmpty { options.prompt = arguments.removeFirst() }
+        case "--system":
+            if !arguments.isEmpty { options.systemPrompt = arguments.removeFirst() }
+        case "--check":
+            options.check = true
+        default:
+            if options.prompt.isEmpty && !argument.hasPrefix("--") {
+                options.prompt = argument
+            }
+        }
+    }
+    return options
+}
+
+func emit(_ payload: [String: Any]) {
+    if let data = try? JSONSerialization.data(withJSONObject: payload, options: [.sortedKeys]),
+       let str = String(data: data, encoding: .utf8) {
+        print(str)
+        fflush(stdout)
+    }
+}
+
+func fail(_ message: String) -> Never {
+    emit(["error": message])
+    exit(1)
+}
+
+@main
+struct LLMHelperApp {
+    static func main() async {
+        let options = parseArguments()
+
+        if options.check {
+            let model = SystemLanguageModel.default
+            let isAvailable = model.isAvailable
+            emit([
+                "available": isAvailable,
+                "model": "macOS SystemLanguageModel",
+                "reason": isAvailable ? "macOS Apple Intelligence model available" : "SystemLanguageModel unavailable on this hardware/OS configuration"
+            ])
+            exit(0)
+        }
+
+        guard !options.prompt.isEmpty else {
+            fail("no prompt provided")
+        }
+
+        do {
+            let model = SystemLanguageModel.default
+            guard model.isAvailable else {
+                fail("macOS SystemLanguageModel is not available on this machine")
+            }
+
+            let instructions = options.systemPrompt ?? ""
+            let session: LanguageModelSession
+            if !instructions.isEmpty {
+                session = LanguageModelSession(model: model, instructions: instructions)
+            } else {
+                session = LanguageModelSession(model: model)
+            }
+
+            let response = try await session.respond(to: options.prompt)
+            let resultText = String(describing: response)
+            emit(["done": true, "text": resultText])
+        } catch {
+            fail("LanguageModel error: \(error.localizedDescription)")
+        }
+    }
+}

swift/build.sh

@@ -13,14 +13,23 @@ swiftc -O -parse-as-library SpeechHelper.swift -o speech-helper
 echo "built $HERE/speech-helper"
 
 if ./speech-helper --check >/dev/null 2>&1; then
-    echo "it runs — ./speech-helper --check for details"
+    echo "speech-helper runs — ./speech-helper --check for details"
 else
     status=$?
     if [ "$status" -eq 137 ]; then
-        echo "built, but killed on launch (137): request binary approval for"
+        echo "built speech-helper, but killed on launch (137): request binary approval for"
         echo "  $HERE/speech-helper"
-        echo "then wait a few minutes for it to sync."
     else
-        echo "built, but exited $status — run ./speech-helper --check to see why"
+        echo "built speech-helper, but exited $status — run ./speech-helper --check to see why"
     fi
 fi
+
+if swiftc -O LLMHelper.swift -o llm-helper 2>/dev/null; then
+    echo "built $HERE/llm-helper"
+    if ./llm-helper --check >/dev/null 2>&1; then
+        echo "llm-helper runs — ./llm-helper --check for details"
+    fi
+else
+    echo "could not build llm-helper with FoundationModels; fallback to Python MLX/PyObjC bridge will be available"
+fi
+