diff --git a/src/integrations/apple-llm.js b/src/integrations/apple-llm.js new file mode 100644 index 0000000..e6f0362 --- /dev/null +++ b/src/integrations/apple-llm.js @@ -0,0 +1,412 @@ +import { execFile, spawn } from "node:child_process"; +import { promisify } from "node:util"; +import { writeFile, rm, mkdtemp } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; + +const execFileAsync = promisify(execFile); + +function swiftSourceLLMPolish() { + // From whisper-translation/apple_speech/Sources/AppleLLMPolish/main.swift + // Added instant-reply mode + general conversation + return ` +import Foundation +import FoundationModels + +struct InMsg: Decodable { + var id: String? + var mode: String // "line" | "paragraph" | "check" | "quick_reply" | "chat" + var text: String? + var prev1: String? + var prev2: String? + var context: String? + var prevSource: String? + var language: String? + var instructions: String? + var history: String? // JSON array [{"role":"user","text":".."},...] +} +struct OutMsg: Encodable { + var id: String? + var ok: Bool + var text: String + var error: String? + var ms: Int? + var mode: String? +} +func log(_ s: String) { fputs(s+"\\n", stderr) } + +@main +struct AppleLLMPolish { + static func main() async { + let args = CommandLine.arguments + if args.contains("--help") || args.contains("-h") { + fputs("Usage: apple-llm-polish [--check]\\nPipe JSONL in stdin, JSONL out\\nModes: line, paragraph, quick_reply, chat, check\\n", stderr); exit(0) + } + if args.contains("--check") { await runCheck(); return } + await runPipe() + } + static func runCheck() async { + let m = SystemLanguageModel.default + var pingText = "unavailable" + var ok = false + if m.isAvailable { + do { + let session = LanguageModelSession(model: m, instructions: "You are concise.") + let r = try await session.respond(to: "Say ok") + pingText = r.content + ok = true + } catch { pingText = error.localizedDescription } + } + let out: [String: Any] = [ + "available": m.isAvailable, + "availability": "\\(m.availability)", + "ping": pingText, + "ok": ok, + "model": "SystemLanguageModel 3B ANE" + ] + if let d = try? JSONSerialization.data(withJSONObject: out), let s = String(data: d, encoding: .utf8) { print(s) } + } + static func runPipe() async { + let m = SystemLanguageModel.default + guard m.isAvailable else { + let reason = "\\(m.availability)" + while let line = readLine() { + if line.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { continue } + var idv: String? = nil + if let data = line.data(using: .utf8), let dict = try? JSONSerialization.jsonObject(with: data) as? [String:Any] { idv = dict["id"] as? String } + let out = OutMsg(id: idv, ok: false, text: "", error: "model unavailable: \\(reason)", ms: nil, mode: "error") + if let d = try? JSONEncoder().encode(out), let s = String(data: d, encoding: .utf8) { print(s); fflush(stdout) } + } + return + } + // Keep sessions warm — separate useCases + let lineSession = LanguageModelSession(model: SystemLanguageModel(useCase: .general, guardrails: .permissiveContentTransformations), instructions: lineSystemPrompt()) + let paraSession = LanguageModelSession(model: SystemLanguageModel(useCase: .general, guardrails: .permissiveContentTransformations), instructions: paraSystemPrompt()) + let quickSession = LanguageModelSession(model: SystemLanguageModel(useCase: .general, guardrails: .permissiveContentTransformations), instructions: quickReplySystemPrompt()) + let chatSession = LanguageModelSession(model: SystemLanguageModel(useCase: .general, guardrails: .permissiveContentTransformations), instructions: "You are Hermes, a concise helpful voice assistant for ESP32 devices. Keep replies under 40 words, warm and concrete, kid-safe.") + + lineSession.prewarm() + paraSession.prewarm() + quickSession.prewarm() + chatSession.prewarm() + + log("[apple-llm] ready, ANE-backed 3B") + + while let line = readLine() { + let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.isEmpty { continue } + guard let data = line.data(using: .utf8), let req = try? JSONDecoder().decode(InMsg.self, from: data) else { + let out = OutMsg(id: nil, ok: false, text: "", error: "bad json", ms: nil, mode: "error") + if let d = try? JSONEncoder().encode(out), let s = String(data: d, encoding: .utf8) { print(s); fflush(stdout) } + continue + } + if req.mode == "check" { + let out = OutMsg(id: req.id, ok: m.isAvailable, text: "\\(m.availability)", error: nil, ms: 0, mode: "check") + if let d = try? JSONEncoder().encode(out), let s = String(data: d, encoding: .utf8) { print(s); fflush(stdout) } + continue + } + let t0 = Date() + do { + let (session, prompt, temp): (LanguageModelSession, String, Double) + switch req.mode { + case "paragraph": + session = paraSession + prompt = buildParagraphPrompt(context: req.context ?? "", prevSource: req.prevSource ?? "", newText: req.text ?? "") + temp = 0.2 + case "quick_reply": + session = quickSession + prompt = buildQuickReplyPrompt(draft: req.text ?? "", context: req.context, instructions: req.instructions) + temp = 0.4 + case "chat": + // For chat, rebuild prompt from history if provided, else use text directly + session = chatSession + if let hist = req.history, !hist.isEmpty { + prompt = buildChatPrompt(historyJSON: hist, newText: req.text ?? "", instructions: req.instructions) + } else { + prompt = req.text ?? "" + } + temp = 0.5 + default: // line + session = lineSession + prompt = buildLinePrompt(text: req.text ?? "", prev1: req.prev1 ?? "", prev2: req.prev2 ?? "") + temp = 0.1 + } + var opts = GenerationOptions() + opts.temperature = temp + let resp = try await session.respond(to: prompt, options: opts) + let ms = Int(Date().timeIntervalSince(t0)*1000) + let cleaned = resp.content.trimmingCharacters(in: .whitespacesAndNewlines) + let out = OutMsg(id: req.id, ok: true, text: cleaned.isEmpty ? (req.text ?? "") : cleaned, error: nil, ms: ms, mode: req.mode) + if let d = try? JSONEncoder().encode(out), let s = String(data: d, encoding: .utf8) { print(s); fflush(stdout) } + } catch { + let ms = Int(Date().timeIntervalSince(t0)*1000) + let out = OutMsg(id: req.id, ok: false, text: req.text ?? "", error: error.localizedDescription, ms: ms, mode: req.mode) + if let d = try? JSONEncoder().encode(out), let s = String(data: d, encoding: .utf8) { print(s); fflush(stdout) } + } + } + } + static func lineSystemPrompt() -> String { + return "You are a real-time caption polisher. Fix punctuation, casing, STT typos. Remove filler (uh, um). Keep meaning. Output one polished line only." + } + static func paraSystemPrompt() -> String { + return "You are a careful live transcript editor. Goal: most faithful readable English. NEW SOURCE TEXT is primary. PREVIOUS CONTEXT only if helps continuity. English only. No meta commentary. Return revised transcript only." + } + static func quickReplySystemPrompt() -> String { + return "You are Hermes instant-reply for voice devices. You get a live DRAFT transcript (partial, may have typos) from user speaking to ESP32. Produce a super-short smart ACK/reply preview in <1 second based on draft intent. Rules: under 20 words, warm, acknowledge intent, say you're working on full answer. Don't hallucinate specifics beyond draft. Examples: 'Got it, looking that up...' 'On it, one sec...' 'Nice, let me think about that...' If draft is greeting, reply friendly greeting. Return only reply text." + } + static func buildLinePrompt(text: String, prev1: String, prev2: String) -> String { + var p = "" + if !prev2.isEmpty { p += "Previous 2: \\(prev2)\\n" } + if !prev1.isEmpty { p += "Previous 1: \\(prev1)\\n" } + p += "Current: \\(text)\\nPolished:" + return p + } + static func buildParagraphPrompt(context: String, prevSource: String, newText: String) -> String { + return "Edit this transcript.\\n[PREVIOUS CONTEXT]\\n\\(context)\\n[PREVIOUS SOURCE]\\n\\(prevSource)\\n[NEW]\\n\\(newText)" + } + static func buildQuickReplyPrompt(draft: String, context: String?, instructions: String?) -> String { + var p = "DRAFT transcript (partial, live): \\\"\\(draft)\\\"\\n" + if let c = context, !c.isEmpty { p += "Recent context: \\(c)\\n" } + if let i = instructions, !i.isEmpty { p += "User instructions: \\(i)\\n" } + p += "\\nProduce instant smart reply (under 20 words) acknowledging intent while full answer prepares. If unclear, say 'Heard you, working on it...'" + return p + } + static func buildChatPrompt(historyJSON: String, newText: String, instructions: String?) -> String { + // history is JSON array serialized + return "Conversation history: \\(historyJSON)\\nUser says (draft/final): \\(newText)\\n\\(instructions != nil ? \"Instructions: \\(instructions!)\\n\" : \"\")Reply concisely for voice device (<40 words):" + } +} +`; +} + +class AppleLLMSession { + constructor() { + this.proc = null; + this.tmpDir = null; + this.binFile = null; + this.ready = false; + this.reqId = 0; + this.pending = new Map(); + this.lastActivity = Date.now(); + } + + async ensureBuilt() { + const tmpBase = path.join(os.tmpdir(), "apple-llm-"); + this.tmpDir = await mkdtemp(tmpBase); + const swiftFile = path.join(this.tmpDir, "Main.swift"); + this.binFile = path.join(this.tmpDir, "apple-llm-polish"); + await writeFile(swiftFile, swiftSourceLLMPolish(), "utf8"); + try { + await execFileAsync("/usr/bin/swiftc", ["-O", "-parse-as-library", swiftFile, "-o", this.binFile, "-framework", "Foundation", "-framework", "FoundationModels"], { timeout: 60000, maxBuffer: 20*1024*1024 }); + } catch (e) { + throw new Error(`swiftc LLM build failed: ${e.stderr||e.message}`); + } + } + + async start() { + if (this.proc && this.proc.exitCode === null && this.ready) return; + if (!this.binFile) await this.ensureBuilt(); + return new Promise((resolve, reject) => { + this.proc = spawn(this.binFile, [], { stdio: ["pipe", "pipe", "pipe"] }); + let stderrBuf = ""; + let stdoutBuf = ""; + this.proc.stderr.on("data", d => { stderrBuf += d.toString(); }); + this.proc.stdout.on("data", d => { + const txt = d.toString(); + stdoutBuf += txt; + // Parse lines for responses + let lines = stdoutBuf.split("\n"); + stdoutBuf = lines.pop() || ""; + for (let line of lines) { + line = line.trim(); + if (!line) continue; + try { + const obj = JSON.parse(line); + const id = obj.id; + if (id && this.pending.has(id)) { + const {resolve} = this.pending.get(id); + this.pending.delete(id); + resolve(obj); + } + } catch {} + } + }); + + setTimeout(() => { + if (this.proc.exitCode !== null) { + reject(new Error(`apple-llm exited early code=${this.proc.exitCode} stderr=${stderrBuf.slice(0,2000)}`)); + } else { + this.ready = true; + // Capture remaining stdout buffering setup + this._stdoutLeftover = ""; + resolve(); + } + }, 1500); + this.proc.on("error", reject); + }); + } + + // Ensure reader continues after initial start + _ensureReader() { + if (this._readerSetup) return; + this._readerSetup = true; + // Already setup in start() via stdout.on data - but need to handle leftover buffering for late responses + let buf = ""; + if (this.proc) { + // Additional listener for any missed + this.proc.stdout.on("data", chunk => { + buf += chunk.toString(); + let lines = buf.split("\n"); + buf = lines.pop() || ""; + for (let line of lines) { + line = line.trim(); + if (!line) continue; + try { + const obj = JSON.parse(line); + const id = obj.id; + if (id && this.pending.has(id)) { + const {resolve} = this.pending.get(id); + this.pending.delete(id); + resolve(obj); + } + } catch {} + } + }); + } + } + + async call(args, timeoutMs = 10000) { + await this.start(); + this._ensureReader(); + const id = String(this.reqId++); + const payload = { id, ...args }; + + return new Promise((resolve, reject) => { + let timer = setTimeout(() => { + if (this.pending.has(id)) { + this.pending.delete(id); + resolve({ ok: false, text: args.text || "", error: `timeout ${timeoutMs}ms`, id, ms: timeoutMs }); + } + }, timeoutMs); + + this.pending.set(id, { + resolve: (obj) => { + clearTimeout(timer); + resolve(obj); + } + }); + + try { + this.proc.stdin.write(JSON.stringify(payload) + "\n"); + } catch (e) { + clearTimeout(timer); + this.pending.delete(id); + reject(e); + } + }); + } + + async close() { + try { + if (this.proc) { + try { this.proc.stdin.end(); } catch {} + await new Promise(r => { this.proc.on("close", r); setTimeout(r, 1500); }); + try { this.proc.kill(); } catch {} + } + } catch {} + try { if (this.tmpDir) await rm(this.tmpDir, {recursive:true, force:true}); } catch {} + this.proc = null; + this.ready = false; + } +} + +// Singleton global session for all LLM calls (fast, keeps KV cache warm) +let globalSession = null; + +async function getSession() { + if (!globalSession) { + globalSession = new AppleLLMSession(); + await globalSession.start(); + } + globalSession.lastActivity = Date.now(); + return globalSession; +} + +export async function appleLLMCheck() { + const s = await getSession(); + const res = await s.call({ mode: "check", text: "check" }, 10000); + return { + ok: res.ok, + available: res.ok, + text: res.text || res.error || "", + ms: res.ms, + engine: "Apple FoundationModels 3B ANE" + }; +} + +export async function appleLLMPolish({ text, prev1, prev2, mode } = {}) { + if (!text) throw new Error("text required"); + const s = await getSession(); + const m = mode || "line"; + const res = await s.call({ mode: m, text, prev1: prev1||"", prev2: prev2||"" }, 8000); + return { + ok: res.ok, + text: res.text || text, + original: text, + ms: res.ms, + mode: m, + error: res.error || undefined + }; +} + +export async function appleLLMQuickReply({ draft, context, instructions } = {}) { + if (!draft) throw new Error("draft required"); + const s = await getSession(); + const res = await s.call({ mode: "quick_reply", text: draft, context: context||"", instructions: instructions||"" }, 3000); + return { + ok: res.ok, + text: res.text || "Got it, working on it...", + draft, + ms: res.ms, + engine: "Apple FoundationModels 3B instant" + }; +} + +export async function appleLLMChat({ text, history, instructions } = {}) { + if (!text) throw new Error("text required"); + const s = await getSession(); + const histStr = history ? JSON.stringify(history) : ""; + const res = await s.call({ mode: "chat", text, history: histStr, instructions: instructions||"" }, 5000); + return { + ok: res.ok, + text: res.text || "", + input: text, + ms: res.ms, + engine: "Apple FoundationModels 3B voice" + }; +} + +export async function appleLLMClose() { + if (globalSession) { + await globalSession.close(); + globalSession = null; + } + return { ok: true, closed: true }; +} + +export async function appleLLMStatus() { + return { + active: !!globalSession, + ready: globalSession?.ready || false, + lastActivity: globalSession ? new Date(globalSession.lastActivity).toISOString() : null, + pid: globalSession?.proc?.pid || null + }; +} + +// Idle cleanup 2 min +setInterval(async () => { + if (globalSession && Date.now() - globalSession.lastActivity > 120000) { + try { await globalSession.close(); } catch {} + globalSession = null; + } +}, 20000); diff --git a/src/server.js b/src/server.js index 1ded876..d523058 100644 --- a/src/server.js +++ b/src/server.js @@ -41,6 +41,14 @@ import { speechLiveClose, speechLiveStatus, } from "./integrations/speech-live.js"; +import { + appleLLMCheck, + appleLLMPolish, + appleLLMQuickReply, + appleLLMChat, + appleLLMClose, + appleLLMStatus, +} from "./integrations/apple-llm.js"; const jsonResult = (value) => ({ content: [{ type: "text", text: JSON.stringify(value, null, 2) }], @@ -376,5 +384,62 @@ export function createMacMiniMcpServer() { handled(buildGeminiChromePrompt), ); + + server.tool( + "apple_llm_check", + "Check if Apple on-device 3B LLM (FoundationModels SystemLanguageModel) is available. ANE-accelerated, offline, ~0.6s vs Ollama 2-3s. Required for polish/quick_reply/chat.", + {}, + handled(() => appleLLMCheck()), + ); + + server.tool( + "apple_llm_polish", + "Polish a caption line/paragraph using Apple on-device 3B LLM ANE. From whisper-translation engine_apple_llm.py: same as AppleLLMPolish binary pipe. Fixes punctuation, casing, typos from live transcript. Mode: line (default) or paragraph. Reuses single warm session.", + { + text: z.string().min(1).describe("Text to polish (STT draft or final)."), + prev1: z.string().optional().describe("Previous line for context (line mode)."), + prev2: z.string().optional().describe("Second previous line."), + mode: z.enum(["line", "paragraph"]).optional().describe("line (default) or paragraph."), + }, + handled(({ text, prev1, prev2, mode }) => appleLLMPolish({ text, prev1, prev2, mode })), + ); + + server.tool( + "apple_llm_quick_reply", + "INSTANT smart reply from live draft — for voice gateway. Feed the volatile draft transcript from speech_live_transcribe while user is speaking. Apple 3B ANE returns <20-word ACK preview in ~0.6s, before final transcription. Use this to provide almost immediate response when voice stops: send draft to quick_reply, get instant text to speak/display, then full Hermes turn in background. Reuses same ANE session as polish.", + { + draft: z.string().min(1).describe("Live draft transcript (partial, may have typos) from speech_live_transcribe. Used to infer intent for instant preview reply."), + context: z.string().optional().describe("Optional previous conversation context or last assistant reply."), + instructions: z.string().optional().describe("Optional system instructions for quick reply persona."), + }, + handled(({ draft, context, instructions }) => appleLLMQuickReply({ draft, context, instructions })), + ); + + server.tool( + "apple_llm_chat", + "Fast voice-assistant chat via Apple 3B ANE (FoundationModels). Under 40 words, warm, kid-safe, for ESP32. Use when you want Mac mini to directly answer draft/final without calling Hermes gateway. Accepts history array and current text. ~0.6-1.2s.", + { + text: z.string().min(1).describe("User message (draft or final transcript)."), + history: z.array(z.object({ role: z.string(), text: z.string() })).optional().describe("Optional conversation history [{role, text}]."), + instructions: z.string().optional().describe("Optional custom instructions."), + }, + handled(({ text, history, instructions }) => appleLLMChat({ text, history, instructions })), + ); + + server.tool( + "apple_llm_status", + "Show Apple LLM session status: active, ready, pid, lastActivity.", + {}, + handled(() => appleLLMStatus()), + ); + + server.tool( + "apple_llm_close", + "Close Apple LLM ANE session (frees Swift process + KV cache). Auto-closes after 2min idle anyway.", + {}, + handled(() => appleLLMClose()), + ); + + return server; }