feat: Apple on-device 3B LLM MCP tools (FoundationModels ANE)
- New src/integrations/apple-llm.js: port of whisper-translation/apple_speech AppleLLMPolish
Swift source embedded, same pipe protocol: JSONL {id,mode,text} -> {id,text,ok,ms}
Persistent warm session (KV cache), ANE-accelerated ~0.6s vs Ollama 2-3s
Modes: line polish, paragraph, quick_reply instant (~20 words draft->reply),
chat voice assistant (<40 words), check/status/close
- MCP tools:
apple_llm_check - availability + ping
apple_llm_polish - caption polish
apple_llm_quick_reply - instant smart reply from live draft for voice gateway
DRAFT (volatile) -> instant ACK while final transcribe + LLM turn runs
apple_llm_chat - fast voice chat via 3B on-device
apple_llm_status/close
- Intended for ESP32 voice gateway: feed draft from speech_live_transcribe
into apple_llm_quick_reply for almost immediate spoken reply when voice stops,
before Hermes full turn.
- Swift impl: SystemLanguageModel + GenerationOptions temp 0.1-0.5
Pipe ready log, prewarm sessions
This commit is contained in:
aeroreyna
@@ -0,0 +1,412 @@
|
||||
import { execFile, spawn } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { writeFile, rm, mkdtemp } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import os from "node:os";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
function swiftSourceLLMPolish() {
|
||||
// From whisper-translation/apple_speech/Sources/AppleLLMPolish/main.swift
|
||||
// Added instant-reply mode + general conversation
|
||||
return `
|
||||
import Foundation
|
||||
import FoundationModels
|
||||
|
||||
struct InMsg: Decodable {
|
||||
var id: String?
|
||||
var mode: String // "line" | "paragraph" | "check" | "quick_reply" | "chat"
|
||||
var text: String?
|
||||
var prev1: String?
|
||||
var prev2: String?
|
||||
var context: String?
|
||||
var prevSource: String?
|
||||
var language: String?
|
||||
var instructions: String?
|
||||
var history: String? // JSON array [{"role":"user","text":".."},...]
|
||||
}
|
||||
struct OutMsg: Encodable {
|
||||
var id: String?
|
||||
var ok: Bool
|
||||
var text: String
|
||||
var error: String?
|
||||
var ms: Int?
|
||||
var mode: String?
|
||||
}
|
||||
func log(_ s: String) { fputs(s+"\\n", stderr) }
|
||||
|
||||
@main
|
||||
struct AppleLLMPolish {
|
||||
static func main() async {
|
||||
let args = CommandLine.arguments
|
||||
if args.contains("--help") || args.contains("-h") {
|
||||
fputs("Usage: apple-llm-polish [--check]\\nPipe JSONL in stdin, JSONL out\\nModes: line, paragraph, quick_reply, chat, check\\n", stderr); exit(0)
|
||||
}
|
||||
if args.contains("--check") { await runCheck(); return }
|
||||
await runPipe()
|
||||
}
|
||||
static func runCheck() async {
|
||||
let m = SystemLanguageModel.default
|
||||
var pingText = "unavailable"
|
||||
var ok = false
|
||||
if m.isAvailable {
|
||||
do {
|
||||
let session = LanguageModelSession(model: m, instructions: "You are concise.")
|
||||
let r = try await session.respond(to: "Say ok")
|
||||
pingText = r.content
|
||||
ok = true
|
||||
} catch { pingText = error.localizedDescription }
|
||||
}
|
||||
let out: [String: Any] = [
|
||||
"available": m.isAvailable,
|
||||
"availability": "\\(m.availability)",
|
||||
"ping": pingText,
|
||||
"ok": ok,
|
||||
"model": "SystemLanguageModel 3B ANE"
|
||||
]
|
||||
if let d = try? JSONSerialization.data(withJSONObject: out), let s = String(data: d, encoding: .utf8) { print(s) }
|
||||
}
|
||||
static func runPipe() async {
|
||||
let m = SystemLanguageModel.default
|
||||
guard m.isAvailable else {
|
||||
let reason = "\\(m.availability)"
|
||||
while let line = readLine() {
|
||||
if line.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { continue }
|
||||
var idv: String? = nil
|
||||
if let data = line.data(using: .utf8), let dict = try? JSONSerialization.jsonObject(with: data) as? [String:Any] { idv = dict["id"] as? String }
|
||||
let out = OutMsg(id: idv, ok: false, text: "", error: "model unavailable: \\(reason)", ms: nil, mode: "error")
|
||||
if let d = try? JSONEncoder().encode(out), let s = String(data: d, encoding: .utf8) { print(s); fflush(stdout) }
|
||||
}
|
||||
return
|
||||
}
|
||||
// Keep sessions warm — separate useCases
|
||||
let lineSession = LanguageModelSession(model: SystemLanguageModel(useCase: .general, guardrails: .permissiveContentTransformations), instructions: lineSystemPrompt())
|
||||
let paraSession = LanguageModelSession(model: SystemLanguageModel(useCase: .general, guardrails: .permissiveContentTransformations), instructions: paraSystemPrompt())
|
||||
let quickSession = LanguageModelSession(model: SystemLanguageModel(useCase: .general, guardrails: .permissiveContentTransformations), instructions: quickReplySystemPrompt())
|
||||
let chatSession = LanguageModelSession(model: SystemLanguageModel(useCase: .general, guardrails: .permissiveContentTransformations), instructions: "You are Hermes, a concise helpful voice assistant for ESP32 devices. Keep replies under 40 words, warm and concrete, kid-safe.")
|
||||
|
||||
lineSession.prewarm()
|
||||
paraSession.prewarm()
|
||||
quickSession.prewarm()
|
||||
chatSession.prewarm()
|
||||
|
||||
log("[apple-llm] ready, ANE-backed 3B")
|
||||
|
||||
while let line = readLine() {
|
||||
let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
if trimmed.isEmpty { continue }
|
||||
guard let data = line.data(using: .utf8), let req = try? JSONDecoder().decode(InMsg.self, from: data) else {
|
||||
let out = OutMsg(id: nil, ok: false, text: "", error: "bad json", ms: nil, mode: "error")
|
||||
if let d = try? JSONEncoder().encode(out), let s = String(data: d, encoding: .utf8) { print(s); fflush(stdout) }
|
||||
continue
|
||||
}
|
||||
if req.mode == "check" {
|
||||
let out = OutMsg(id: req.id, ok: m.isAvailable, text: "\\(m.availability)", error: nil, ms: 0, mode: "check")
|
||||
if let d = try? JSONEncoder().encode(out), let s = String(data: d, encoding: .utf8) { print(s); fflush(stdout) }
|
||||
continue
|
||||
}
|
||||
let t0 = Date()
|
||||
do {
|
||||
let (session, prompt, temp): (LanguageModelSession, String, Double)
|
||||
switch req.mode {
|
||||
case "paragraph":
|
||||
session = paraSession
|
||||
prompt = buildParagraphPrompt(context: req.context ?? "", prevSource: req.prevSource ?? "", newText: req.text ?? "")
|
||||
temp = 0.2
|
||||
case "quick_reply":
|
||||
session = quickSession
|
||||
prompt = buildQuickReplyPrompt(draft: req.text ?? "", context: req.context, instructions: req.instructions)
|
||||
temp = 0.4
|
||||
case "chat":
|
||||
// For chat, rebuild prompt from history if provided, else use text directly
|
||||
session = chatSession
|
||||
if let hist = req.history, !hist.isEmpty {
|
||||
prompt = buildChatPrompt(historyJSON: hist, newText: req.text ?? "", instructions: req.instructions)
|
||||
} else {
|
||||
prompt = req.text ?? ""
|
||||
}
|
||||
temp = 0.5
|
||||
default: // line
|
||||
session = lineSession
|
||||
prompt = buildLinePrompt(text: req.text ?? "", prev1: req.prev1 ?? "", prev2: req.prev2 ?? "")
|
||||
temp = 0.1
|
||||
}
|
||||
var opts = GenerationOptions()
|
||||
opts.temperature = temp
|
||||
let resp = try await session.respond(to: prompt, options: opts)
|
||||
let ms = Int(Date().timeIntervalSince(t0)*1000)
|
||||
let cleaned = resp.content.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
let out = OutMsg(id: req.id, ok: true, text: cleaned.isEmpty ? (req.text ?? "") : cleaned, error: nil, ms: ms, mode: req.mode)
|
||||
if let d = try? JSONEncoder().encode(out), let s = String(data: d, encoding: .utf8) { print(s); fflush(stdout) }
|
||||
} catch {
|
||||
let ms = Int(Date().timeIntervalSince(t0)*1000)
|
||||
let out = OutMsg(id: req.id, ok: false, text: req.text ?? "", error: error.localizedDescription, ms: ms, mode: req.mode)
|
||||
if let d = try? JSONEncoder().encode(out), let s = String(data: d, encoding: .utf8) { print(s); fflush(stdout) }
|
||||
}
|
||||
}
|
||||
}
|
||||
static func lineSystemPrompt() -> String {
|
||||
return "You are a real-time caption polisher. Fix punctuation, casing, STT typos. Remove filler (uh, um). Keep meaning. Output one polished line only."
|
||||
}
|
||||
static func paraSystemPrompt() -> String {
|
||||
return "You are a careful live transcript editor. Goal: most faithful readable English. NEW SOURCE TEXT is primary. PREVIOUS CONTEXT only if helps continuity. English only. No meta commentary. Return revised transcript only."
|
||||
}
|
||||
static func quickReplySystemPrompt() -> String {
|
||||
return "You are Hermes instant-reply for voice devices. You get a live DRAFT transcript (partial, may have typos) from user speaking to ESP32. Produce a super-short smart ACK/reply preview in <1 second based on draft intent. Rules: under 20 words, warm, acknowledge intent, say you're working on full answer. Don't hallucinate specifics beyond draft. Examples: 'Got it, looking that up...' 'On it, one sec...' 'Nice, let me think about that...' If draft is greeting, reply friendly greeting. Return only reply text."
|
||||
}
|
||||
static func buildLinePrompt(text: String, prev1: String, prev2: String) -> String {
|
||||
var p = ""
|
||||
if !prev2.isEmpty { p += "Previous 2: \\(prev2)\\n" }
|
||||
if !prev1.isEmpty { p += "Previous 1: \\(prev1)\\n" }
|
||||
p += "Current: \\(text)\\nPolished:"
|
||||
return p
|
||||
}
|
||||
static func buildParagraphPrompt(context: String, prevSource: String, newText: String) -> String {
|
||||
return "Edit this transcript.\\n[PREVIOUS CONTEXT]\\n\\(context)\\n[PREVIOUS SOURCE]\\n\\(prevSource)\\n[NEW]\\n\\(newText)"
|
||||
}
|
||||
static func buildQuickReplyPrompt(draft: String, context: String?, instructions: String?) -> String {
|
||||
var p = "DRAFT transcript (partial, live): \\\"\\(draft)\\\"\\n"
|
||||
if let c = context, !c.isEmpty { p += "Recent context: \\(c)\\n" }
|
||||
if let i = instructions, !i.isEmpty { p += "User instructions: \\(i)\\n" }
|
||||
p += "\\nProduce instant smart reply (under 20 words) acknowledging intent while full answer prepares. If unclear, say 'Heard you, working on it...'"
|
||||
return p
|
||||
}
|
||||
static func buildChatPrompt(historyJSON: String, newText: String, instructions: String?) -> String {
|
||||
// history is JSON array serialized
|
||||
return "Conversation history: \\(historyJSON)\\nUser says (draft/final): \\(newText)\\n\\(instructions != nil ? \"Instructions: \\(instructions!)\\n\" : \"\")Reply concisely for voice device (<40 words):"
|
||||
}
|
||||
}
|
||||
`;
|
||||
}
|
||||
|
||||
class AppleLLMSession {
|
||||
constructor() {
|
||||
this.proc = null;
|
||||
this.tmpDir = null;
|
||||
this.binFile = null;
|
||||
this.ready = false;
|
||||
this.reqId = 0;
|
||||
this.pending = new Map();
|
||||
this.lastActivity = Date.now();
|
||||
}
|
||||
|
||||
async ensureBuilt() {
|
||||
const tmpBase = path.join(os.tmpdir(), "apple-llm-");
|
||||
this.tmpDir = await mkdtemp(tmpBase);
|
||||
const swiftFile = path.join(this.tmpDir, "Main.swift");
|
||||
this.binFile = path.join(this.tmpDir, "apple-llm-polish");
|
||||
await writeFile(swiftFile, swiftSourceLLMPolish(), "utf8");
|
||||
try {
|
||||
await execFileAsync("/usr/bin/swiftc", ["-O", "-parse-as-library", swiftFile, "-o", this.binFile, "-framework", "Foundation", "-framework", "FoundationModels"], { timeout: 60000, maxBuffer: 20*1024*1024 });
|
||||
} catch (e) {
|
||||
throw new Error(`swiftc LLM build failed: ${e.stderr||e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async start() {
|
||||
if (this.proc && this.proc.exitCode === null && this.ready) return;
|
||||
if (!this.binFile) await this.ensureBuilt();
|
||||
return new Promise((resolve, reject) => {
|
||||
this.proc = spawn(this.binFile, [], { stdio: ["pipe", "pipe", "pipe"] });
|
||||
let stderrBuf = "";
|
||||
let stdoutBuf = "";
|
||||
this.proc.stderr.on("data", d => { stderrBuf += d.toString(); });
|
||||
this.proc.stdout.on("data", d => {
|
||||
const txt = d.toString();
|
||||
stdoutBuf += txt;
|
||||
// Parse lines for responses
|
||||
let lines = stdoutBuf.split("\n");
|
||||
stdoutBuf = lines.pop() || "";
|
||||
for (let line of lines) {
|
||||
line = line.trim();
|
||||
if (!line) continue;
|
||||
try {
|
||||
const obj = JSON.parse(line);
|
||||
const id = obj.id;
|
||||
if (id && this.pending.has(id)) {
|
||||
const {resolve} = this.pending.get(id);
|
||||
this.pending.delete(id);
|
||||
resolve(obj);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
});
|
||||
|
||||
setTimeout(() => {
|
||||
if (this.proc.exitCode !== null) {
|
||||
reject(new Error(`apple-llm exited early code=${this.proc.exitCode} stderr=${stderrBuf.slice(0,2000)}`));
|
||||
} else {
|
||||
this.ready = true;
|
||||
// Capture remaining stdout buffering setup
|
||||
this._stdoutLeftover = "";
|
||||
resolve();
|
||||
}
|
||||
}, 1500);
|
||||
this.proc.on("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
// Ensure reader continues after initial start
|
||||
_ensureReader() {
|
||||
if (this._readerSetup) return;
|
||||
this._readerSetup = true;
|
||||
// Already setup in start() via stdout.on data - but need to handle leftover buffering for late responses
|
||||
let buf = "";
|
||||
if (this.proc) {
|
||||
// Additional listener for any missed
|
||||
this.proc.stdout.on("data", chunk => {
|
||||
buf += chunk.toString();
|
||||
let lines = buf.split("\n");
|
||||
buf = lines.pop() || "";
|
||||
for (let line of lines) {
|
||||
line = line.trim();
|
||||
if (!line) continue;
|
||||
try {
|
||||
const obj = JSON.parse(line);
|
||||
const id = obj.id;
|
||||
if (id && this.pending.has(id)) {
|
||||
const {resolve} = this.pending.get(id);
|
||||
this.pending.delete(id);
|
||||
resolve(obj);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async call(args, timeoutMs = 10000) {
|
||||
await this.start();
|
||||
this._ensureReader();
|
||||
const id = String(this.reqId++);
|
||||
const payload = { id, ...args };
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
let timer = setTimeout(() => {
|
||||
if (this.pending.has(id)) {
|
||||
this.pending.delete(id);
|
||||
resolve({ ok: false, text: args.text || "", error: `timeout ${timeoutMs}ms`, id, ms: timeoutMs });
|
||||
}
|
||||
}, timeoutMs);
|
||||
|
||||
this.pending.set(id, {
|
||||
resolve: (obj) => {
|
||||
clearTimeout(timer);
|
||||
resolve(obj);
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
this.proc.stdin.write(JSON.stringify(payload) + "\n");
|
||||
} catch (e) {
|
||||
clearTimeout(timer);
|
||||
this.pending.delete(id);
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async close() {
|
||||
try {
|
||||
if (this.proc) {
|
||||
try { this.proc.stdin.end(); } catch {}
|
||||
await new Promise(r => { this.proc.on("close", r); setTimeout(r, 1500); });
|
||||
try { this.proc.kill(); } catch {}
|
||||
}
|
||||
} catch {}
|
||||
try { if (this.tmpDir) await rm(this.tmpDir, {recursive:true, force:true}); } catch {}
|
||||
this.proc = null;
|
||||
this.ready = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Singleton global session for all LLM calls (fast, keeps KV cache warm)
|
||||
let globalSession = null;
|
||||
|
||||
async function getSession() {
|
||||
if (!globalSession) {
|
||||
globalSession = new AppleLLMSession();
|
||||
await globalSession.start();
|
||||
}
|
||||
globalSession.lastActivity = Date.now();
|
||||
return globalSession;
|
||||
}
|
||||
|
||||
export async function appleLLMCheck() {
|
||||
const s = await getSession();
|
||||
const res = await s.call({ mode: "check", text: "check" }, 10000);
|
||||
return {
|
||||
ok: res.ok,
|
||||
available: res.ok,
|
||||
text: res.text || res.error || "",
|
||||
ms: res.ms,
|
||||
engine: "Apple FoundationModels 3B ANE"
|
||||
};
|
||||
}
|
||||
|
||||
export async function appleLLMPolish({ text, prev1, prev2, mode } = {}) {
|
||||
if (!text) throw new Error("text required");
|
||||
const s = await getSession();
|
||||
const m = mode || "line";
|
||||
const res = await s.call({ mode: m, text, prev1: prev1||"", prev2: prev2||"" }, 8000);
|
||||
return {
|
||||
ok: res.ok,
|
||||
text: res.text || text,
|
||||
original: text,
|
||||
ms: res.ms,
|
||||
mode: m,
|
||||
error: res.error || undefined
|
||||
};
|
||||
}
|
||||
|
||||
export async function appleLLMQuickReply({ draft, context, instructions } = {}) {
|
||||
if (!draft) throw new Error("draft required");
|
||||
const s = await getSession();
|
||||
const res = await s.call({ mode: "quick_reply", text: draft, context: context||"", instructions: instructions||"" }, 3000);
|
||||
return {
|
||||
ok: res.ok,
|
||||
text: res.text || "Got it, working on it...",
|
||||
draft,
|
||||
ms: res.ms,
|
||||
engine: "Apple FoundationModels 3B instant"
|
||||
};
|
||||
}
|
||||
|
||||
export async function appleLLMChat({ text, history, instructions } = {}) {
|
||||
if (!text) throw new Error("text required");
|
||||
const s = await getSession();
|
||||
const histStr = history ? JSON.stringify(history) : "";
|
||||
const res = await s.call({ mode: "chat", text, history: histStr, instructions: instructions||"" }, 5000);
|
||||
return {
|
||||
ok: res.ok,
|
||||
text: res.text || "",
|
||||
input: text,
|
||||
ms: res.ms,
|
||||
engine: "Apple FoundationModels 3B voice"
|
||||
};
|
||||
}
|
||||
|
||||
export async function appleLLMClose() {
|
||||
if (globalSession) {
|
||||
await globalSession.close();
|
||||
globalSession = null;
|
||||
}
|
||||
return { ok: true, closed: true };
|
||||
}
|
||||
|
||||
export async function appleLLMStatus() {
|
||||
return {
|
||||
active: !!globalSession,
|
||||
ready: globalSession?.ready || false,
|
||||
lastActivity: globalSession ? new Date(globalSession.lastActivity).toISOString() : null,
|
||||
pid: globalSession?.proc?.pid || null
|
||||
};
|
||||
}
|
||||
|
||||
// Idle cleanup 2 min
|
||||
setInterval(async () => {
|
||||
if (globalSession && Date.now() - globalSession.lastActivity > 120000) {
|
||||
try { await globalSession.close(); } catch {}
|
||||
globalSession = null;
|
||||
}
|
||||
}, 20000);
|
||||
@@ -41,6 +41,14 @@ import {
|
||||
speechLiveClose,
|
||||
speechLiveStatus,
|
||||
} from "./integrations/speech-live.js";
|
||||
import {
|
||||
appleLLMCheck,
|
||||
appleLLMPolish,
|
||||
appleLLMQuickReply,
|
||||
appleLLMChat,
|
||||
appleLLMClose,
|
||||
appleLLMStatus,
|
||||
} from "./integrations/apple-llm.js";
|
||||
|
||||
const jsonResult = (value) => ({
|
||||
content: [{ type: "text", text: JSON.stringify(value, null, 2) }],
|
||||
@@ -376,5 +384,62 @@ export function createMacMiniMcpServer() {
|
||||
handled(buildGeminiChromePrompt),
|
||||
);
|
||||
|
||||
|
||||
server.tool(
|
||||
"apple_llm_check",
|
||||
"Check if Apple on-device 3B LLM (FoundationModels SystemLanguageModel) is available. ANE-accelerated, offline, ~0.6s vs Ollama 2-3s. Required for polish/quick_reply/chat.",
|
||||
{},
|
||||
handled(() => appleLLMCheck()),
|
||||
);
|
||||
|
||||
server.tool(
|
||||
"apple_llm_polish",
|
||||
"Polish a caption line/paragraph using Apple on-device 3B LLM ANE. From whisper-translation engine_apple_llm.py: same as AppleLLMPolish binary pipe. Fixes punctuation, casing, typos from live transcript. Mode: line (default) or paragraph. Reuses single warm session.",
|
||||
{
|
||||
text: z.string().min(1).describe("Text to polish (STT draft or final)."),
|
||||
prev1: z.string().optional().describe("Previous line for context (line mode)."),
|
||||
prev2: z.string().optional().describe("Second previous line."),
|
||||
mode: z.enum(["line", "paragraph"]).optional().describe("line (default) or paragraph."),
|
||||
},
|
||||
handled(({ text, prev1, prev2, mode }) => appleLLMPolish({ text, prev1, prev2, mode })),
|
||||
);
|
||||
|
||||
server.tool(
|
||||
"apple_llm_quick_reply",
|
||||
"INSTANT smart reply from live draft — for voice gateway. Feed the volatile draft transcript from speech_live_transcribe while user is speaking. Apple 3B ANE returns <20-word ACK preview in ~0.6s, before final transcription. Use this to provide almost immediate response when voice stops: send draft to quick_reply, get instant text to speak/display, then full Hermes turn in background. Reuses same ANE session as polish.",
|
||||
{
|
||||
draft: z.string().min(1).describe("Live draft transcript (partial, may have typos) from speech_live_transcribe. Used to infer intent for instant preview reply."),
|
||||
context: z.string().optional().describe("Optional previous conversation context or last assistant reply."),
|
||||
instructions: z.string().optional().describe("Optional system instructions for quick reply persona."),
|
||||
},
|
||||
handled(({ draft, context, instructions }) => appleLLMQuickReply({ draft, context, instructions })),
|
||||
);
|
||||
|
||||
server.tool(
|
||||
"apple_llm_chat",
|
||||
"Fast voice-assistant chat via Apple 3B ANE (FoundationModels). Under 40 words, warm, kid-safe, for ESP32. Use when you want Mac mini to directly answer draft/final without calling Hermes gateway. Accepts history array and current text. ~0.6-1.2s.",
|
||||
{
|
||||
text: z.string().min(1).describe("User message (draft or final transcript)."),
|
||||
history: z.array(z.object({ role: z.string(), text: z.string() })).optional().describe("Optional conversation history [{role, text}]."),
|
||||
instructions: z.string().optional().describe("Optional custom instructions."),
|
||||
},
|
||||
handled(({ text, history, instructions }) => appleLLMChat({ text, history, instructions })),
|
||||
);
|
||||
|
||||
server.tool(
|
||||
"apple_llm_status",
|
||||
"Show Apple LLM session status: active, ready, pid, lastActivity.",
|
||||
{},
|
||||
handled(() => appleLLMStatus()),
|
||||
);
|
||||
|
||||
server.tool(
|
||||
"apple_llm_close",
|
||||
"Close Apple LLM ANE session (frees Swift process + KV cache). Auto-closes after 2min idle anyway.",
|
||||
{},
|
||||
handled(() => appleLLMClose()),
|
||||
);
|
||||
|
||||
|
||||
return server;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user