feat: Apple on-device LLM (ANE) polish replaces Ollama as primary

- apple-llm-polish binary (120KB Swift, FoundationModels): --check
  now reports available:true after Apple Intelligence enabled,
  ping OK, line polish 258-388ms ANE vs Ollama 3-8s CPU
- engine_apple_llm.py: AppleLLM class with persistent pipe,
  thread reader, polish_line() + polish_paragraph(), JSONL protocol
  {id,mode,text,prev1,prev2,context} -> {id,text,ok,ms}
- engine_llm.py: try Apple ANE first (permissiveContentTransformations,
  temp 0.1 line / 0.2 para), log provider=apple apple_ms, fallback
  to Ollama on guardrail/timeout. Fixes missing LLM log — now
  logs provider=apple with ms.
- main_v3.py: --apple-llm / --no-apple-llm flag (on by default)
- Verified: direct polish 258ms, last log provider=apple ms=498,
  speech binary still 240KB with 31 word-by-word drafts

Co-authored-by: internal-model

apple_speech/Sources/AppleLLMPolish/main.swift

@@ -0,0 +1,228 @@
+import Foundation
+import FoundationModels
+
+// apple-llm-polish: pipe JSON {id?, mode: "line"|"paragraph", text, prev1?, prev2?, context?, language?} -> {id, text, ok, error?}
+// Uses Apple on-device 3B Foundation Model (SystemLanguageModel)
+// Much faster than Ollama, ANE-accelerated, offline
+
+struct InMsg: Decodable {
+    var id: String?
+    var mode: String // "line" | "paragraph" | "check" | "bench"
+    var text: String?
+    var prev1: String?
+    var prev2: String?
+    var context: String?
+    var prevSource: String?
+    var language: String?
+    var bench: String?
+}
+
+struct OutMsg: Encodable {
+    var id: String?
+    var ok: Bool
+    var text: String
+    var error: String?
+    var ms: Int?
+}
+
+func log(_ s: String) { fputs(s+"\n", stderr) }
+
+@main
+struct AppleLLMPolish {
+    static func main() async {
+        let args = CommandLine.arguments
+        if args.contains("--help") || args.contains("-h") {
+            fputs("Usage: apple-llm-polish [--check] [--bench \"text\"] [--pipe]\nPipe JSONL in stdin, JSONL out\n", stderr); exit(0)
+        }
+        if args.contains("--check") {
+            await runCheck(); return
+        }
+        if let idx = args.firstIndex(of: "--bench"), idx+1 < args.count {
+            await runBench(args[idx+1]); return
+        }
+        if args.contains("--bench-text") {
+            // bench via stdin text file? not needed
+        }
+        await runPipe()
+    }
+
+    static func runCheck() async {
+        let m = SystemLanguageModel.default
+        print("{\"available\": \(m.isAvailable), \"availability\": \"\(String(describing: m.availability))\"}")
+        if case .unavailable(let reason) = m.availability {
+            print("{\"reason\":\"\(String(describing: reason))\"}")
+        }
+        if m.isAvailable {
+            // quick ping
+            do {
+                let session = LanguageModelSession(model: m, instructions: "You are a concise assistant.")
+                let r = try await session.respond(to: "Say ok")
+                print("{\"ping\":\"\(r.content)\",\"ok\":true}")
+            } catch {
+                print("{\"ping_error\":\"\(error.localizedDescription)\",\"ok\":false}")
+            }
+        }
+    }
+
+    static func runBench(_ text: String) async {
+        let m = SystemLanguageModel.default
+        guard m.isAvailable else { print("{\"ok\":false,\"error\":\"model unavailable \(m.availability)\"}"); return }
+        let session = LanguageModelSession(model: m, instructions: instructionsFor(mode: "line", language: nil))
+        let t0 = Date()
+        do {
+            let r = try await session.respond(to: promptFor(mode: "line", text: text, prev1: "", prev2: "", context: nil, prevSource: nil, language: nil))
+            let ms = Int(Date().timeIntervalSince(t0)*1000)
+            print("{\"ok\":true,\"text\":\"\(r.content.replacingOccurrences(of: "\"", with: "\\\""))\",\"ms\":\(ms)}")
+        } catch {
+            print("{\"ok\":false,\"error\":\"\(error.localizedDescription)\"}")
+        }
+    }
+
+    static func runPipe() async {
+        let m = SystemLanguageModel.default
+        guard m.isAvailable else {
+            // emit errors for all input lines so caller doesn't hang
+            let reason = "\(m.availability)"
+            while let line = readLine() {
+                if line.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { continue }
+                var idv: String? = nil
+                if let data = line.data(using: .utf8), let dict = try? JSONSerialization.jsonObject(with: data) as? [String:Any] {
+                    idv = dict["id"] as? String
+                }
+                let out = OutMsg(id: idv, ok: false, text: "", error: "model unavailable: \(reason)", ms: nil)
+                if let d = try? JSONEncoder().encode(out), let s = String(data: d, encoding: .utf8) { print(s); fflush(stdout) }
+            }
+            return
+        }
+
+        // Create single session reused across requests (keeps KV cache warm)
+        let lineSession = LanguageModelSession(model: SystemLanguageModel(useCase: .general, guardrails: .permissiveContentTransformations), instructions: freeflowSystemPrompt(language: nil, mode: "line"))
+        let paraSession = LanguageModelSession(model: SystemLanguageModel(useCase: .general, guardrails: .permissiveContentTransformations), instructions: freeflowSystemPrompt(language: nil, mode: "paragraph"))
+        // Prewarm
+        lineSession.prewarm()
+        paraSession.prewarm()
+
+        log("[apple-llm-polish] ready, ANE-backed, model available")
+
+        while let line = readLine() {
+            let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines)
+            if trimmed.isEmpty { continue }
+            guard let data = line.data(using: .utf8) else { continue }
+            guard let req = try? JSONDecoder().decode(InMsg.self, from: data) else {
+                let out = OutMsg(id: nil, ok: false, text: "", error: "bad json", ms: nil)
+                if let d = try? JSONEncoder().encode(out), let s = String(data: d, encoding: .utf8) { print(s); fflush(stdout) }
+                continue
+            }
+            if req.mode == "check" {
+                let out = OutMsg(id: req.id, ok: m.isAvailable, text: "\(m.availability)", error: nil, ms: 0)
+                if let d = try? JSONEncoder().encode(out), let s = String(data: d, encoding: .utf8) { print(s); fflush(stdout) }
+                continue
+            }
+            let t0 = Date()
+            do {
+                let (session, prompt): (LanguageModelSession, String)
+                let lang = req.language
+                switch req.mode {
+                case "paragraph":
+                    // For paragraph we want deterministic polish first? We do full LLM here, but freeflow deterministic already did pre-clean.
+                    // Use freeflow system prompt
+                    session = paraSession
+                    prompt = buildParagraphPrompt(context: req.context ?? "", prevSource: req.prevSource ?? "", newText: req.text ?? "")
+                default: // line
+                    session = lineSession
+                    prompt = buildLinePrompt(text: req.text ?? "", prev1: req.prev1 ?? "", prev2: req.prev2 ?? "", language: lang)
+                }
+                // Generation options - low temp for deterministic polish
+                var opts = GenerationOptions()
+                opts.temperature = req.mode == "paragraph" ? 0.2 : 0.1
+                let resp = try await session.respond(to: prompt, options: opts)
+                let ms = Int(Date().timeIntervalSince(t0)*1000)
+                let cleaned = resp.content.trimmingCharacters(in: .whitespacesAndNewlines)
+                let out = OutMsg(id: req.id, ok: true, text: cleaned.isEmpty ? (req.text ?? "") : cleaned, error: nil, ms: ms)
+                if let d = try? JSONEncoder().encode(out), let s = String(data: d, encoding: .utf8) { print(s); fflush(stdout) }
+            } catch {
+                let ms = Int(Date().timeIntervalSince(t0)*1000)
+                let err = error.localizedDescription
+                // guardrail or unavailable -> fallback to original text so pipeline doesn't break
+                let fallback = req.text ?? ""
+                let out = OutMsg(id: req.id, ok: false, text: fallback, error: err, ms: ms)
+                if let d = try? JSONEncoder().encode(out), let s = String(data: d, encoding: .utf8) { print(s); fflush(stdout) }
+            }
+        }
+    }
+
+    // MARK: - Prompts (ported from freeflow_polish.py + engine_llm.py freeflow style)
+
+    static func freeflowSystemPrompt(language: String?, mode: String) -> String {
+        if mode == "paragraph" {
+            return """
+            You are a careful live transcript editor working from noisy translated source text.
+
+            Goal:
+            Produce the most faithful readable English update.
+
+            Decision rules:
+            1. NEW SOURCE TEXT is the primary evidence and should dominate the output.
+            2. Use PREVIOUS SOURCE TEXT and PREVIOUS REFINED CONTEXT only when they clearly help resolve or continue the new material.
+            3. The final answer must be English only.
+            4. Never copy non-English words into the output unless they are proper names.
+            5. If a non-English fragment appears as an isolated clause, trailing fragment, or side comment without a clear English anchor, drop it.
+            6. Only translate a non-English fragment when it is clearly central to the same idea and its meaning is reasonably obvious from context.
+            7. If the new material starts a fresh thought, output only the new material.
+            8. Remove exact or near-exact repetition unless the repetition is clearly intentional rhetoric.
+            9. Do not add explanations, disclaimers, or meta commentary.
+            10. Prefer conservative wording over guessed meaning.
+            11. Return only the revised transcript text.
+            """
+        } else {
+            // line
+            return """
+            You are a real-time caption polisher.
+
+            Task: Polish the current caption line to be readable, preserving meaning exactly.
+
+            Rules:
+            - Fix punctuation, casing, obvious STT typos.
+            - Expand spoken punctuation like <keep>comma</keep> already expanded upstream.
+            - Remove filler words (uh, um, erm) if clearly disfluency.
+            - Keep all semantic content; do not rewrite with prior context.
+            - If uncertain, return original.
+            - Output only one polished line, no explanation.
+            """
+        }
+    }
+
+    static func buildLinePrompt(text: String, prev1: String, prev2: String, language: String?) -> String {
+        var p = ""
+        if !prev2.isEmpty { p += "Previous line 2: \(prev2)\n" }
+        if !prev1.isEmpty { p += "Previous line 1: \(prev1)\n" }
+        p += "Current line to polish:\n\(text)\n\nPolished:"
+        return p
+    }
+
+    static func buildParagraphPrompt(context: String, prevSource: String, newText: String) -> String {
+        return """
+        Edit this transcript update.
+        [PREVIOUS REFINED CONTEXT]
+        \(context)
+
+        [PREVIOUS SOURCE TEXT]
+        \(prevSource)
+
+        [NEW SOURCE TEXT]
+        \(newText)
+        """
+    }
+
+    static func instructionsFor(mode: String, language: String?) -> String {
+        return freeflowSystemPrompt(language: language, mode: mode)
+    }
+
+    static func promptFor(mode: String, text: String, prev1: String, prev2: String, context: String?, prevSource: String?, language: String?) -> String {
+        if mode == "paragraph" {
+            return buildParagraphPrompt(context: context ?? "", prevSource: prevSource ?? "", newText: text)
+        } else {
+            return buildLinePrompt(text: text, prev1: prev1, prev2: prev2, language: language)
+        }
+    }
+}

apple_speech/build_llm.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -e
+cd "$(dirname "$0")"
+SDK="/Library/Developer/CommandLineTools/SDKs/MacOSX26.5.sdk"
+if [ ! -d "$SDK" ]; then SDK="/Library/Developer/CommandLineTools/SDKs/MacOSX26.sdk"; fi
+mkdir -p .build/release
+echo "Compiling apple-llm-polish with swiftc..."
+swiftc -O -parse-as-library -target arm64-apple-macosx26.0 -sdk "$SDK" \
+  Sources/AppleLLMPolish/main.swift -o .build/release/apple-llm-polish \
+  -framework Foundation -framework FoundationModels
+
+echo "Built: .build/release/apple-llm-polish"
+ls -lh .build/release/apple-llm-polish
+
+# check
+echo "=== check ==="
+.build/release/apple-llm-polish --check 2>&1 | head -20
+
+echo "Copying to root copies..."
+cp .build/release/apple-llm-polish ../apple-llm-polish 2>/dev/null || true
+cp .build/release/apple-llm-polish ./apple-llm-polish 2>/dev/null || true
+echo "Done"