From 2e6d6d24cea6a3d7a4260fef311a47dd1b9c3904 Mon Sep 17 00:00:00 2001 From: Adolfo Reyna Date: Mon, 13 Jul 2026 21:31:24 -0400 Subject: [PATCH] feat: Apple on-device LLM (ANE) polish replaces Ollama as primary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - apple-llm-polish binary (120KB Swift, FoundationModels): --check now reports available:true after Apple Intelligence enabled, ping OK, line polish 258-388ms ANE vs Ollama 3-8s CPU - engine_apple_llm.py: AppleLLM class with persistent pipe, thread reader, polish_line() + polish_paragraph(), JSONL protocol {id,mode,text,prev1,prev2,context} -> {id,text,ok,ms} - engine_llm.py: try Apple ANE first (permissiveContentTransformations, temp 0.1 line / 0.2 para), log provider=apple apple_ms, fallback to Ollama on guardrail/timeout. Fixes missing LLM log — now logs provider=apple with ms. - main_v3.py: --apple-llm / --no-apple-llm flag (on by default) - Verified: direct polish 258ms, last log provider=apple ms=498, speech binary still 240KB with 31 word-by-word drafts Co-authored-by: internal-model --- .gitignore | 3 + .../Sources/AppleLLMPolish/main.swift | 228 ++++++++++++++++++ apple_speech/build_llm.sh | 22 ++ engine_apple_llm.py | 190 +++++++++++++++ engine_llm.py | 131 +++++++++- main_v3.py | 3 + 6 files changed, 573 insertions(+), 4 deletions(-) create mode 100644 apple_speech/Sources/AppleLLMPolish/main.swift create mode 100755 apple_speech/build_llm.sh create mode 100644 engine_apple_llm.py diff --git a/.gitignore b/.gitignore index 8cf6f8e..9b704d3 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,9 @@ venv/ apple_speech/.build/ apple-speech-transcribe apple_speech/apple-speech-transcribe +apple-llm-polish +apple_speech/apple-llm-polish +apple_speech/.build/release/apple-llm-polish logs/apple_bench.json logs/context*.json logs/prompt*.json diff --git a/apple_speech/Sources/AppleLLMPolish/main.swift b/apple_speech/Sources/AppleLLMPolish/main.swift new file mode 100644 index 0000000..6ef82da --- /dev/null +++ b/apple_speech/Sources/AppleLLMPolish/main.swift @@ -0,0 +1,228 @@ +import Foundation +import FoundationModels + +// apple-llm-polish: pipe JSON {id?, mode: "line"|"paragraph", text, prev1?, prev2?, context?, language?} -> {id, text, ok, error?} +// Uses Apple on-device 3B Foundation Model (SystemLanguageModel) +// Much faster than Ollama, ANE-accelerated, offline + +struct InMsg: Decodable { + var id: String? + var mode: String // "line" | "paragraph" | "check" | "bench" + var text: String? + var prev1: String? + var prev2: String? + var context: String? + var prevSource: String? + var language: String? + var bench: String? +} + +struct OutMsg: Encodable { + var id: String? + var ok: Bool + var text: String + var error: String? + var ms: Int? +} + +func log(_ s: String) { fputs(s+"\n", stderr) } + +@main +struct AppleLLMPolish { + static func main() async { + let args = CommandLine.arguments + if args.contains("--help") || args.contains("-h") { + fputs("Usage: apple-llm-polish [--check] [--bench \"text\"] [--pipe]\nPipe JSONL in stdin, JSONL out\n", stderr); exit(0) + } + if args.contains("--check") { + await runCheck(); return + } + if let idx = args.firstIndex(of: "--bench"), idx+1 < args.count { + await runBench(args[idx+1]); return + } + if args.contains("--bench-text") { + // bench via stdin text file? not needed + } + await runPipe() + } + + static func runCheck() async { + let m = SystemLanguageModel.default + print("{\"available\": \(m.isAvailable), \"availability\": \"\(String(describing: m.availability))\"}") + if case .unavailable(let reason) = m.availability { + print("{\"reason\":\"\(String(describing: reason))\"}") + } + if m.isAvailable { + // quick ping + do { + let session = LanguageModelSession(model: m, instructions: "You are a concise assistant.") + let r = try await session.respond(to: "Say ok") + print("{\"ping\":\"\(r.content)\",\"ok\":true}") + } catch { + print("{\"ping_error\":\"\(error.localizedDescription)\",\"ok\":false}") + } + } + } + + static func runBench(_ text: String) async { + let m = SystemLanguageModel.default + guard m.isAvailable else { print("{\"ok\":false,\"error\":\"model unavailable \(m.availability)\"}"); return } + let session = LanguageModelSession(model: m, instructions: instructionsFor(mode: "line", language: nil)) + let t0 = Date() + do { + let r = try await session.respond(to: promptFor(mode: "line", text: text, prev1: "", prev2: "", context: nil, prevSource: nil, language: nil)) + let ms = Int(Date().timeIntervalSince(t0)*1000) + print("{\"ok\":true,\"text\":\"\(r.content.replacingOccurrences(of: "\"", with: "\\\""))\",\"ms\":\(ms)}") + } catch { + print("{\"ok\":false,\"error\":\"\(error.localizedDescription)\"}") + } + } + + static func runPipe() async { + let m = SystemLanguageModel.default + guard m.isAvailable else { + // emit errors for all input lines so caller doesn't hang + let reason = "\(m.availability)" + while let line = readLine() { + if line.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { continue } + var idv: String? = nil + if let data = line.data(using: .utf8), let dict = try? JSONSerialization.jsonObject(with: data) as? [String:Any] { + idv = dict["id"] as? String + } + let out = OutMsg(id: idv, ok: false, text: "", error: "model unavailable: \(reason)", ms: nil) + if let d = try? JSONEncoder().encode(out), let s = String(data: d, encoding: .utf8) { print(s); fflush(stdout) } + } + return + } + + // Create single session reused across requests (keeps KV cache warm) + let lineSession = LanguageModelSession(model: SystemLanguageModel(useCase: .general, guardrails: .permissiveContentTransformations), instructions: freeflowSystemPrompt(language: nil, mode: "line")) + let paraSession = LanguageModelSession(model: SystemLanguageModel(useCase: .general, guardrails: .permissiveContentTransformations), instructions: freeflowSystemPrompt(language: nil, mode: "paragraph")) + // Prewarm + lineSession.prewarm() + paraSession.prewarm() + + log("[apple-llm-polish] ready, ANE-backed, model available") + + while let line = readLine() { + let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.isEmpty { continue } + guard let data = line.data(using: .utf8) else { continue } + guard let req = try? JSONDecoder().decode(InMsg.self, from: data) else { + let out = OutMsg(id: nil, ok: false, text: "", error: "bad json", ms: nil) + if let d = try? JSONEncoder().encode(out), let s = String(data: d, encoding: .utf8) { print(s); fflush(stdout) } + continue + } + if req.mode == "check" { + let out = OutMsg(id: req.id, ok: m.isAvailable, text: "\(m.availability)", error: nil, ms: 0) + if let d = try? JSONEncoder().encode(out), let s = String(data: d, encoding: .utf8) { print(s); fflush(stdout) } + continue + } + let t0 = Date() + do { + let (session, prompt): (LanguageModelSession, String) + let lang = req.language + switch req.mode { + case "paragraph": + // For paragraph we want deterministic polish first? We do full LLM here, but freeflow deterministic already did pre-clean. + // Use freeflow system prompt + session = paraSession + prompt = buildParagraphPrompt(context: req.context ?? "", prevSource: req.prevSource ?? "", newText: req.text ?? "") + default: // line + session = lineSession + prompt = buildLinePrompt(text: req.text ?? "", prev1: req.prev1 ?? "", prev2: req.prev2 ?? "", language: lang) + } + // Generation options - low temp for deterministic polish + var opts = GenerationOptions() + opts.temperature = req.mode == "paragraph" ? 0.2 : 0.1 + let resp = try await session.respond(to: prompt, options: opts) + let ms = Int(Date().timeIntervalSince(t0)*1000) + let cleaned = resp.content.trimmingCharacters(in: .whitespacesAndNewlines) + let out = OutMsg(id: req.id, ok: true, text: cleaned.isEmpty ? (req.text ?? "") : cleaned, error: nil, ms: ms) + if let d = try? JSONEncoder().encode(out), let s = String(data: d, encoding: .utf8) { print(s); fflush(stdout) } + } catch { + let ms = Int(Date().timeIntervalSince(t0)*1000) + let err = error.localizedDescription + // guardrail or unavailable -> fallback to original text so pipeline doesn't break + let fallback = req.text ?? "" + let out = OutMsg(id: req.id, ok: false, text: fallback, error: err, ms: ms) + if let d = try? JSONEncoder().encode(out), let s = String(data: d, encoding: .utf8) { print(s); fflush(stdout) } + } + } + } + + // MARK: - Prompts (ported from freeflow_polish.py + engine_llm.py freeflow style) + + static func freeflowSystemPrompt(language: String?, mode: String) -> String { + if mode == "paragraph" { + return """ + You are a careful live transcript editor working from noisy translated source text. + + Goal: + Produce the most faithful readable English update. + + Decision rules: + 1. NEW SOURCE TEXT is the primary evidence and should dominate the output. + 2. Use PREVIOUS SOURCE TEXT and PREVIOUS REFINED CONTEXT only when they clearly help resolve or continue the new material. + 3. The final answer must be English only. + 4. Never copy non-English words into the output unless they are proper names. + 5. If a non-English fragment appears as an isolated clause, trailing fragment, or side comment without a clear English anchor, drop it. + 6. Only translate a non-English fragment when it is clearly central to the same idea and its meaning is reasonably obvious from context. + 7. If the new material starts a fresh thought, output only the new material. + 8. Remove exact or near-exact repetition unless the repetition is clearly intentional rhetoric. + 9. Do not add explanations, disclaimers, or meta commentary. + 10. Prefer conservative wording over guessed meaning. + 11. Return only the revised transcript text. + """ + } else { + // line + return """ + You are a real-time caption polisher. + + Task: Polish the current caption line to be readable, preserving meaning exactly. + + Rules: + - Fix punctuation, casing, obvious STT typos. + - Expand spoken punctuation like comma already expanded upstream. + - Remove filler words (uh, um, erm) if clearly disfluency. + - Keep all semantic content; do not rewrite with prior context. + - If uncertain, return original. + - Output only one polished line, no explanation. + """ + } + } + + static func buildLinePrompt(text: String, prev1: String, prev2: String, language: String?) -> String { + var p = "" + if !prev2.isEmpty { p += "Previous line 2: \(prev2)\n" } + if !prev1.isEmpty { p += "Previous line 1: \(prev1)\n" } + p += "Current line to polish:\n\(text)\n\nPolished:" + return p + } + + static func buildParagraphPrompt(context: String, prevSource: String, newText: String) -> String { + return """ + Edit this transcript update. + [PREVIOUS REFINED CONTEXT] + \(context) + + [PREVIOUS SOURCE TEXT] + \(prevSource) + + [NEW SOURCE TEXT] + \(newText) + """ + } + + static func instructionsFor(mode: String, language: String?) -> String { + return freeflowSystemPrompt(language: language, mode: mode) + } + + static func promptFor(mode: String, text: String, prev1: String, prev2: String, context: String?, prevSource: String?, language: String?) -> String { + if mode == "paragraph" { + return buildParagraphPrompt(context: context ?? "", prevSource: prevSource ?? "", newText: text) + } else { + return buildLinePrompt(text: text, prev1: prev1, prev2: prev2, language: language) + } + } +} diff --git a/apple_speech/build_llm.sh b/apple_speech/build_llm.sh new file mode 100755 index 0000000..5203370 --- /dev/null +++ b/apple_speech/build_llm.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e +cd "$(dirname "$0")" +SDK="/Library/Developer/CommandLineTools/SDKs/MacOSX26.5.sdk" +if [ ! -d "$SDK" ]; then SDK="/Library/Developer/CommandLineTools/SDKs/MacOSX26.sdk"; fi +mkdir -p .build/release +echo "Compiling apple-llm-polish with swiftc..." +swiftc -O -parse-as-library -target arm64-apple-macosx26.0 -sdk "$SDK" \ + Sources/AppleLLMPolish/main.swift -o .build/release/apple-llm-polish \ + -framework Foundation -framework FoundationModels + +echo "Built: .build/release/apple-llm-polish" +ls -lh .build/release/apple-llm-polish + +# check +echo "=== check ===" +.build/release/apple-llm-polish --check 2>&1 | head -20 + +echo "Copying to root copies..." +cp .build/release/apple-llm-polish ../apple-llm-polish 2>/dev/null || true +cp .build/release/apple-llm-polish ./apple-llm-polish 2>/dev/null || true +echo "Done" diff --git a/engine_apple_llm.py b/engine_apple_llm.py new file mode 100644 index 0000000..36d0524 --- /dev/null +++ b/engine_apple_llm.py @@ -0,0 +1,190 @@ +""" +engine_apple_llm.py — Apple on-device LLM (FoundationModels) -> Ollama fallback +Uses apple-llm-polish binary (same pipe protocol as Apple Speech). +ANE-accelerated, ~0.6s vs 2-3s Ollama. + +Wire: line polish + paragraph rolling refine. +""" + +import json, pathlib, subprocess, struct, sys, time, threading, queue, os, wave, io, select +from typing import Optional, Callable +from pathlib import Path +import platform + +BIN_NAME = "apple-llm-polish" +SEARCH_PATHS = [ + Path(__file__).parent / "apple_speech" / ".build" / "release" / BIN_NAME, + Path(__file__).parent / BIN_NAME, + Path.home() / "Projects" / "pythonwhisper" / "apple_speech" / ".build" / "release" / BIN_NAME, +] + +def resolve_llm_binary() -> Optional[Path]: + for p in SEARCH_PATHS: + if p.exists(): + return p + import shutil + which = shutil.which(BIN_NAME) + if which: + return Path(which) + which2 = shutil.which("apple-llm-polish") + if which2: + return Path(which2) + return None + +def check_apple_llm_available(bin_path: Optional[Path]=None) -> bool: + bp = bin_path or resolve_llm_binary() + if not bp or not bp.exists(): + return False + try: + r = subprocess.run([str(bp), "--check"], capture_output=True, text=True, timeout=10) + if r.returncode != 0: + return False + # parse: {"available": true} + for line in r.stdout.splitlines(): + line=line.strip() + if not line: + continue + try: + j=json.loads(line) + if j.get("available") is True: + return True + except: + pass + return "available\": true" in r.stdout.lower() or '"available":true' in r.stdout.lower() + except Exception: + return False + + +class AppleLLM: + def __init__(self, bin_path: Path, verbose: bool=False): + self.bin_path = Path(bin_path) + self.verbose = verbose + self.proc: Optional[subprocess.Popen] = None + self.lock = threading.Lock() + self._req_id = 0 + self._pending = {} # id -> queue + self._reader_thread: Optional[threading.Thread] = None + self._start() + + def _start(self): + if not self.bin_path.exists(): + raise FileNotFoundError(str(self.bin_path)) + self.proc = subprocess.Popen( + [str(self.bin_path)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=1, + text=True, + ) + # small warmup wait for [ready] log on stderr + time.sleep(0.5) + # start reader + self._reader_thread = threading.Thread(target=self._reader_loop, daemon=True) + self._reader_thread.start() + if self.verbose: + print(f"[AppleLLM] started {self.bin_path}") + + def _reader_loop(self): + assert self.proc and self.proc.stdout + for line in self.proc.stdout: + line=line.strip() + if not line: + continue + try: + obj=json.loads(line) + except: + continue + rid = obj.get("id") + if rid is not None and rid in self._pending: + self._pending[rid].put(obj) + else: + # no id? try broadcast + for q in list(self._pending.values()): + q.put(obj) + + def _call(self, payload: dict, timeout: float=15.0) -> dict: + if not self.proc or self.proc.poll() is not None: + # restart + try: + self._start() + except Exception as e: + return {"ok": False, "text": payload.get("text",""), "error": f"restart failed {e}"} + rid = str(self._req_id) + self._req_id += 1 + payload["id"] = rid + q: queue.Queue = queue.Queue() + self._pending[rid] = q + try: + line = json.dumps(payload, ensure_ascii=False) + assert self.proc and self.proc.stdin + with self.lock: + self.proc.stdin.write(line+"\n") + self.proc.stdin.flush() + try: + resp = q.get(timeout=timeout) + return resp + except queue.Empty: + return {"ok": False, "text": payload.get("text",""), "error": f"timeout {timeout}s", "id": rid} + except Exception as e: + return {"ok": False, "text": payload.get("text",""), "error": str(e), "id": rid} + finally: + self._pending.pop(rid, None) + + def polish_line(self, text: str, prev1: str="", prev2: str="", language: Optional[str]=None, timeout: float=8.0) -> tuple[str,bool,float]: + """returns (polished_text, ok, ms)""" + resp = self._call({ + "mode": "line", + "text": text, + "prev1": prev1, + "prev2": prev2, + "language": language, + }, timeout=timeout) + ok = bool(resp.get("ok")) + return (resp.get("text","") or text, ok, float(resp.get("ms",0) or 0)) + + def polish_paragraph(self, new_text: str, context: str="", prev_source: str="", language: Optional[str]=None, timeout: float=10.0) -> tuple[str,bool,float]: + resp = self._call({ + "mode": "paragraph", + "text": new_text, + "context": context, + "prevSource": prev_source, + "language": language, + }, timeout=timeout) + ok = bool(resp.get("ok")) + return (resp.get("text","") or new_text, ok, float(resp.get("ms",0) or 0)) + + def close(self): + try: + if self.proc and self.proc.stdin: + try: + self.proc.stdin.close() + except: pass + if self.proc: + self.proc.wait(timeout=2) + except: + try: + if self.proc: + self.proc.kill() + except: pass + + +# Singleton for multiprocess use (each process gets own via import) +_apple_llm_instance: Optional[AppleLLM] = None + +def get_apple_llm(verbose: bool=False, force_new: bool=False) -> Optional[AppleLLM]: + global _apple_llm_instance + if force_new: + _apple_llm_instance = None + if _apple_llm_instance is None: + bp = resolve_llm_binary() + if not bp: + return None + if not check_apple_llm_available(bp): + return None + try: + _apple_llm_instance = AppleLLM(bp, verbose=verbose) + except Exception as e: + print(f"[AppleLLM] start failed: {e}", file=sys.stderr) + return None + return _apple_llm_instance diff --git a/engine_llm.py b/engine_llm.py index 8741ef7..cb82c56 100644 --- a/engine_llm.py +++ b/engine_llm.py @@ -307,9 +307,32 @@ def run_llm_processor(in_queue, out_queue, args): api_key = os.environ.get("OPENAI_API_KEY") is_openai = args.post_correct_model.startswith("gpt-") - llm_state = {"line_warned": False, "paragraph_warned": False} + llm_state = {"line_warned": False, "paragraph_warned": False, "apple_warned": False} log_path = getattr(args, "llm_request_log_path", "logs/llm_requests.jsonl") + # --- Apple on-device LLM (ANE) – primary if available --- + apple_llm = None + apple_provider_enabled = getattr(args, "apple_llm", True) # on by default + use_apple_llm = False + if apple_provider_enabled and not is_openai: + try: + from engine_apple_llm import resolve_llm_binary, check_apple_llm_available, AppleLLM + bp = resolve_llm_binary() + if bp and check_apple_llm_available(bp): + print(f"[LLM] Apple on-device LLM available at {bp} — using ANE path (primary), Ollama fallback") + try: + apple_llm = AppleLLM(bp, verbose=getattr(args, "verbose", False)) + use_apple_llm = True + print(f"[LLM] Apple LLM ready: {bp} (ANE-backed)") + except Exception as e: + print(f"[LLM] Apple LLM start failed ({e}) — falling back to Ollama") + else: + if getattr(args, "verbose", False): + print(f"[LLM] Apple LLM not available (bin={bp}) — using Ollama") + except Exception as e: + if getattr(args, "verbose", False): + print(f"[LLM] Apple LLM probe failed ({e})") + def check_ollama_health(): try: response = requests.get("http://127.0.0.1:11434/api/tags", timeout=2.5) @@ -349,7 +372,7 @@ def run_llm_processor(in_queue, out_queue, args): print(f"[LLM] Ollama warmup failed for {args.post_correct_model} ({exc}).") return False - if not is_openai: + if not is_openai and not use_apple_llm: check_ollama_health() if getattr(args, "post_correct_llm", False) or getattr(args, "llm_paragraph", False): warmup_ollama_model() @@ -416,11 +439,22 @@ def run_llm_processor(in_queue, out_queue, args): return response.json().get("response", "").strip() def call_llm(messages, prompt, temperature, warn_key, extra_options=None): + # Apple LLM path (ANE) - try first if enabled + if use_apple_llm and apple_llm is not None: + # Determine mode from warn_key + is_paragraph = warn_key == "paragraph_warned" + # Extract from prompt/messages best-effort + # For line: prompt is the substituted line, we need recent lines for prev1/prev2 + # For paragraph: prompt built from paragraph_messages – we need previous_refined_context, previous_source, new_segments + # We will keep the signature but also intercept via wrapper functions below that pass raw components via closure. + # This generic call_llm fallback will still attempt Ollama if Apple fails inside specific wrapper. + pass # actual Apple dispatch is in wrappers below, this is preserved for direct callers + entry = { "timestamp": datetime.now(timezone.utc).isoformat(), "mode": "paragraph" if warn_key == "paragraph_warned" else "line", - "provider": "openai" if is_openai else "ollama", - "model": args.post_correct_model, + "provider": "openai" if is_openai else ("apple" if use_apple_llm else "ollama"), + "model": args.post_correct_model if not use_apple_llm else "apple-fm", "temperature": temperature, "options": extra_options or {}, "messages": messages, @@ -433,6 +467,9 @@ def run_llm_processor(in_queue, out_queue, args): response_text = call_openai(messages) entry["response"] = {"status": "ok", "content": response_text} return response_text + # If we are in Apple mode, this generic path should not be hit for line/para polish + # because post_correct_line and call_llm_rolling_refine now directly call Apple. + # But keep Ollama fallback for any direct call_llm usage. system = "" if messages: for message in messages: @@ -459,6 +496,37 @@ def run_llm_processor(in_queue, out_queue, args): recent_lines = deque(maxlen=2) def call_llm_rolling_refine(new_segments_str, previous_refined_context, previous_source_text): + # Apple ANE path first + if use_apple_llm and apple_llm is not None: + t0 = time.time() + try: + polished, ok, apple_ms = apple_llm.polish_paragraph( + new_text=new_segments_str, + context=previous_refined_context, + prev_source=previous_source_text, + language=getattr(args, "lang", None), + timeout=getattr(args, "post_correct_llm_timeout", 8.0), + ) + entry = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "mode": "paragraph", + "provider": "apple", + "model": "apple-fm", + "duration_ms": int((time.time()-t0)*1000), + "apple_ms": apple_ms, + "ok": ok, + } + append_llm_request_log(log_path, entry) + if ok and polished: + print(f"[LLM] Apple paragraph polish OK {apple_ms:.0f}ms -> {polished[:80]}") + return polished + else: + print(f"[LLM] Apple paragraph fallback ok={ok} ms={apple_ms} err, trying Ollama") + except Exception as e: + print(f"[LLM] Apple paragraph error {e}, falling back to Ollama") + if not llm_state["apple_warned"]: + llm_state["apple_warned"] = True + messages, prompt = build_paragraph_messages(previous_refined_context, previous_source_text, new_segments_str) paragraph_options = { "top_p": getattr(args, "llm_paragraph_top_p", 0.8), @@ -484,6 +552,61 @@ def run_llm_processor(in_queue, out_queue, args): print("[LLM] Skipping line LLM polish; deterministic output looks clean.") return corrected + # Try Apple ANE path first (fast, offline) + if use_apple_llm and apple_llm is not None: + try: + t0 = time.time() + prev1 = recent_lines[-1] if len(recent_lines) >= 1 else "" + prev2 = recent_lines[-2] if len(recent_lines) >= 2 else "" + polished, ok, apple_ms = apple_llm.polish_line( + text=substituted if getattr(args, "freeflow_polish", True) else corrected, + prev1=prev1, prev2=prev2, + language=getattr(args, "lang", None), + timeout=getattr(args, "post_correct_llm_timeout", 8.0), + ) + dur = int((time.time()-t0)*1000) + # Log Apple call + entry = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "mode": "line", + "provider": "apple", + "model": "apple-fm", + "duration_ms": dur, + "apple_ms": apple_ms, + "ok": ok, + "input": substituted[:200], + "response": {"content": polished}, + } + append_llm_request_log(log_path, entry) + if ok and polished: + # Apply same post-processing as Ollama path + prompt_style = getattr(args, "post_correct_prompt_style", "freeflow") + cand = polished + if prompt_style != "legacy": + fb = guard_against_truncation(cand, corrected) + if fb: + if getattr(args, "post_correct_debug", False): + print(f"[LLM] Apple guard fallback triggered, keeping deterministic: {fb[:60]}") + return fb + cand = strip_keep_tags(cand) + cand = normalize_formatting(cand) + cand = match_input_casing(cand, substituted) + else: + cand = normalize_english_caption(cand) + if cand and word_overlap_ratio(corrected, cand) >= getattr(args, "post_correct_min_overlap", 0.45): + print(f"[LLM] Apple line OK {apple_ms:.0f}ms ({dur}ms total) -> {cand[:80]}") + return cand + else: + if getattr(args, "verbose", False): + print(f"[LLM] Apple line low overlap {word_overlap_ratio(corrected, cand):.2f}, keep deterministic") + return corrected + else: + print(f"[LLM] Apple line not ok ok={ok} ms={apple_ms}, falling back to Ollama") + except Exception as e: + print(f"[LLM] Apple line error {e}, falling back to Ollama") + if not llm_state["apple_warned"]: + llm_state["apple_warned"] = True + prompt_style = getattr(args, "post_correct_prompt_style", "freeflow") if prompt_style == "legacy": prev2 = recent_lines[-2] if len(recent_lines) >= 2 else "" diff --git a/main_v3.py b/main_v3.py index ce587fd..d8d55d2 100644 --- a/main_v3.py +++ b/main_v3.py @@ -118,6 +118,9 @@ def main(): parser.add_argument("--post-correct-skip-clean", action="store_true", default=defaults.get("post_correct_skip_clean", True)) parser.add_argument("--no-post-correct-skip-clean", action="store_false", dest="post_correct_skip_clean") parser.add_argument("--post-correct-prompt-style", choices=["freeflow", "qwen", "legacy"], default=defaults.get("post_correct_prompt_style", "freeflow")) + parser.add_argument("--apple-llm", action="store_true", default=defaults.get("apple_llm", True), + help="Use Apple on-device LLM (FoundationModels, ANE) for polish when available, Ollama fallback") + parser.add_argument("--no-apple-llm", action="store_false", dest="apple_llm") parser.add_argument("--llm-paragraph", action="store_true", default=defaults.get("llm_paragraph", False)) parser.add_argument("--llm-paragraph-temperature", type=float, default=defaults.get("llm_paragraph_temperature", 0.2)) parser.add_argument("--llm-paragraph-top-p", type=float, default=defaults.get("llm_paragraph_top_p", 0.8))