feat: Apple on-device LLM (ANE) polish replaces Ollama as primary

- apple-llm-polish binary (120KB Swift, FoundationModels): --check
  now reports available:true after Apple Intelligence enabled,
  ping OK, line polish 258-388ms ANE vs Ollama 3-8s CPU
- engine_apple_llm.py: AppleLLM class with persistent pipe,
  thread reader, polish_line() + polish_paragraph(), JSONL protocol
  {id,mode,text,prev1,prev2,context} -> {id,text,ok,ms}
- engine_llm.py: try Apple ANE first (permissiveContentTransformations,
  temp 0.1 line / 0.2 para), log provider=apple apple_ms, fallback
  to Ollama on guardrail/timeout. Fixes missing LLM log — now
  logs provider=apple with ms.
- main_v3.py: --apple-llm / --no-apple-llm flag (on by default)
- Verified: direct polish 258ms, last log provider=apple ms=498,
  speech binary still 240KB with 31 word-by-word drafts

Co-authored-by: internal-model

engine_llm.py

@@ -307,9 +307,32 @@ def run_llm_processor(in_queue, out_queue, args):
     
     api_key = os.environ.get("OPENAI_API_KEY")
     is_openai = args.post_correct_model.startswith("gpt-")
-    llm_state = {"line_warned": False, "paragraph_warned": False}
+    llm_state = {"line_warned": False, "paragraph_warned": False, "apple_warned": False}
     log_path = getattr(args, "llm_request_log_path", "logs/llm_requests.jsonl")
 
+    # --- Apple on-device LLM (ANE) – primary if available ---
+    apple_llm = None
+    apple_provider_enabled = getattr(args, "apple_llm", True)  # on by default
+    use_apple_llm = False
+    if apple_provider_enabled and not is_openai:
+        try:
+            from engine_apple_llm import resolve_llm_binary, check_apple_llm_available, AppleLLM
+            bp = resolve_llm_binary()
+            if bp and check_apple_llm_available(bp):
+                print(f"[LLM] Apple on-device LLM available at {bp} — using ANE path (primary), Ollama fallback")
+                try:
+                    apple_llm = AppleLLM(bp, verbose=getattr(args, "verbose", False))
+                    use_apple_llm = True
+                    print(f"[LLM] Apple LLM ready: {bp} (ANE-backed)")
+                except Exception as e:
+                    print(f"[LLM] Apple LLM start failed ({e}) — falling back to Ollama")
+            else:
+                if getattr(args, "verbose", False):
+                    print(f"[LLM] Apple LLM not available (bin={bp}) — using Ollama")
+        except Exception as e:
+            if getattr(args, "verbose", False):
+                print(f"[LLM] Apple LLM probe failed ({e})")
+
     def check_ollama_health():
         try:
             response = requests.get("http://127.0.0.1:11434/api/tags", timeout=2.5)
@@ -349,7 +372,7 @@ def run_llm_processor(in_queue, out_queue, args):
             print(f"[LLM] Ollama warmup failed for {args.post_correct_model} ({exc}).")
         return False
 
-    if not is_openai:
+    if not is_openai and not use_apple_llm:
         check_ollama_health()
         if getattr(args, "post_correct_llm", False) or getattr(args, "llm_paragraph", False):
             warmup_ollama_model()
@@ -416,11 +439,22 @@ def run_llm_processor(in_queue, out_queue, args):
         return response.json().get("response", "").strip()
 
     def call_llm(messages, prompt, temperature, warn_key, extra_options=None):
+        # Apple LLM path (ANE) - try first if enabled
+        if use_apple_llm and apple_llm is not None:
+            # Determine mode from warn_key
+            is_paragraph = warn_key == "paragraph_warned"
+            # Extract from prompt/messages best-effort
+            # For line: prompt is the substituted line, we need recent lines for prev1/prev2
+            # For paragraph: prompt built from paragraph_messages – we need previous_refined_context, previous_source, new_segments
+            # We will keep the signature but also intercept via wrapper functions below that pass raw components via closure.
+            # This generic call_llm fallback will still attempt Ollama if Apple fails inside specific wrapper.
+            pass  # actual Apple dispatch is in wrappers below, this is preserved for direct callers
+
         entry = {
             "timestamp": datetime.now(timezone.utc).isoformat(),
             "mode": "paragraph" if warn_key == "paragraph_warned" else "line",
-            "provider": "openai" if is_openai else "ollama",
-            "model": args.post_correct_model,
+            "provider": "openai" if is_openai else ("apple" if use_apple_llm else "ollama"),
+            "model": args.post_correct_model if not use_apple_llm else "apple-fm",
             "temperature": temperature,
             "options": extra_options or {},
             "messages": messages,
@@ -433,6 +467,9 @@ def run_llm_processor(in_queue, out_queue, args):
                 response_text = call_openai(messages)
                 entry["response"] = {"status": "ok", "content": response_text}
                 return response_text
+            # If we are in Apple mode, this generic path should not be hit for line/para polish
+            # because post_correct_line and call_llm_rolling_refine now directly call Apple.
+            # But keep Ollama fallback for any direct call_llm usage.
             system = ""
             if messages:
                 for message in messages:
@@ -459,6 +496,37 @@ def run_llm_processor(in_queue, out_queue, args):
     recent_lines = deque(maxlen=2)
     
     def call_llm_rolling_refine(new_segments_str, previous_refined_context, previous_source_text):
+        # Apple ANE path first
+        if use_apple_llm and apple_llm is not None:
+            t0 = time.time()
+            try:
+                polished, ok, apple_ms = apple_llm.polish_paragraph(
+                    new_text=new_segments_str,
+                    context=previous_refined_context,
+                    prev_source=previous_source_text,
+                    language=getattr(args, "lang", None),
+                    timeout=getattr(args, "post_correct_llm_timeout", 8.0),
+                )
+                entry = {
+                    "timestamp": datetime.now(timezone.utc).isoformat(),
+                    "mode": "paragraph",
+                    "provider": "apple",
+                    "model": "apple-fm",
+                    "duration_ms": int((time.time()-t0)*1000),
+                    "apple_ms": apple_ms,
+                    "ok": ok,
+                }
+                append_llm_request_log(log_path, entry)
+                if ok and polished:
+                    print(f"[LLM] Apple paragraph polish OK {apple_ms:.0f}ms -> {polished[:80]}")
+                    return polished
+                else:
+                    print(f"[LLM] Apple paragraph fallback ok={ok} ms={apple_ms} err, trying Ollama")
+            except Exception as e:
+                print(f"[LLM] Apple paragraph error {e}, falling back to Ollama")
+                if not llm_state["apple_warned"]:
+                    llm_state["apple_warned"] = True
+
         messages, prompt = build_paragraph_messages(previous_refined_context, previous_source_text, new_segments_str)
         paragraph_options = {
             "top_p": getattr(args, "llm_paragraph_top_p", 0.8),
@@ -484,6 +552,61 @@ def run_llm_processor(in_queue, out_queue, args):
                 print("[LLM] Skipping line LLM polish; deterministic output looks clean.")
             return corrected
 
+        # Try Apple ANE path first (fast, offline)
+        if use_apple_llm and apple_llm is not None:
+            try:
+                t0 = time.time()
+                prev1 = recent_lines[-1] if len(recent_lines) >= 1 else ""
+                prev2 = recent_lines[-2] if len(recent_lines) >= 2 else ""
+                polished, ok, apple_ms = apple_llm.polish_line(
+                    text=substituted if getattr(args, "freeflow_polish", True) else corrected,
+                    prev1=prev1, prev2=prev2,
+                    language=getattr(args, "lang", None),
+                    timeout=getattr(args, "post_correct_llm_timeout", 8.0),
+                )
+                dur = int((time.time()-t0)*1000)
+                # Log Apple call
+                entry = {
+                    "timestamp": datetime.now(timezone.utc).isoformat(),
+                    "mode": "line",
+                    "provider": "apple",
+                    "model": "apple-fm",
+                    "duration_ms": dur,
+                    "apple_ms": apple_ms,
+                    "ok": ok,
+                    "input": substituted[:200],
+                    "response": {"content": polished},
+                }
+                append_llm_request_log(log_path, entry)
+                if ok and polished:
+                    # Apply same post-processing as Ollama path
+                    prompt_style = getattr(args, "post_correct_prompt_style", "freeflow")
+                    cand = polished
+                    if prompt_style != "legacy":
+                        fb = guard_against_truncation(cand, corrected)
+                        if fb:
+                            if getattr(args, "post_correct_debug", False):
+                                print(f"[LLM] Apple guard fallback triggered, keeping deterministic: {fb[:60]}")
+                            return fb
+                        cand = strip_keep_tags(cand)
+                        cand = normalize_formatting(cand)
+                        cand = match_input_casing(cand, substituted)
+                    else:
+                        cand = normalize_english_caption(cand)
+                    if cand and word_overlap_ratio(corrected, cand) >= getattr(args, "post_correct_min_overlap", 0.45):
+                        print(f"[LLM] Apple line OK {apple_ms:.0f}ms ({dur}ms total) -> {cand[:80]}")
+                        return cand
+                    else:
+                        if getattr(args, "verbose", False):
+                            print(f"[LLM] Apple line low overlap {word_overlap_ratio(corrected, cand):.2f}, keep deterministic")
+                        return corrected
+                else:
+                    print(f"[LLM] Apple line not ok ok={ok} ms={apple_ms}, falling back to Ollama")
+            except Exception as e:
+                print(f"[LLM] Apple line error {e}, falling back to Ollama")
+                if not llm_state["apple_warned"]:
+                    llm_state["apple_warned"] = True
+
         prompt_style = getattr(args, "post_correct_prompt_style", "freeflow")
         if prompt_style == "legacy":
             prev2 = recent_lines[-2] if len(recent_lines) >= 2 else ""