feat: warm up ollama and keep post-correct model loaded

history.md

@@ -194,3 +194,12 @@ The project now features a high-performance, Apple Silicon-optimized pipeline th
     - Added token-overlap acceptance guard (`--post-correct-min-overlap`) to reject LLM rewrites that diverge too far from source text.
     - Updated merge-decider prompt to stricter JSON behavior and added overlap-based fallback to heuristic merge.
 - **Outcome:** Improved protection against context bleed (e.g., replacing current line with prior sentence) while keeping optional LLM improvements.
+
+## Phase 24: Ollama Warmup + Keep-Alive
+- **Goal:** Reduce intermittent post-correction timeouts caused by cold model loads.
+- **Approach:**
+    - Added startup warmup request for the local post-correct model when LLM correction is enabled.
+    - Added configurable Ollama `keep_alive` setting so the model stays resident between caption calls.
+    - Routed post-correct and merge-decider requests through a shared Ollama caller with keep-alive support.
+    - Added flags for warmup timeout and optional warmup skip for troubleshooting.
+- **Outcome:** Lower first-request latency and fewer fallback-to-rules events due to local model cold starts.

transcribe.py

@@ -282,6 +282,21 @@ def word_overlap_ratio(source_text, candidate_text):
         return 1.0
     return len(src_tokens & cand_tokens) / len(src_tokens)
 
+def call_ollama_generate(args, prompt, timeout, temperature=0.1):
+    """Call local Ollama with keep_alive to reduce cold-start latency."""
+    payload = {
+        "model": args.post_correct_model,
+        "prompt": prompt,
+        "stream": False,
+        "options": {"temperature": temperature},
+        "keep_alive": args.post_correct_keep_alive,
+    }
+    return requests.post(
+        args.post_correct_ollama_url,
+        json=payload,
+        timeout=timeout,
+    )
+
 def post_correct_with_local_llm(text, args, llm_state, context_lines=None):
     """Optionally rewrite finalized English caption via a local Ollama model."""
     if not args.post_correct_llm or not llm_state.get("available", True):
@@ -313,16 +328,7 @@ def post_correct_with_local_llm(text, args, llm_state, context_lines=None):
     prompt += f"{text}\nCorrected:"
 
     try:
-        response = requests.post(
-            args.post_correct_ollama_url,
-            json={
-                "model": args.post_correct_model,
-                "prompt": prompt,
-                "stream": False,
-                "options": {"temperature": 0.1},
-            },
-            timeout=args.post_correct_llm_timeout,
-        )
+        response = call_ollama_generate(args, prompt, args.post_correct_llm_timeout, temperature=0.1)
         response.raise_for_status()
         raw = response.json().get("response", "").strip()
         cleaned = normalize_english_caption(raw)
@@ -394,16 +400,7 @@ def llm_decide_caption_merge(previous_text, current_text, candidate_text, args,
     )
 
     try:
-        response = requests.post(
-            args.post_correct_ollama_url,
-            json={
-                "model": args.post_correct_model,
-                "prompt": prompt,
-                "stream": False,
-                "options": {"temperature": 0.0},
-            },
-            timeout=args.llm_merge_timeout,
-        )
+        response = call_ollama_generate(args, prompt, args.llm_merge_timeout, temperature=0.0)
         response.raise_for_status()
         raw = response.json().get("response", "").strip()
         if not raw:
@@ -520,6 +517,22 @@ def ingest_worker():
         
         ingest_queue.task_done()
 
+def warmup_ollama_model(args):
+    """Warm up local Ollama model so first correction request avoids cold-start latency."""
+    try:
+        response = call_ollama_generate(
+            args,
+            "Return exactly: ok",
+            timeout=args.post_correct_warmup_timeout,
+            temperature=0.0,
+        )
+        response.raise_for_status()
+        content = response.json().get("response", "").strip().lower()
+        if content:
+            print(f"[SYSTEM]: Ollama warmup OK ({args.post_correct_model}).")
+    except Exception as exc:
+        print(f"[SYSTEM]: Ollama warmup failed ({exc}). Continuing; runtime fallback will apply.")
+
 def main():
     global WHISPER_MODEL
     parser = argparse.ArgumentParser(description="Live transcription and translation with Whisper.")
@@ -563,6 +576,9 @@ def main():
     parser.add_argument("--post-correct-model", type=str, default="llama3.1:8b-instruct", help="Local Ollama model for post-correction")
     parser.add_argument("--post-correct-ollama-url", type=str, default="http://127.0.0.1:11434/api/generate", help="Ollama generate endpoint for local post-correction")
     parser.add_argument("--post-correct-llm-timeout", type=float, default=8.0, help="Timeout seconds for local LLM post-correction (default: 8.0)")
+    parser.add_argument("--post-correct-keep-alive", type=str, default="30m", help="Ollama keep_alive duration to keep the model loaded (default: 30m)")
+    parser.add_argument("--post-correct-warmup-timeout", type=float, default=12.0, help="Timeout seconds for startup Ollama warmup (default: 12.0)")
+    parser.add_argument("--skip-post-correct-warmup", action="store_true", help="Skip startup warmup call for the post-correct LLM")
     parser.add_argument("--post-correct-min-overlap", type=float, default=0.45, help="Minimum token overlap ratio required to accept LLM rewrite (default: 0.45)")
     parser.add_argument("--post-correct-debug", action="store_true", help="Print pre/post LLM correction text for debugging")
     parser.add_argument("--llm-merge-decider", action="store_true", help="Use local LLM to validate/refine smart-correct line merges")
@@ -583,7 +599,10 @@ def main():
     if args.post_correct:
         print("Post-correct mode: rules enabled.")
     if args.post_correct_llm:
-        print(f"Post-correct LLM: {args.post_correct_model} via {args.post_correct_ollama_url}")
+        print(
+            f"Post-correct LLM: {args.post_correct_model} via {args.post_correct_ollama_url} "
+            f"(keep_alive={args.post_correct_keep_alive})"
+        )
     if args.post_correct_debug:
         print("Post-correct debug logging enabled.")
     if args.llm_merge_decider and not args.post_correct_llm:
@@ -614,6 +633,9 @@ def main():
     # 1. Load models
     print(f"Loading Multilingual Whisper model '{WHISPER_MODEL}'...")
 
+    if args.post_correct_llm and not args.skip_post_correct_warmup:
+        warmup_ollama_model(args)
+    
     # Filter translation models to only those enabled by args
     active_target_langs = {k: v for k, v in TARGET_LANGS.items() if getattr(args, k, False)}