From 4115ddfa8b12c91b97f3ceee572542367f1d6b80 Mon Sep 17 00:00:00 2001 From: Adolfo Reyna Date: Fri, 6 Mar 2026 20:48:11 -0500 Subject: [PATCH] feat: warm up ollama and keep post-correct model loaded --- history.md | 9 ++++++++ transcribe.py | 64 ++++++++++++++++++++++++++++++++++----------------- 2 files changed, 52 insertions(+), 21 deletions(-) diff --git a/history.md b/history.md index 165ec1b..56aa0b6 100644 --- a/history.md +++ b/history.md @@ -194,3 +194,12 @@ The project now features a high-performance, Apple Silicon-optimized pipeline th - Added token-overlap acceptance guard (`--post-correct-min-overlap`) to reject LLM rewrites that diverge too far from source text. - Updated merge-decider prompt to stricter JSON behavior and added overlap-based fallback to heuristic merge. - **Outcome:** Improved protection against context bleed (e.g., replacing current line with prior sentence) while keeping optional LLM improvements. + +## Phase 24: Ollama Warmup + Keep-Alive +- **Goal:** Reduce intermittent post-correction timeouts caused by cold model loads. +- **Approach:** + - Added startup warmup request for the local post-correct model when LLM correction is enabled. + - Added configurable Ollama `keep_alive` setting so the model stays resident between caption calls. + - Routed post-correct and merge-decider requests through a shared Ollama caller with keep-alive support. + - Added flags for warmup timeout and optional warmup skip for troubleshooting. +- **Outcome:** Lower first-request latency and fewer fallback-to-rules events due to local model cold starts. diff --git a/transcribe.py b/transcribe.py index c2030f1..8d6da7c 100644 --- a/transcribe.py +++ b/transcribe.py @@ -282,6 +282,21 @@ def word_overlap_ratio(source_text, candidate_text): return 1.0 return len(src_tokens & cand_tokens) / len(src_tokens) +def call_ollama_generate(args, prompt, timeout, temperature=0.1): + """Call local Ollama with keep_alive to reduce cold-start latency.""" + payload = { + "model": args.post_correct_model, + "prompt": prompt, + "stream": False, + "options": {"temperature": temperature}, + "keep_alive": args.post_correct_keep_alive, + } + return requests.post( + args.post_correct_ollama_url, + json=payload, + timeout=timeout, + ) + def post_correct_with_local_llm(text, args, llm_state, context_lines=None): """Optionally rewrite finalized English caption via a local Ollama model.""" if not args.post_correct_llm or not llm_state.get("available", True): @@ -313,16 +328,7 @@ def post_correct_with_local_llm(text, args, llm_state, context_lines=None): prompt += f"{text}\nCorrected:" try: - response = requests.post( - args.post_correct_ollama_url, - json={ - "model": args.post_correct_model, - "prompt": prompt, - "stream": False, - "options": {"temperature": 0.1}, - }, - timeout=args.post_correct_llm_timeout, - ) + response = call_ollama_generate(args, prompt, args.post_correct_llm_timeout, temperature=0.1) response.raise_for_status() raw = response.json().get("response", "").strip() cleaned = normalize_english_caption(raw) @@ -394,16 +400,7 @@ def llm_decide_caption_merge(previous_text, current_text, candidate_text, args, ) try: - response = requests.post( - args.post_correct_ollama_url, - json={ - "model": args.post_correct_model, - "prompt": prompt, - "stream": False, - "options": {"temperature": 0.0}, - }, - timeout=args.llm_merge_timeout, - ) + response = call_ollama_generate(args, prompt, args.llm_merge_timeout, temperature=0.0) response.raise_for_status() raw = response.json().get("response", "").strip() if not raw: @@ -520,6 +517,22 @@ def ingest_worker(): ingest_queue.task_done() +def warmup_ollama_model(args): + """Warm up local Ollama model so first correction request avoids cold-start latency.""" + try: + response = call_ollama_generate( + args, + "Return exactly: ok", + timeout=args.post_correct_warmup_timeout, + temperature=0.0, + ) + response.raise_for_status() + content = response.json().get("response", "").strip().lower() + if content: + print(f"[SYSTEM]: Ollama warmup OK ({args.post_correct_model}).") + except Exception as exc: + print(f"[SYSTEM]: Ollama warmup failed ({exc}). Continuing; runtime fallback will apply.") + def main(): global WHISPER_MODEL parser = argparse.ArgumentParser(description="Live transcription and translation with Whisper.") @@ -563,6 +576,9 @@ def main(): parser.add_argument("--post-correct-model", type=str, default="llama3.1:8b-instruct", help="Local Ollama model for post-correction") parser.add_argument("--post-correct-ollama-url", type=str, default="http://127.0.0.1:11434/api/generate", help="Ollama generate endpoint for local post-correction") parser.add_argument("--post-correct-llm-timeout", type=float, default=8.0, help="Timeout seconds for local LLM post-correction (default: 8.0)") + parser.add_argument("--post-correct-keep-alive", type=str, default="30m", help="Ollama keep_alive duration to keep the model loaded (default: 30m)") + parser.add_argument("--post-correct-warmup-timeout", type=float, default=12.0, help="Timeout seconds for startup Ollama warmup (default: 12.0)") + parser.add_argument("--skip-post-correct-warmup", action="store_true", help="Skip startup warmup call for the post-correct LLM") parser.add_argument("--post-correct-min-overlap", type=float, default=0.45, help="Minimum token overlap ratio required to accept LLM rewrite (default: 0.45)") parser.add_argument("--post-correct-debug", action="store_true", help="Print pre/post LLM correction text for debugging") parser.add_argument("--llm-merge-decider", action="store_true", help="Use local LLM to validate/refine smart-correct line merges") @@ -583,7 +599,10 @@ def main(): if args.post_correct: print("Post-correct mode: rules enabled.") if args.post_correct_llm: - print(f"Post-correct LLM: {args.post_correct_model} via {args.post_correct_ollama_url}") + print( + f"Post-correct LLM: {args.post_correct_model} via {args.post_correct_ollama_url} " + f"(keep_alive={args.post_correct_keep_alive})" + ) if args.post_correct_debug: print("Post-correct debug logging enabled.") if args.llm_merge_decider and not args.post_correct_llm: @@ -613,6 +632,9 @@ def main(): # 1. Load models print(f"Loading Multilingual Whisper model '{WHISPER_MODEL}'...") + + if args.post_correct_llm and not args.skip_post_correct_warmup: + warmup_ollama_model(args) # Filter translation models to only those enabled by args active_target_langs = {k: v for k, v in TARGET_LANGS.items() if getattr(args, k, False)}