diff --git a/README.md b/README.md index 6764ebf..7619295 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,32 @@ A real-time, low-latency audio transcription and translation tool utilizing Open Run the script using `python3 transcribe.py` with optional flags. +### LLM Prompt Testing +You can now send the line-correction or paragraph-refinement prompts directly without starting live transcription. Every LLM request is appended as JSON Lines to `logs/llm_requests.jsonl` by default. + +- **Test the line-correction prompt:** + ```bash + python3 main_v2.py --post-correct-model qwen3.5:0.8b --llm-test-line "so um we should probably ship it tomorrow" --llm-test-prev1 "We finished the staging deploy." --llm-test-prev2 "QA signed off this morning." + ``` +- **Test the paragraph-refinement prompt:** + ```bash + python3 main_v2.py --post-correct-model qwen3.5:0.8b --llm-test-context "We reviewed the launch checklist." --llm-test-segments "we confirmed monitoring we confirmed rollback and then talked about the release window" + ``` +- **Change where requests are logged:** + ```bash + python3 main_v2.py --llm-request-log-path tmp/my_llm_requests.jsonl --llm-test-line "example text" + ``` +- **Replay the most recent logged request:** + ```bash + python3 main_v2.py --llm-test-from-log -1 + ``` +- **Replay a logged request using the original model from the log entry:** + ```bash + python3 main_v2.py --llm-test-from-log 12 --llm-test-use-logged-model + ``` + +Each log entry includes the timestamp, provider, model, mode, temperature, and full `messages` payload that was sent to the LLM. + ### Common Commands - **List available audio devices:** ```bash diff --git a/config.json b/config.json index 8f97b9b..3489e1e 100644 --- a/config.json +++ b/config.json @@ -25,6 +25,7 @@ "post_correct_min_overlap": 0.45, "post_correct_debug": false, "llm_paragraph": false, + "llm_request_log_path": "logs/llm_requests.jsonl", "only_translate_llm": false, "temperature_fallback": [0.0, 0.2, 0.4, 0.6, 0.8, 1.0], "logprob_threshold": -0.8, diff --git a/engine_llm.py b/engine_llm.py index 6ed296c..0a17b5e 100644 --- a/engine_llm.py +++ b/engine_llm.py @@ -19,6 +19,244 @@ import re import json import requests from collections import deque +from datetime import datetime, timezone + + +def build_paragraph_messages(previous_refined_context, previous_source_text, new_source_text): + system_prompt = """You are a careful live transcript editor working from noisy translated source text. + +Goal: +Produce the most faithful readable English update. + +Decision rules: +1. NEW SOURCE TEXT is the primary evidence and should dominate the output. +2. Use PREVIOUS SOURCE TEXT and PREVIOUS REFINED CONTEXT only when they clearly help resolve or continue the new material. +3. If the new material starts a fresh thought, output only the new material. +4. Remove exact or near-exact repetition unless the repetition is clearly intentional rhetoric. +5. Do not add explanations, disclaimers, or meta commentary. +6. Prefer conservative wording over guessed meaning. +7. Return only the revised transcript text.""" + + prompt = f"""Edit this transcript update. + +[PREVIOUS REFINED CONTEXT] +{previous_refined_context} + +[PREVIOUS SOURCE TEXT] +{previous_source_text} + +[NEW SOURCE TEXT] +{new_source_text} +""" + return [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": prompt}, + ], prompt + + +def build_line_messages(prev1, prev2, corrected): + system_prompt = """You are a real-time English caption corrector for live speech. + +Task: +Clean only the current caption line. + +Hard rules: +1. Preserve meaning exactly. Never replace current content with prior context. +2. Remove disfluencies and false starts. +3. Fix punctuation, casing, and obvious STT typos. +4. If uncertain, return the original line unchanged. +5. Output only one corrected English line.""" + + prompt = ( + "Context (reference only):\n" + f"Previous line 1: {prev1}\n" + f"Previous line 2: {prev2}\n" + "Current line to correct:\n" + f"{corrected}\n" + "Corrected:" + ) + return [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": prompt}, + ], prompt + + +def append_llm_request_log(log_path, entry): + if not log_path: + return + log_dir = os.path.dirname(log_path) + if log_dir: + os.makedirs(log_dir, exist_ok=True) + with open(log_path, "a", encoding="utf-8") as handle: + json.dump(entry, handle, ensure_ascii=False) + handle.write("\n") + + +def load_llm_request_log_entry(log_path, entry_index): + if not log_path or not os.path.exists(log_path): + raise FileNotFoundError(f"LLM request log not found: {log_path}") + + with open(log_path, "r", encoding="utf-8") as handle: + entries = [json.loads(line) for line in handle if line.strip()] + + if not entries: + raise ValueError(f"LLM request log is empty: {log_path}") + + if entry_index is None or entry_index == -1: + return entries[-1] + + if entry_index < 0: + entry_index = len(entries) + entry_index + + if entry_index < 0 or entry_index >= len(entries): + raise IndexError(f"LLM request log index {entry_index} is out of range for {len(entries)} entries") + + return entries[entry_index] + + +def run_llm_prompt_test(args): + from dotenv import load_dotenv + load_dotenv() + + api_key = os.environ.get("OPENAI_API_KEY") + log_path = getattr(args, "llm_request_log_path", "logs/llm_requests.jsonl") + + def call_openai(model, messages): + response = requests.post( + "https://api.openai.com/v1/chat/completions", + json={"model": model, "messages": messages}, + headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, + timeout=getattr(args, "post_correct_llm_timeout", 8.0), + ) + if response.status_code >= 400: + try: + payload = response.json() + detail = json.dumps(payload, ensure_ascii=True) + except Exception: + detail = response.text.strip() + raise RuntimeError(f"OpenAI API error {response.status_code}: {detail}") + return response.json()["choices"][0]["message"]["content"].strip() + + def call_ollama(model, prompt, temperature, system=None): + response = requests.post( + getattr(args, "post_correct_ollama_url", "http://127.0.0.1:11434/api/generate"), + json={ + "model": model, + "prompt": prompt, + "system": system or "", + "stream": False, + "options": {"temperature": temperature}, + "keep_alive": getattr(args, "post_correct_keep_alive", "30m"), + }, + timeout=getattr(args, "post_correct_llm_timeout", 8.0), + ) + response.raise_for_status() + return response.json().get("response", "").strip() + + def send_test_request(mode, messages, prompt, temperature, metadata, model=None): + request_model = model or args.post_correct_model + is_openai = request_model.startswith("gpt-") + entry = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "mode": mode, + "provider": "openai" if is_openai else "ollama", + "model": request_model, + "temperature": temperature, + "messages": messages, + "metadata": metadata, + } + started_at = time.time() + if is_openai: + if not api_key: + raise RuntimeError("OPENAI_API_KEY is not set") + try: + response_text = call_openai(request_model, messages) + entry["response"] = {"status": "ok", "content": response_text} + return response_text + except Exception as exc: + entry["response"] = {"status": "error", "error": str(exc)} + raise + finally: + entry["duration_ms"] = round((time.time() - started_at) * 1000, 2) + append_llm_request_log(log_path, entry) + system = "" + for message in messages: + if message.get("role") == "system": + system = message.get("content", "") + break + try: + response_text = call_ollama(request_model, prompt, temperature, system=system) + entry["response"] = {"status": "ok", "content": response_text} + return response_text + except Exception as exc: + entry["response"] = {"status": "error", "error": str(exc)} + raise + finally: + entry["duration_ms"] = round((time.time() - started_at) * 1000, 2) + append_llm_request_log(log_path, entry) + + if getattr(args, "llm_test_from_log", None) is not None: + logged_entry = load_llm_request_log_entry(log_path, args.llm_test_from_log) + logged_messages = logged_entry.get("messages") + if not logged_messages: + raise ValueError("Selected log entry does not contain messages") + logged_prompt = next((m.get("content", "") for m in logged_messages if m.get("role") == "user"), "") + logged_model = logged_entry.get("model", args.post_correct_model) + replay_model = logged_model if getattr(args, "llm_test_use_logged_model", False) else args.post_correct_model + response = send_test_request( + logged_entry.get("mode", "replay"), + logged_messages, + logged_prompt, + logged_entry.get("temperature", 0.1), + { + "replay_source_log_path": log_path, + "replay_source_index": args.llm_test_from_log, + "replay_source_timestamp": logged_entry.get("timestamp"), + "replay_source_model": logged_model, + }, + model=replay_model, + ) + print("[LLM TEST] Replayed request from log:") + print(f"source_index={args.llm_test_from_log} source_model={logged_model} replay_model={replay_model}") + print(response) + + if getattr(args, "llm_test_line", None): + messages, prompt = build_line_messages( + getattr(args, "llm_test_prev1", ""), + getattr(args, "llm_test_prev2", ""), + args.llm_test_line, + ) + response = send_test_request( + "line", + messages, + prompt, + 0.1, + { + "prev1": getattr(args, "llm_test_prev1", ""), + "prev2": getattr(args, "llm_test_prev2", ""), + "input": args.llm_test_line, + }, + ) + print("[LLM TEST] Line response:") + print(response) + + if getattr(args, "llm_test_segments", None): + messages, prompt = build_paragraph_messages( + getattr(args, "llm_test_context", ""), + args.llm_test_segments, + ) + response = send_test_request( + "paragraph", + messages, + prompt, + 0.1, + { + "context": getattr(args, "llm_test_context", ""), + "segments": args.llm_test_segments, + }, + ) + print("[LLM TEST] Paragraph response:") + print(response) def run_llm_processor(in_queue, out_queue, args): from dotenv import load_dotenv @@ -29,6 +267,7 @@ def run_llm_processor(in_queue, out_queue, args): api_key = os.environ.get("OPENAI_API_KEY") is_openai = args.post_correct_model.startswith("gpt-") llm_state = {"line_warned": False, "paragraph_warned": False} + log_path = getattr(args, "llm_request_log_path", "logs/llm_requests.jsonl") def check_ollama_health(): try: @@ -130,49 +369,49 @@ def run_llm_processor(in_queue, out_queue, args): return response.json().get("response", "").strip() def call_llm(messages, prompt, temperature, warn_key): + entry = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "mode": "paragraph" if warn_key == "paragraph_warned" else "line", + "provider": "openai" if is_openai else "ollama", + "model": args.post_correct_model, + "temperature": temperature, + "messages": messages, + } + started_at = time.time() try: if is_openai: if not api_key: raise RuntimeError("OPENAI_API_KEY is not set") - return call_openai(messages) + response_text = call_openai(messages) + entry["response"] = {"status": "ok", "content": response_text} + return response_text system = "" if messages: for message in messages: if message.get("role") == "system": system = message.get("content", "") break - return call_ollama(prompt, temperature, system=system) + response_text = call_ollama(prompt, temperature, system=system) + entry["response"] = {"status": "ok", "content": response_text} + return response_text except Exception as exc: + entry["response"] = {"status": "error", "error": str(exc)} if not llm_state[warn_key]: label = "Paragraph LLM" if warn_key == "paragraph_warned" else "Line LLM" print(f"[LLM] {label} unavailable ({exc}). Falling back to deterministic text.") llm_state[warn_key] = True return "" + finally: + entry["duration_ms"] = round((time.time() - started_at) * 1000, 2) + append_llm_request_log(log_path, entry) # State: This is the ONLY text we send to the LLM as context active_context = "" + previous_source_window = "" recent_lines = deque(maxlen=2) - def call_llm_rolling_refine(new_segments_str, context): - prompt = f"""Task: Refine the following live transcription stream into clean, professional paragraphs. - -[PREVIOUS WORKING CONTEXT] -"{context}" - -[NEW RAW ASR SEGMENTS] -"{new_segments_str}" - -[INSTRUCTIONS] -1. INTEGRATE: Polished and merge the new segments into the flow of the 'PREVIOUS WORKING CONTEXT'. -2. CONSOLIDATE: Remove redundant repetitions and translator echoes. -3. ORGANIZE: Use a double newline (\\n\\n) to start a new paragraph when a topic changes or the current one is complete. -4. TARGET LANGUAGE: Output ONLY in English. -5. OUTPUT: Provide ONLY the refined, consolidated text. Do not explain anything. -""" - messages = [ - {"role": "system", "content": "You are a live transcript editor."}, - {"role": "user", "content": prompt}, - ] + def call_llm_rolling_refine(new_segments_str, context, previous_source_text): + messages, prompt = build_paragraph_messages(context, previous_source_text, new_segments_str) return call_llm(messages, prompt, 0.1, "paragraph_warned") def post_correct_line(text): @@ -185,27 +424,7 @@ def run_llm_processor(in_queue, out_queue, args): prev2 = recent_lines[-2] if len(recent_lines) >= 2 else "" prev1 = recent_lines[-1] if len(recent_lines) >= 1 else "" - prompt = ( - "You are a real-time English caption corrector for live speech.\n" - "Task:\n" - "Clean ONLY the current caption line.\n" - "Hard rules:\n" - "1. Preserve meaning exactly. Never replace current content with prior context.\n" - "2. Remove disfluencies and false starts.\n" - "3. Fix punctuation, casing, and obvious STT typos.\n" - "4. If uncertain, return the original line unchanged.\n" - "5. Output ONLY one corrected English line.\n" - "Context (reference only):\n" - f"Previous line 1: {prev1}\n" - f"Previous line 2: {prev2}\n" - "Current line to correct:\n" - f"{corrected}\n" - "Corrected:" - ) - messages = [ - {"role": "system", "content": "You are a real-time English caption corrector."}, - {"role": "user", "content": prompt}, - ] + messages, prompt = build_line_messages(prev1, prev2, corrected) candidate = normalize_english_caption(call_llm(messages, prompt, 0.1, "line_warned")) if candidate and word_overlap_ratio(corrected, candidate) >= getattr(args, "post_correct_min_overlap", 0.45): return candidate @@ -244,7 +463,7 @@ def run_llm_processor(in_queue, out_queue, args): paragraph_from_llm = False if args.llm_paragraph and should_refine: new_batch = " ".join(pending_buffer) - result = call_llm_rolling_refine(new_batch, active_context) + result = call_llm_rolling_refine(new_batch, active_context, previous_source_window) if result: # Logic to handle paragraph breaks @@ -263,6 +482,7 @@ def run_llm_processor(in_queue, out_queue, args): else: structured_paragraph = new_batch active_context = new_batch + previous_source_window = new_batch pending_buffer = [] last_llm_call_time = time.time() diff --git a/history.md b/history.md index 358b8e4..15565bf 100644 --- a/history.md +++ b/history.md @@ -279,3 +279,21 @@ The project now features a high-performance, Apple Silicon-optimized pipeline th - Removed the explicit Chat Completions temperature override after discovering current GPT-5-family models reject non-default temperature values on this endpoint. - Aligned the Ollama path with the OpenAI path by forwarding the same system instructions to local generation requests, including warmup calls. - **Outcome:** GPT-backed runs now fail with useful diagnostics, OpenAI requests are compatible with current GPT-5-mini behavior, and local-vs-remote prompting is more consistent. + +## Phase 32: LLM Request Logging + Prompt Replay Harness +- **Goal:** Make prompt behavior observable and reproducible so local LLM tuning can happen without rerunning live transcription. +- **Approach:** + - Added JSONL request logging for every LLM call, including full message payloads, timing, and the final response or error outcome. + - Added prompt-only test entry points in `main_v2.py` so line-correction and paragraph-refinement prompts can be sent directly without starting the transcription pipeline. + - Added replay-from-log support so a captured request can be resent with the current model or the originally logged model. + - Documented the new logging and replay workflow in `README.md` and added a default log path in `config.json`. +- **Outcome:** We can now inspect exactly what the app sends to the LLM, replay real requests on demand, and compare prompt changes against stored payloads and outputs. + +## Phase 33: Grounded Paragraph Prompt Experiments +- **Goal:** Reduce paragraph drift and self-reinforcing hallucinations in the rolling LLM refiner. +- **Approach:** + - Built `prompt_experiments.py` to reconstruct representative paragraph-refinement samples from `distribute_debug.log` and send multiple prompt variants directly to Ollama. + - Compared several context arrangements, including the previous refined paragraph alone, raw-only windows, and mixed refined-plus-raw grounding strategies. + - Identified the best-performing variant as one that treats the new source window as primary evidence while using prior refined and prior raw text only when they clearly continue the thought. + - Updated `engine_llm.py` so live paragraph refinement now sends `PREVIOUS REFINED CONTEXT`, `PREVIOUS SOURCE TEXT`, and `NEW SOURCE TEXT` with a more conservative, grounding-oriented system prompt. +- **Outcome:** Paragraph refinement is now substantially better anchored to current source text, with less stale-context carryover and a much lower rate of unsupported content in direct Ollama tests. diff --git a/main_v2.py b/main_v2.py index 3141db2..854339a 100644 --- a/main_v2.py +++ b/main_v2.py @@ -10,7 +10,7 @@ load_dotenv() # Light imports (heavy ones moved inside run functions) from engine_transcribe import run_transcription, list_audio_devices -from engine_llm import run_llm_processor +from engine_llm import run_llm_processor, run_llm_prompt_test from engine_translate import run_translation from engine_distribute import run_distribution @@ -57,6 +57,14 @@ def main(): parser.add_argument("--post-correct-min-overlap", type=float, default=defaults.get("post_correct_min_overlap", 0.45)) parser.add_argument("--post-correct-debug", action="store_true", default=defaults.get("post_correct_debug", False)) parser.add_argument("--llm-paragraph", action="store_true", default=defaults.get("llm_paragraph", False)) + parser.add_argument("--llm-request-log-path", type=str, default=defaults.get("llm_request_log_path", "logs/llm_requests.jsonl")) + parser.add_argument("--llm-test-line", type=str, default=None, help="Send a single line-correction prompt without starting transcription.") + parser.add_argument("--llm-test-prev1", type=str, default="", help="Optional previous line 1 context for --llm-test-line.") + parser.add_argument("--llm-test-prev2", type=str, default="", help="Optional previous line 2 context for --llm-test-line.") + parser.add_argument("--llm-test-segments", type=str, default=None, help="Send a single paragraph-refinement prompt without starting transcription.") + parser.add_argument("--llm-test-context", type=str, default="", help="Optional existing paragraph context for --llm-test-segments.") + parser.add_argument("--llm-test-from-log", type=int, default=None, help="Replay a logged LLM request by JSONL entry index. Use -1 for the most recent entry.") + parser.add_argument("--llm-test-use-logged-model", action="store_true", default=False, help="When replaying from log, use the model stored in the log entry instead of --post-correct-model.") # Translate Args parser.add_argument("-es", action="store_true", default=defaults.get("es", False)) @@ -80,6 +88,10 @@ def main(): print("[Main] Enabling --llm-paragraph because --only-translate-llm was requested.") args.llm_paragraph = True + if args.llm_test_line or args.llm_test_segments or args.llm_test_from_log is not None: + run_llm_prompt_test(args) + return + # Handle device listing if args.list_devices: list_audio_devices() diff --git a/prompt_experiments.py b/prompt_experiments.py new file mode 100644 index 0000000..f310b3d --- /dev/null +++ b/prompt_experiments.py @@ -0,0 +1,356 @@ +import argparse +import json +import os +import re +import statistics +from collections import Counter +from urllib import request, error + + +def parse_distribute_debug_log(path): + entries = [] + current = {"final": None, "en_bridge": None, "llm_paragraph": None} + + with open(path, "r", encoding="utf-8") as handle: + for raw_line in handle: + line = raw_line.rstrip("\n") + if "[Final]:" in line: + current["final"] = line.split("[Final]:", 1)[1].strip() + elif "[EN-BRIDGE]:" in line: + current["en_bridge"] = line.split("[EN-BRIDGE]:", 1)[1].strip() + elif "[LLM-PARAGRAPH]:" in line: + current["llm_paragraph"] = line.split("[LLM-PARAGRAPH]:", 1)[1].strip() + entries.append(current) + current = {"final": None, "en_bridge": None, "llm_paragraph": None} + elif not line.strip(): + continue + + return entries + + +def normalize_paragraph(text): + return (text or "").strip().strip('"').strip() + + +def reconstruct_samples(path): + samples = [] + rows = parse_distribute_debug_log(path) + previous_llm = "" + pending_bridges = [] + recent_raw_windows = [] + + for row in rows: + if row.get("en_bridge"): + pending_bridges.append(row["en_bridge"]) + if not row.get("llm_paragraph"): + continue + + llm_output = normalize_paragraph(row["llm_paragraph"]) + new_raw = " ".join(pending_bridges).strip() + prev_raw = recent_raw_windows[-1] if recent_raw_windows else "" + + samples.append( + { + "previous_refined": previous_llm, + "previous_raw": prev_raw, + "new_raw": new_raw, + "observed_output": llm_output, + } + ) + + previous_llm = llm_output + recent_raw_windows.append(new_raw) + pending_bridges = [] + + return samples + + +def build_variants(sample): + prev_refined = sample["previous_refined"] + prev_raw = sample["previous_raw"] + new_raw = sample["new_raw"] + + variants = [] + + system_a = """You are a live transcript editor. + +Task: +Refine live transcription stream into clean, professional paragraphs. + +Rules: +1. Integrate new segments into the flow of the previous working context. +2. Remove redundant repetitions and translator echoes. +3. Use a double newline (\\n\\n) to start a new paragraph when a topic changes or the current one is complete. +4. Output only in English. +5. Return only the refined, consolidated text with no explanation.""" + user_a = f"""Refine the following live transcription stream. + +[PREVIOUS WORKING CONTEXT] +"{prev_refined}" + +[NEW SOURCE TEXT] +"{new_raw}" +""" + variants.append(("current_refined_plus_new", system_a, user_a)) + + system_b = """You are a live transcript editor for noisy speech recognition output. + +Goal: +Produce a clean English paragraph update grounded in the source text. + +Grounding rules: +1. Treat PREVIOUS REFINED CONTEXT as style and continuity help, not as unquestionable truth. +2. Use PREVIOUS SOURCE TEXT and NEW SOURCE TEXT as the factual anchor. +3. Do not add details that are not supported by the source text. +4. Preserve uncertainty when the source is unclear instead of inventing a cleaner claim. +5. Remove obvious repetitions and translator echoes. +6. Start a new paragraph with a double newline only when there is a clear topic shift. +7. Return only the refined English text with no explanation.""" + user_b = f"""Update the transcript using the source text below. + +[PREVIOUS REFINED CONTEXT] +"{prev_refined}" + +[PREVIOUS SOURCE TEXT] +"{prev_raw}" + +[NEW SOURCE TEXT] +"{new_raw}" +""" + variants.append(("refined_plus_prev_raw_plus_new_raw", system_b, user_b)) + + system_c = """You are a careful transcript editor. + +Goal: +Produce clean English paragraphs directly from noisy source text. + +Rules: +1. Use only the supplied source text as evidence. +2. Do not infer missing facts. +3. Keep wording conservative when the source is noisy. +4. Remove repeated fragments and obvious translation artifacts. +5. Use a double newline only for a clear paragraph break. +6. Return only the refined English text.""" + user_c = f"""Refine this transcript source. + +[RECENT SOURCE TEXT] +"{prev_raw}" + +[NEW SOURCE TEXT] +"{new_raw}" +""" + variants.append(("prev_raw_plus_new_raw_only", system_c, user_c)) + + system_d = """You are a live transcript editor for sermon audio translated into English. + +Goal: +Create the most faithful readable English paragraph you can from imperfect source text. + +Priority order: +1. Faithfulness to source text. +2. Remove duplicated phrases, stutters, and translation echoes. +3. Maintain continuity with the previous refined context only when it matches the source text. +4. Prefer slight awkwardness over hallucination. +5. If a phrase is unclear, keep it modestly literal instead of guessing. +6. Use a double newline only for a real topic change. +7. Return only the revised English transcript. + +Hard constraints: +- Do not add people, events, or meanings absent from the source text. +- Do not turn rhetorical questions into factual claims unless the source clearly does that. +- Do not overwrite NEW SOURCE TEXT with PREVIOUS REFINED CONTEXT.""" + user_d = f"""Revise the transcript update. + +[PREVIOUS REFINED CONTEXT] +"{prev_refined}" + +[RECENT SOURCE WINDOW] +"{prev_raw}" + +[CURRENT SOURCE WINDOW] +"{new_raw}" +""" + variants.append(("grounded_sermon_editor", system_d, user_d)) + + system_e = """You are a careful live transcript editor working from noisy translated source text. + +Goal: +Produce the most faithful readable English update. + +Decision rules: +1. NEW SOURCE TEXT is the primary evidence and should dominate the output. +2. Use PREVIOUS SOURCE TEXT and PREVIOUS REFINED CONTEXT only when they clearly help resolve or continue the new material. +3. If the new material starts a fresh thought, output only the new material. +4. Remove exact or near-exact repetition unless the repetition is clearly intentional rhetoric. +5. Do not add explanations, disclaimers, or meta commentary. +6. Prefer conservative wording over guessed meaning. +7. Return only the revised transcript text.""" + user_e = f"""Edit this transcript update. + +[PREVIOUS REFINED CONTEXT] +"{prev_refined}" + +[PREVIOUS SOURCE TEXT] +"{prev_raw}" + +[NEW SOURCE TEXT] +"{new_raw}" +""" + variants.append(("grounded_new_dominant", system_e, user_e)) + + return variants + + +def ollama_generate(model, system_prompt, user_prompt, url, timeout, keep_alive): + payload = json.dumps( + { + "model": model, + "prompt": user_prompt, + "system": system_prompt, + "stream": False, + "options": {"temperature": 0.1}, + "keep_alive": keep_alive, + } + ).encode("utf-8") + req = request.Request( + url, + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with request.urlopen(req, timeout=timeout) as response: + body = json.loads(response.read().decode("utf-8")) + except error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"Ollama HTTP error {exc.code}: {detail}") from exc + return body.get("response", "").strip() + + +def tokenize(text): + return re.findall(r"[a-z0-9']+", (text or "").lower()) + + +def repeated_ngram_ratio(text, n=3): + tokens = tokenize(text) + if len(tokens) < n: + return 0.0 + grams = [tuple(tokens[i:i + n]) for i in range(len(tokens) - n + 1)] + counts = Counter(grams) + repeated = sum(count - 1 for count in counts.values() if count > 1) + return repeated / max(1, len(grams)) + + +def unsupported_token_ratio(output, evidence): + evidence_tokens = set(tokenize(evidence)) + output_tokens = tokenize(output) + if not output_tokens: + return 0.0 + unsupported = [tok for tok in output_tokens if tok not in evidence_tokens] + return len(unsupported) / len(output_tokens) + + +def evaluate_output(output, sample): + evidence = " ".join( + part for part in [sample["previous_raw"], sample["new_raw"]] if part + ) + return { + "output_len": len(tokenize(output)), + "repeated_ngram_ratio": round(repeated_ngram_ratio(output), 4), + "unsupported_token_ratio": round(unsupported_token_ratio(output, evidence), 4), + } + + +def run_experiments(args): + samples = reconstruct_samples(args.log_file) + if args.limit: + samples = samples[: args.limit] + + results = [] + for index, sample in enumerate(samples): + print(f"Running sample {index + 1}/{len(samples)}") + sample_results = [] + for variant_name, system_prompt, user_prompt in build_variants(sample): + print(f" variant={variant_name}") + try: + output = ollama_generate( + args.model, + system_prompt, + user_prompt, + args.ollama_url, + args.timeout, + args.keep_alive, + ) + metrics = evaluate_output(output, sample) + error_message = None + except Exception as exc: + output = "" + metrics = None + error_message = str(exc) + sample_results.append( + { + "variant": variant_name, + "output": output, + "metrics": metrics, + "error": error_message, + } + ) + results.append( + { + "sample_index": index, + "sample": sample, + "results": sample_results, + } + ) + return results + + +def summarize_results(results): + summary = {} + for sample_result in results: + for variant_result in sample_result["results"]: + bucket = summary.setdefault( + variant_result["variant"], + {"repeated_ngram_ratio": [], "unsupported_token_ratio": [], "output_len": [], "errors": 0}, + ) + if variant_result.get("error"): + bucket["errors"] += 1 + continue + for key, value in variant_result["metrics"].items(): + bucket[key].append(value) + + for variant_name, metrics in summary.items(): + summary[variant_name] = { + key: round(statistics.mean(values), 4) if isinstance(values, list) and values else values + for key, values in metrics.items() + } + return summary + + +def main(): + parser = argparse.ArgumentParser(description="Run prompt experiments against Ollama using distribute_debug.log samples.") + parser.add_argument("--log-file", type=str, default="distribute_debug.log") + parser.add_argument("--model", type=str, default="qwen2.5:3b-instruct") + parser.add_argument("--ollama-url", type=str, default="http://127.0.0.1:11434/api/generate") + parser.add_argument("--timeout", type=float, default=40.0) + parser.add_argument("--keep-alive", type=str, default="30m") + parser.add_argument("--limit", type=int, default=0) + parser.add_argument("--output-file", type=str, default="logs/prompt_experiment_results.json") + args = parser.parse_args() + + results = run_experiments(args) + summary = summarize_results(results) + payload = {"model": args.model, "summary": summary, "results": results} + + output_dir = os.path.dirname(args.output_file) + if output_dir: + os.makedirs(output_dir, exist_ok=True) + with open(args.output_file, "w", encoding="utf-8") as handle: + json.dump(payload, handle, ensure_ascii=False, indent=2) + + print(json.dumps(summary, ensure_ascii=False, indent=2)) + print(f"Wrote detailed results to {args.output_file}") + + +if __name__ == "__main__": + main()