Add LLM logging and prompt experiment harness
This commit is contained in:
Adolfo Reyna
+264
-44
@@ -19,6 +19,244 @@ import re
|
||||
import json
|
||||
import requests
|
||||
from collections import deque
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
def build_paragraph_messages(previous_refined_context, previous_source_text, new_source_text):
|
||||
system_prompt = """You are a careful live transcript editor working from noisy translated source text.
|
||||
|
||||
Goal:
|
||||
Produce the most faithful readable English update.
|
||||
|
||||
Decision rules:
|
||||
1. NEW SOURCE TEXT is the primary evidence and should dominate the output.
|
||||
2. Use PREVIOUS SOURCE TEXT and PREVIOUS REFINED CONTEXT only when they clearly help resolve or continue the new material.
|
||||
3. If the new material starts a fresh thought, output only the new material.
|
||||
4. Remove exact or near-exact repetition unless the repetition is clearly intentional rhetoric.
|
||||
5. Do not add explanations, disclaimers, or meta commentary.
|
||||
6. Prefer conservative wording over guessed meaning.
|
||||
7. Return only the revised transcript text."""
|
||||
|
||||
prompt = f"""Edit this transcript update.
|
||||
|
||||
[PREVIOUS REFINED CONTEXT]
|
||||
{previous_refined_context}
|
||||
|
||||
[PREVIOUS SOURCE TEXT]
|
||||
{previous_source_text}
|
||||
|
||||
[NEW SOURCE TEXT]
|
||||
{new_source_text}
|
||||
"""
|
||||
return [
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": prompt},
|
||||
], prompt
|
||||
|
||||
|
||||
def build_line_messages(prev1, prev2, corrected):
|
||||
system_prompt = """You are a real-time English caption corrector for live speech.
|
||||
|
||||
Task:
|
||||
Clean only the current caption line.
|
||||
|
||||
Hard rules:
|
||||
1. Preserve meaning exactly. Never replace current content with prior context.
|
||||
2. Remove disfluencies and false starts.
|
||||
3. Fix punctuation, casing, and obvious STT typos.
|
||||
4. If uncertain, return the original line unchanged.
|
||||
5. Output only one corrected English line."""
|
||||
|
||||
prompt = (
|
||||
"Context (reference only):\n"
|
||||
f"Previous line 1: {prev1}\n"
|
||||
f"Previous line 2: {prev2}\n"
|
||||
"Current line to correct:\n"
|
||||
f"{corrected}\n"
|
||||
"Corrected:"
|
||||
)
|
||||
return [
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": prompt},
|
||||
], prompt
|
||||
|
||||
|
||||
def append_llm_request_log(log_path, entry):
|
||||
if not log_path:
|
||||
return
|
||||
log_dir = os.path.dirname(log_path)
|
||||
if log_dir:
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
with open(log_path, "a", encoding="utf-8") as handle:
|
||||
json.dump(entry, handle, ensure_ascii=False)
|
||||
handle.write("\n")
|
||||
|
||||
|
||||
def load_llm_request_log_entry(log_path, entry_index):
|
||||
if not log_path or not os.path.exists(log_path):
|
||||
raise FileNotFoundError(f"LLM request log not found: {log_path}")
|
||||
|
||||
with open(log_path, "r", encoding="utf-8") as handle:
|
||||
entries = [json.loads(line) for line in handle if line.strip()]
|
||||
|
||||
if not entries:
|
||||
raise ValueError(f"LLM request log is empty: {log_path}")
|
||||
|
||||
if entry_index is None or entry_index == -1:
|
||||
return entries[-1]
|
||||
|
||||
if entry_index < 0:
|
||||
entry_index = len(entries) + entry_index
|
||||
|
||||
if entry_index < 0 or entry_index >= len(entries):
|
||||
raise IndexError(f"LLM request log index {entry_index} is out of range for {len(entries)} entries")
|
||||
|
||||
return entries[entry_index]
|
||||
|
||||
|
||||
def run_llm_prompt_test(args):
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
|
||||
api_key = os.environ.get("OPENAI_API_KEY")
|
||||
log_path = getattr(args, "llm_request_log_path", "logs/llm_requests.jsonl")
|
||||
|
||||
def call_openai(model, messages):
|
||||
response = requests.post(
|
||||
"https://api.openai.com/v1/chat/completions",
|
||||
json={"model": model, "messages": messages},
|
||||
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
||||
timeout=getattr(args, "post_correct_llm_timeout", 8.0),
|
||||
)
|
||||
if response.status_code >= 400:
|
||||
try:
|
||||
payload = response.json()
|
||||
detail = json.dumps(payload, ensure_ascii=True)
|
||||
except Exception:
|
||||
detail = response.text.strip()
|
||||
raise RuntimeError(f"OpenAI API error {response.status_code}: {detail}")
|
||||
return response.json()["choices"][0]["message"]["content"].strip()
|
||||
|
||||
def call_ollama(model, prompt, temperature, system=None):
|
||||
response = requests.post(
|
||||
getattr(args, "post_correct_ollama_url", "http://127.0.0.1:11434/api/generate"),
|
||||
json={
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"system": system or "",
|
||||
"stream": False,
|
||||
"options": {"temperature": temperature},
|
||||
"keep_alive": getattr(args, "post_correct_keep_alive", "30m"),
|
||||
},
|
||||
timeout=getattr(args, "post_correct_llm_timeout", 8.0),
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json().get("response", "").strip()
|
||||
|
||||
def send_test_request(mode, messages, prompt, temperature, metadata, model=None):
|
||||
request_model = model or args.post_correct_model
|
||||
is_openai = request_model.startswith("gpt-")
|
||||
entry = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"mode": mode,
|
||||
"provider": "openai" if is_openai else "ollama",
|
||||
"model": request_model,
|
||||
"temperature": temperature,
|
||||
"messages": messages,
|
||||
"metadata": metadata,
|
||||
}
|
||||
started_at = time.time()
|
||||
if is_openai:
|
||||
if not api_key:
|
||||
raise RuntimeError("OPENAI_API_KEY is not set")
|
||||
try:
|
||||
response_text = call_openai(request_model, messages)
|
||||
entry["response"] = {"status": "ok", "content": response_text}
|
||||
return response_text
|
||||
except Exception as exc:
|
||||
entry["response"] = {"status": "error", "error": str(exc)}
|
||||
raise
|
||||
finally:
|
||||
entry["duration_ms"] = round((time.time() - started_at) * 1000, 2)
|
||||
append_llm_request_log(log_path, entry)
|
||||
system = ""
|
||||
for message in messages:
|
||||
if message.get("role") == "system":
|
||||
system = message.get("content", "")
|
||||
break
|
||||
try:
|
||||
response_text = call_ollama(request_model, prompt, temperature, system=system)
|
||||
entry["response"] = {"status": "ok", "content": response_text}
|
||||
return response_text
|
||||
except Exception as exc:
|
||||
entry["response"] = {"status": "error", "error": str(exc)}
|
||||
raise
|
||||
finally:
|
||||
entry["duration_ms"] = round((time.time() - started_at) * 1000, 2)
|
||||
append_llm_request_log(log_path, entry)
|
||||
|
||||
if getattr(args, "llm_test_from_log", None) is not None:
|
||||
logged_entry = load_llm_request_log_entry(log_path, args.llm_test_from_log)
|
||||
logged_messages = logged_entry.get("messages")
|
||||
if not logged_messages:
|
||||
raise ValueError("Selected log entry does not contain messages")
|
||||
logged_prompt = next((m.get("content", "") for m in logged_messages if m.get("role") == "user"), "")
|
||||
logged_model = logged_entry.get("model", args.post_correct_model)
|
||||
replay_model = logged_model if getattr(args, "llm_test_use_logged_model", False) else args.post_correct_model
|
||||
response = send_test_request(
|
||||
logged_entry.get("mode", "replay"),
|
||||
logged_messages,
|
||||
logged_prompt,
|
||||
logged_entry.get("temperature", 0.1),
|
||||
{
|
||||
"replay_source_log_path": log_path,
|
||||
"replay_source_index": args.llm_test_from_log,
|
||||
"replay_source_timestamp": logged_entry.get("timestamp"),
|
||||
"replay_source_model": logged_model,
|
||||
},
|
||||
model=replay_model,
|
||||
)
|
||||
print("[LLM TEST] Replayed request from log:")
|
||||
print(f"source_index={args.llm_test_from_log} source_model={logged_model} replay_model={replay_model}")
|
||||
print(response)
|
||||
|
||||
if getattr(args, "llm_test_line", None):
|
||||
messages, prompt = build_line_messages(
|
||||
getattr(args, "llm_test_prev1", ""),
|
||||
getattr(args, "llm_test_prev2", ""),
|
||||
args.llm_test_line,
|
||||
)
|
||||
response = send_test_request(
|
||||
"line",
|
||||
messages,
|
||||
prompt,
|
||||
0.1,
|
||||
{
|
||||
"prev1": getattr(args, "llm_test_prev1", ""),
|
||||
"prev2": getattr(args, "llm_test_prev2", ""),
|
||||
"input": args.llm_test_line,
|
||||
},
|
||||
)
|
||||
print("[LLM TEST] Line response:")
|
||||
print(response)
|
||||
|
||||
if getattr(args, "llm_test_segments", None):
|
||||
messages, prompt = build_paragraph_messages(
|
||||
getattr(args, "llm_test_context", ""),
|
||||
args.llm_test_segments,
|
||||
)
|
||||
response = send_test_request(
|
||||
"paragraph",
|
||||
messages,
|
||||
prompt,
|
||||
0.1,
|
||||
{
|
||||
"context": getattr(args, "llm_test_context", ""),
|
||||
"segments": args.llm_test_segments,
|
||||
},
|
||||
)
|
||||
print("[LLM TEST] Paragraph response:")
|
||||
print(response)
|
||||
|
||||
def run_llm_processor(in_queue, out_queue, args):
|
||||
from dotenv import load_dotenv
|
||||
@@ -29,6 +267,7 @@ def run_llm_processor(in_queue, out_queue, args):
|
||||
api_key = os.environ.get("OPENAI_API_KEY")
|
||||
is_openai = args.post_correct_model.startswith("gpt-")
|
||||
llm_state = {"line_warned": False, "paragraph_warned": False}
|
||||
log_path = getattr(args, "llm_request_log_path", "logs/llm_requests.jsonl")
|
||||
|
||||
def check_ollama_health():
|
||||
try:
|
||||
@@ -130,49 +369,49 @@ def run_llm_processor(in_queue, out_queue, args):
|
||||
return response.json().get("response", "").strip()
|
||||
|
||||
def call_llm(messages, prompt, temperature, warn_key):
|
||||
entry = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"mode": "paragraph" if warn_key == "paragraph_warned" else "line",
|
||||
"provider": "openai" if is_openai else "ollama",
|
||||
"model": args.post_correct_model,
|
||||
"temperature": temperature,
|
||||
"messages": messages,
|
||||
}
|
||||
started_at = time.time()
|
||||
try:
|
||||
if is_openai:
|
||||
if not api_key:
|
||||
raise RuntimeError("OPENAI_API_KEY is not set")
|
||||
return call_openai(messages)
|
||||
response_text = call_openai(messages)
|
||||
entry["response"] = {"status": "ok", "content": response_text}
|
||||
return response_text
|
||||
system = ""
|
||||
if messages:
|
||||
for message in messages:
|
||||
if message.get("role") == "system":
|
||||
system = message.get("content", "")
|
||||
break
|
||||
return call_ollama(prompt, temperature, system=system)
|
||||
response_text = call_ollama(prompt, temperature, system=system)
|
||||
entry["response"] = {"status": "ok", "content": response_text}
|
||||
return response_text
|
||||
except Exception as exc:
|
||||
entry["response"] = {"status": "error", "error": str(exc)}
|
||||
if not llm_state[warn_key]:
|
||||
label = "Paragraph LLM" if warn_key == "paragraph_warned" else "Line LLM"
|
||||
print(f"[LLM] {label} unavailable ({exc}). Falling back to deterministic text.")
|
||||
llm_state[warn_key] = True
|
||||
return ""
|
||||
finally:
|
||||
entry["duration_ms"] = round((time.time() - started_at) * 1000, 2)
|
||||
append_llm_request_log(log_path, entry)
|
||||
|
||||
# State: This is the ONLY text we send to the LLM as context
|
||||
active_context = ""
|
||||
previous_source_window = ""
|
||||
recent_lines = deque(maxlen=2)
|
||||
|
||||
def call_llm_rolling_refine(new_segments_str, context):
|
||||
prompt = f"""Task: Refine the following live transcription stream into clean, professional paragraphs.
|
||||
|
||||
[PREVIOUS WORKING CONTEXT]
|
||||
"{context}"
|
||||
|
||||
[NEW RAW ASR SEGMENTS]
|
||||
"{new_segments_str}"
|
||||
|
||||
[INSTRUCTIONS]
|
||||
1. INTEGRATE: Polished and merge the new segments into the flow of the 'PREVIOUS WORKING CONTEXT'.
|
||||
2. CONSOLIDATE: Remove redundant repetitions and translator echoes.
|
||||
3. ORGANIZE: Use a double newline (\\n\\n) to start a new paragraph when a topic changes or the current one is complete.
|
||||
4. TARGET LANGUAGE: Output ONLY in English.
|
||||
5. OUTPUT: Provide ONLY the refined, consolidated text. Do not explain anything.
|
||||
"""
|
||||
messages = [
|
||||
{"role": "system", "content": "You are a live transcript editor."},
|
||||
{"role": "user", "content": prompt},
|
||||
]
|
||||
def call_llm_rolling_refine(new_segments_str, context, previous_source_text):
|
||||
messages, prompt = build_paragraph_messages(context, previous_source_text, new_segments_str)
|
||||
return call_llm(messages, prompt, 0.1, "paragraph_warned")
|
||||
|
||||
def post_correct_line(text):
|
||||
@@ -185,27 +424,7 @@ def run_llm_processor(in_queue, out_queue, args):
|
||||
|
||||
prev2 = recent_lines[-2] if len(recent_lines) >= 2 else ""
|
||||
prev1 = recent_lines[-1] if len(recent_lines) >= 1 else ""
|
||||
prompt = (
|
||||
"You are a real-time English caption corrector for live speech.\n"
|
||||
"Task:\n"
|
||||
"Clean ONLY the current caption line.\n"
|
||||
"Hard rules:\n"
|
||||
"1. Preserve meaning exactly. Never replace current content with prior context.\n"
|
||||
"2. Remove disfluencies and false starts.\n"
|
||||
"3. Fix punctuation, casing, and obvious STT typos.\n"
|
||||
"4. If uncertain, return the original line unchanged.\n"
|
||||
"5. Output ONLY one corrected English line.\n"
|
||||
"Context (reference only):\n"
|
||||
f"Previous line 1: {prev1}\n"
|
||||
f"Previous line 2: {prev2}\n"
|
||||
"Current line to correct:\n"
|
||||
f"{corrected}\n"
|
||||
"Corrected:"
|
||||
)
|
||||
messages = [
|
||||
{"role": "system", "content": "You are a real-time English caption corrector."},
|
||||
{"role": "user", "content": prompt},
|
||||
]
|
||||
messages, prompt = build_line_messages(prev1, prev2, corrected)
|
||||
candidate = normalize_english_caption(call_llm(messages, prompt, 0.1, "line_warned"))
|
||||
if candidate and word_overlap_ratio(corrected, candidate) >= getattr(args, "post_correct_min_overlap", 0.45):
|
||||
return candidate
|
||||
@@ -244,7 +463,7 @@ def run_llm_processor(in_queue, out_queue, args):
|
||||
paragraph_from_llm = False
|
||||
if args.llm_paragraph and should_refine:
|
||||
new_batch = " ".join(pending_buffer)
|
||||
result = call_llm_rolling_refine(new_batch, active_context)
|
||||
result = call_llm_rolling_refine(new_batch, active_context, previous_source_window)
|
||||
|
||||
if result:
|
||||
# Logic to handle paragraph breaks
|
||||
@@ -263,6 +482,7 @@ def run_llm_processor(in_queue, out_queue, args):
|
||||
else:
|
||||
structured_paragraph = new_batch
|
||||
active_context = new_batch
|
||||
previous_source_window = new_batch
|
||||
|
||||
pending_buffer = []
|
||||
last_llm_call_time = time.time()
|
||||
|
||||
Reference in New Issue
Block a user