diff --git a/engine_distribute.py b/engine_distribute.py index f231ee6..573cd51 100644 --- a/engine_distribute.py +++ b/engine_distribute.py @@ -53,7 +53,7 @@ def run_distribution(in_queue, args): speaker_label = payload.get('speaker') speaker_prefix = f"[{speaker_label}]: " if speaker_label else "" - msg = f"\n[Final]: {speaker_prefix}{payload.get('original', '')}" + msg = f"[Final]: {speaker_prefix}{payload.get('original', '')}" print(msg, flush=True) log_debug(msg) @@ -73,9 +73,13 @@ def run_distribution(in_queue, args): log_debug(llm_line_msg) if payload.get("used_llm_paragraph") and paragraph_text: + print("", flush=True) + log_debug("") llm_paragraph_msg = f"[LLM-PARAGRAPH]: {speaker_prefix}{paragraph_text}" print(llm_paragraph_msg, flush=True) log_debug(llm_paragraph_msg) + print("", flush=True) + log_debug("") if payload.get("paragraph_fallback") and paragraph_text: paragraph_msg = f"[PARAGRAPH]: {speaker_prefix}{paragraph_text}" diff --git a/history.md b/history.md index 204ffd6..a51dc96 100644 --- a/history.md +++ b/history.md @@ -307,3 +307,11 @@ The project now features a high-performance, Apple Silicon-optimized pipeline th - Tightened the live paragraph system prompt in `engine_llm.py` so output must stay English-only, non-English fragments should not be copied through, isolated trailing foreign fragments should be dropped, and only clearly central foreign content should be translated into English. - Added paragraph-specific generation controls to `main_v2.py` and `config.json` so paragraph tuning can evolve independently of line-correction behavior. - **Outcome:** Paragraph refinement is now stricter about keeping the final output in English while preserving the more grounded context layout and the best-tested Ollama generation settings. + +## Phase 35: Previous-Context Scope + Previous-Source Final Check +- **Goal:** Decide whether the paragraph refiner should keep receiving previous source text when experimenting with multiple prior refined paragraphs as context. +- **Approach:** + - Extended `prompt_experiments.py` to compare `2` and `4` previous refined paragraphs with and without `PREVIOUS SOURCE TEXT`. + - Ran the final focused comparison on later reconstructed samples using deterministic generation (`temperature=0.0`) to minimize sampling noise. + - Compared the resulting tradeoffs in unsupported content, repetition, and output length instead of looking only at one-off wins. +- **Outcome:** Keeping `PREVIOUS SOURCE TEXT` remains the safer choice. The best result from this final sweep was `2` previous refined paragraphs plus previous source text; removing previous source text consistently hurt grounding, while larger refined-context windows increased drift risk. diff --git a/prompt_experiments.py b/prompt_experiments.py index d0f8e10..b2b5ee1 100644 --- a/prompt_experiments.py +++ b/prompt_experiments.py @@ -44,6 +44,7 @@ def reconstruct_samples(path): samples = [] rows = parse_distribute_debug_log(path) previous_llm = "" + previous_llm_history = [] pending_bridges = [] recent_raw_windows = [] @@ -60,6 +61,7 @@ def reconstruct_samples(path): samples.append( { "previous_refined": previous_llm, + "previous_refined_history": list(previous_llm_history), "previous_raw": prev_raw, "new_raw": new_raw, "observed_output": llm_output, @@ -67,6 +69,7 @@ def reconstruct_samples(path): ) previous_llm = llm_output + previous_llm_history.append(llm_output) recent_raw_windows.append(new_raw) pending_bridges = [] @@ -107,6 +110,7 @@ def maybe_augment_samples(samples, mode): def build_variants(sample, variant_filter=None): prev_refined = sample["previous_refined"] + prev_refined_history = sample.get("previous_refined_history", []) prev_raw = sample["previous_raw"] new_raw = sample["new_raw"] @@ -354,6 +358,79 @@ Decision rules: """ variants.append(("grounded_new_dominant_dedupe", system_f, user_f)) + for window_size in [2, 4, 6, 8, 10]: + context_window = "\n\n".join(prev_refined_history[-window_size:]).strip() + system_ctx = """You are a careful live transcript editor working from noisy translated source text. + +Goal: +Produce the most faithful readable English update. + +Decision rules: +1. NEW SOURCE TEXT is the primary evidence and should dominate the output. +2. Use PREVIOUS REFINED CONTEXT and PREVIOUS SOURCE TEXT only when they clearly help resolve or continue the new material. +3. The final answer must be English only. +4. Never copy non-English words into the output unless they are proper names. +5. If a non-English fragment appears as an isolated clause, trailing fragment, or side comment without a clear English anchor, drop it. +6. Only translate a non-English fragment when it is clearly central to the same idea and its meaning is reasonably obvious from context. +7. If the new material starts a fresh thought, output only the new material. +8. Remove exact or near-exact repetition unless the repetition is clearly intentional rhetoric. +9. Do not add explanations, disclaimers, or meta commentary. +10. Prefer conservative wording over guessed meaning. +11. Return only the revised transcript text.""" + user_ctx = f"""Edit this transcript update. + +[PREVIOUS REFINED CONTEXT WINDOW] +{context_window} + +[PREVIOUS SOURCE TEXT] +"{prev_raw}" + +[NEW SOURCE TEXT] +"{new_raw}" + """ + variants.append((f"grounded_ctx{window_size}", system_ctx, user_ctx)) + + if window_size in (2, 4): + user_ctx_no_prev_source = f"""Edit this transcript update. + +[PREVIOUS REFINED CONTEXT WINDOW] +{context_window} + +[NEW SOURCE TEXT] +"{new_raw}" +""" + variants.append((f"grounded_ctx{window_size}_no_prev_source", system_ctx, user_ctx_no_prev_source)) + + system_ctx_repair = """You are a careful live transcript editor working from noisy translated source text. + +Goal: +Produce the most faithful readable English update. + +Decision rules: +1. NEW SOURCE TEXT is the primary evidence and should dominate the output. +2. Use PREVIOUS REFINED CONTEXT and PREVIOUS SOURCE TEXT only when they clearly help resolve or continue the new material. +3. The final answer must be English only. +4. Never copy non-English words into the output unless they are proper names. +5. If a non-English fragment appears as an isolated clause, trailing fragment, or side comment without a clear English anchor, drop it. +6. Correct likely caption-word mistakes only when the surrounding context makes the intended English wording reasonably clear. +7. If a word or phrase is unclear, keep it conservative instead of guessing. +8. If the new material starts a fresh thought, output only the new material. +9. Remove exact or near-exact repetition unless the repetition is clearly intentional rhetoric. +10. Do not add explanations, disclaimers, or meta commentary. +11. Return only the revised transcript text.""" + user_ctx_repair = f"""Edit this transcript update. + +[PREVIOUS REFINED CONTEXT WINDOW] +{context_window} + +[PREVIOUS SOURCE TEXT] +"{prev_raw}" + +[NEW SOURCE TEXT] +"{new_raw}" +""" + variants.append((f"grounded_repair_ctx{window_size}", system_ctx_repair, user_ctx_repair)) + if variant_filter: allowed = set(x.strip() for x in variant_filter.split(",") if x.strip()) variants = [variant for variant in variants if variant[0] in allowed] @@ -468,6 +545,8 @@ def build_param_sets(args): def run_experiments(args): samples = reconstruct_samples(args.log_file) samples = maybe_augment_samples(samples, args.sample_mode) + if args.start_index: + samples = samples[args.start_index:] if args.limit: samples = samples[: args.limit] @@ -554,6 +633,7 @@ def main(): parser.add_argument("--timeout", type=float, default=40.0) parser.add_argument("--keep-alive", type=str, default="30m") parser.add_argument("--limit", type=int, default=0) + parser.add_argument("--start-index", type=int, default=0) parser.add_argument("--output-file", type=str, default="logs/prompt_experiment_results.json") parser.add_argument("--variant-filter", type=str, default="", help="Comma-separated subset of variants to run.") parser.add_argument("--temperature-grid", type=str, default="", help="Comma-separated temperatures to test.")