adolforeyna/whisper-translation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tune paragraph prompts for multilingual noise

Browse Source
This commit is contained in:
Adolfo Reyna
2026-03-17 11:51:53 -04:00
parent 7da02a9697
commit 7451d164b7
5 changed files with 304 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
prompt_experiments.py
+252 -28
View File
@@ -7,6 +7,14 @@ from collections import Counter
from urllib import request, error
FOREIGN_INSERTIONS = [
"pero en espanol dice que no retrocedas",
"y tambien dijo que Dios responde",
"mas ele continuou em fe",
"et il a dit de rester ferme",
]
def parse_distribute_debug_log(path):
entries = []
current = {"final": None, "en_bridge": None, "llm_paragraph": None}
@@ -65,7 +73,39 @@ def reconstruct_samples(path):
return samples
def build_variants(sample):
def inject_multilingual_noise(text, sample_index, slot_name):
if not text:
return text
inserts = [
FOREIGN_INSERTIONS[(sample_index + 0) % len(FOREIGN_INSERTIONS)],
FOREIGN_INSERTIONS[(sample_index + 1) % len(FOREIGN_INSERTIONS)],
]
sentences = [part.strip() for part in re.split(r"(?<=[.!?])\s+", text.strip()) if part.strip()]
if not sentences:
return text
if len(sentences) == 1:
return f'{sentences[0]} {inserts[0]}.'
head = sentences[0]
tail = " ".join(sentences[1:])
if slot_name == "previous_raw":
return f"{head} {inserts[0]}. {tail}"
return f"{head} {tail} {inserts[1]}."
def maybe_augment_samples(samples, mode):
if mode != "multilingual_noise":
return samples
augmented = []
for idx, sample in enumerate(samples):
updated = dict(sample)
updated["previous_raw"] = inject_multilingual_noise(sample["previous_raw"], idx, "previous_raw")
updated["new_raw"] = inject_multilingual_noise(sample["new_raw"], idx, "new_raw")
augmented.append(updated)
return augmented
def build_variants(sample, variant_filter=None):
prev_refined = sample["previous_refined"]
prev_raw = sample["previous_raw"]
new_raw = sample["new_raw"]
@@ -198,17 +238,140 @@ Decision rules:
"""
variants.append(("grounded_new_dominant", system_e, user_e))
system_g = """You are a careful live transcript editor working from noisy translated source text.
Goal:
Produce the most faithful readable English update.
Decision rules:
1. NEW SOURCE TEXT is the primary evidence and should dominate the output.
2. Use PREVIOUS SOURCE TEXT and PREVIOUS REFINED CONTEXT only when they clearly help resolve or continue the new material.
3. The final answer must be English only.
4. If source text contains Spanish, Portuguese, French, or other non-English fragments, translate their meaning into English when reasonably clear.
5. Never copy non-English words into the output unless they are proper names.
6. If the new material starts a fresh thought, output only the new material.
7. Remove exact or near-exact repetition unless the repetition is clearly intentional rhetoric.
8. Do not add explanations, disclaimers, or meta commentary.
9. Prefer conservative wording over guessed meaning.
10. Return only the revised transcript text."""
user_g = f"""Edit this transcript update.
[PREVIOUS REFINED CONTEXT]
"{prev_refined}"
[PREVIOUS SOURCE TEXT]
"{prev_raw}"
[NEW SOURCE TEXT]
"{new_raw}"
"""
variants.append(("grounded_english_only_translate", system_g, user_g))
system_h = """You are a careful live transcript editor working from noisy translated source text.
Goal:
Produce the most faithful readable English update.
Decision rules:
1. NEW SOURCE TEXT is the primary evidence and should dominate the output.
2. Use PREVIOUS SOURCE TEXT and PREVIOUS REFINED CONTEXT only when they clearly help resolve or continue the new material.
3. The final answer must be English only.
4. Treat any Spanish, Portuguese, French, or other non-English fragments as contamination from surrounding audio or translation noise unless they clearly repeat the same idea as nearby English.
5. When a non-English fragment clearly repeats nearby English content, keep only the English meaning once.
6. Never copy non-English words into the output unless they are proper names.
7. If the new material starts a fresh thought, output only the new material.
8. Remove exact or near-exact repetition unless the repetition is clearly intentional rhetoric.
9. Do not add explanations, disclaimers, or meta commentary.
10. Prefer conservative wording over guessed meaning.
11. Return only the revised transcript text."""
user_h = f"""Edit this transcript update.
[PREVIOUS REFINED CONTEXT]
"{prev_refined}"
[PREVIOUS SOURCE TEXT]
"{prev_raw}"
[NEW SOURCE TEXT]
"{new_raw}"
"""
variants.append(("grounded_english_only_ignore_noise", system_h, user_h))
system_i = """You are a careful live transcript editor working from noisy translated source text.
Goal:
Produce the most faithful readable English update.
Decision rules:
1. NEW SOURCE TEXT is the primary evidence and should dominate the output.
2. Use PREVIOUS SOURCE TEXT and PREVIOUS REFINED CONTEXT only when they clearly help resolve or continue the new material.
3. The final answer must be English only.
4. Never copy non-English words into the output unless they are proper names.
5. If a non-English fragment appears as an isolated clause, trailing fragment, or side comment without a clear English anchor, drop it.
6. Only translate a non-English fragment when it is clearly central to the same idea and its meaning is reasonably obvious from context.
7. If the new material starts a fresh thought, output only the new material.
8. Remove exact or near-exact repetition unless the repetition is clearly intentional rhetoric.
9. Do not add explanations, disclaimers, or meta commentary.
10. Prefer conservative wording over guessed meaning.
11. Return only the revised transcript text."""
user_i = f"""Edit this transcript update.
[PREVIOUS REFINED CONTEXT]
"{prev_refined}"
[PREVIOUS SOURCE TEXT]
"{prev_raw}"
[NEW SOURCE TEXT]
"{new_raw}"
"""
variants.append(("grounded_english_only_drop_isolated", system_i, user_i))
system_f = """You are a careful live transcript editor working from noisy translated source text.
Goal:
Produce the most faithful readable English update.
Decision rules:
1. NEW SOURCE TEXT is the primary evidence and should dominate the output.
2. Use PREVIOUS SOURCE TEXT and PREVIOUS REFINED CONTEXT only when they clearly help resolve or continue the new material.
3. If the new material starts a fresh thought, output only the new material.
4. Remove exact or near-exact repetition unless the repetition is clearly intentional rhetoric.
5. If the source repeats the same short clause three or more times in a row, keep at most two repetitions unless losing more would change the speaker's emphasis.
6. Do not add explanations, disclaimers, or meta commentary.
7. Prefer conservative wording over guessed meaning.
8. Return only the revised transcript text."""
user_f = f"""Edit this transcript update.
[PREVIOUS REFINED CONTEXT]
"{prev_refined}"
[PREVIOUS SOURCE TEXT]
"{prev_raw}"
[NEW SOURCE TEXT]
"{new_raw}"
"""
variants.append(("grounded_new_dominant_dedupe", system_f, user_f))
if variant_filter:
allowed = set(x.strip() for x in variant_filter.split(",") if x.strip())
variants = [variant for variant in variants if variant[0] in allowed]
return variants
def ollama_generate(model, system_prompt, user_prompt, url, timeout, keep_alive):
def ollama_generate(model, system_prompt, user_prompt, url, timeout, keep_alive, temperature, top_p, repeat_penalty):
payload = json.dumps(
{
"model": model,
"prompt": user_prompt,
"system": system_prompt,
"stream": False,
"options": {"temperature": 0.1},
"options": {
"temperature": temperature,
"top_p": top_p,
"repeat_penalty": repeat_penalty,
},
"keep_alive": keep_alive,
}
).encode("utf-8")
@@ -258,43 +421,92 @@ def evaluate_output(output, sample):
"output_len": len(tokenize(output)),
"repeated_ngram_ratio": round(repeated_ngram_ratio(output), 4),
"unsupported_token_ratio": round(unsupported_token_ratio(output, evidence), 4),
"foreign_leak_ratio": round(foreign_leak_ratio(output), 4),
}
def foreign_leak_ratio(output):
foreign_markers = {
"pero", "espanol", "tambien", "dijo", "dios", "responde",
"mas", "ele", "continuou", "fe",
"et", "rester", "ferme",
}
tokens = tokenize(output)
if not tokens:
return 0.0
leaked = [tok for tok in tokens if tok in foreign_markers]
return len(leaked) / len(tokens)
def parse_float_grid(raw_value, default_values):
if not raw_value:
return default_values
return [float(x.strip()) for x in raw_value.split(",") if x.strip()]
def build_param_sets(args):
temperatures = parse_float_grid(args.temperature_grid, [0.1])
top_ps = parse_float_grid(args.top_p_grid, [0.9])
repeat_penalties = parse_float_grid(args.repeat_penalty_grid, [1.1])
param_sets = []
for temperature in temperatures:
for top_p in top_ps:
for repeat_penalty in repeat_penalties:
label = f"temp={temperature}_top_p={top_p}_repeat_penalty={repeat_penalty}"
param_sets.append(
{
"label": label,
"temperature": temperature,
"top_p": top_p,
"repeat_penalty": repeat_penalty,
}
)
return param_sets
def run_experiments(args):
samples = reconstruct_samples(args.log_file)
samples = maybe_augment_samples(samples, args.sample_mode)
if args.limit:
samples = samples[: args.limit]
param_sets = build_param_sets(args)
results = []
for index, sample in enumerate(samples):
print(f"Running sample {index + 1}/{len(samples)}")
sample_results = []
for variant_name, system_prompt, user_prompt in build_variants(sample):
for variant_name, system_prompt, user_prompt in build_variants(sample, args.variant_filter):
print(f" variant={variant_name}")
try:
output = ollama_generate(
args.model,
system_prompt,
user_prompt,
args.ollama_url,
args.timeout,
args.keep_alive,
for params in param_sets:
print(f" params={params['label']}")
try:
output = ollama_generate(
args.model,
system_prompt,
user_prompt,
args.ollama_url,
args.timeout,
args.keep_alive,
params["temperature"],
params["top_p"],
params["repeat_penalty"],
)
metrics = evaluate_output(output, sample)
error_message = None
except Exception as exc:
output = ""
metrics = None
error_message = str(exc)
sample_results.append(
{
"variant": variant_name,
"params": params,
"output": output,
"metrics": metrics,
"error": error_message,
}
)
metrics = evaluate_output(output, sample)
error_message = None
except Exception as exc:
output = ""
metrics = None
error_message = str(exc)
sample_results.append(
{
"variant": variant_name,
"output": output,
"metrics": metrics,
"error": error_message,
}
)
results.append(
{
"sample_index": index,
@@ -309,9 +521,16 @@ def summarize_results(results):
summary = {}
for sample_result in results:
for variant_result in sample_result["results"]:
summary_key = f"{variant_result['variant']}|{variant_result['params']['label']}"
bucket = summary.setdefault(
variant_result["variant"],
{"repeated_ngram_ratio": [], "unsupported_token_ratio": [], "output_len": [], "errors": 0},
summary_key,
{
"repeated_ngram_ratio": [],
"unsupported_token_ratio": [],
"foreign_leak_ratio": [],
"output_len": [],
"errors": 0,
},
)
if variant_result.get("error"):
bucket["errors"] += 1
@@ -336,6 +555,11 @@ def main():
parser.add_argument("--keep-alive", type=str, default="30m")
parser.add_argument("--limit", type=int, default=0)
parser.add_argument("--output-file", type=str, default="logs/prompt_experiment_results.json")
parser.add_argument("--variant-filter", type=str, default="", help="Comma-separated subset of variants to run.")
parser.add_argument("--temperature-grid", type=str, default="", help="Comma-separated temperatures to test.")
parser.add_argument("--top-p-grid", type=str, default="", help="Comma-separated top_p values to test.")
parser.add_argument("--repeat-penalty-grid", type=str, default="", help="Comma-separated repeat_penalty values to test.")
parser.add_argument("--sample-mode", type=str, default="base", help="Sample mode: base or multilingual_noise.")
args = parser.parse_args()
results = run_experiments(args)