From e3cd538dcaea5a661088ff5bf6afc049cedf4127 Mon Sep 17 00:00:00 2001 From: Adolfo Reyna Date: Fri, 6 Mar 2026 19:11:09 -0500 Subject: [PATCH] feat: load glossary corrections from file when available --- history.md | 8 ++++++++ transcribe.py | 25 ++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/history.md b/history.md index 58d59d9..470cda1 100644 --- a/history.md +++ b/history.md @@ -144,3 +144,11 @@ The project now features a high-performance, Apple Silicon-optimized pipeline th - **Glossary Replacements:** Added configurable `SOURCE=TARGET` term normalization (`--glossary-pair`) to consistently correct names/terms in English captions. - **Rolling Caption Revision Buffer:** Added a short correction window (`--caption-correction-window`, default 3s) with a 2-line rolling buffer that can merge incomplete English lines into a cleaner sentence as new context arrives. - **Outcome:** Captions now support lightweight post-correction behavior that improves continuity and term consistency while remaining compatible with real-time streaming. + +## Phase 18: File-Based Glossary Loading +- **Goal:** Make glossary corrections persistent and easier to maintain without long CLI commands. +- **Approach:** + - Added `--glossary-file` (default `glossary.txt`) to load `SOURCE=TARGET` term mappings when the file exists. + - Implemented comment/blank-line support and invalid-line skipping with line-level warnings. + - Merged file-based glossary entries with repeatable `--glossary-pair` CLI entries for runtime overrides. +- **Outcome:** English caption correction can now use a maintained glossary file automatically, while preserving ad-hoc CLI term fixes. diff --git a/transcribe.py b/transcribe.py index 61e739f..84a9bd2 100644 --- a/transcribe.py +++ b/transcribe.py @@ -1,6 +1,7 @@ import sys import time import re +import os import requests import threading import json @@ -181,6 +182,23 @@ def apply_glossary(text, glossary_pairs): updated = pattern.sub(target, updated) return updated +def load_glossary_file(path): + """Load SOURCE=TARGET glossary pairs from a file.""" + pairs = [] + if not path or not os.path.exists(path): + return pairs + + with open(path, "r", encoding="utf-8") as f: + for line_no, raw in enumerate(f, start=1): + line = raw.strip() + if not line or line.startswith("#"): + continue + try: + pairs.append(parse_glossary_pair(line)) + except argparse.ArgumentTypeError: + print(f"[SYSTEM]: Skipping invalid glossary line {line_no} in {path}: {line}") + return pairs + def normalize_english_caption(text): normalized = " ".join(text.split()).strip() if not normalized: @@ -306,9 +324,14 @@ def main(): parser.add_argument("--retry-logprob-threshold", type=float, default=-1.05, help="Retry transcription when mean avg_logprob is below this threshold (default: -1.05)") parser.add_argument("--retry-compression-threshold", type=float, default=2.4, help="Retry transcription when max compression ratio exceeds this threshold (default: 2.4)") parser.add_argument("--caption-correction-window", type=float, default=3.0, help="Seconds where previous English line can still be merged/corrected (default: 3.0)") + parser.add_argument("--glossary-file", type=str, default="glossary.txt", help="Path to glossary file with SOURCE=TARGET pairs (default: glossary.txt if present)") parser.add_argument("--glossary-pair", action="append", type=parse_glossary_pair, default=[], help="Term replacement pair SOURCE=TARGET (repeatable)") args = parser.parse_args() + file_glossary_pairs = load_glossary_file(args.glossary_file) + merged_glossary_pairs = file_glossary_pairs + args.glossary_pair + if file_glossary_pairs: + print(f"Loaded {len(file_glossary_pairs)} glossary terms from {args.glossary_file}") if args.quantize: WHISPER_MODEL = "mlx-community/whisper-small-mlx-4bit" @@ -530,7 +553,7 @@ def main(): english_for_translation = english_text if args.smart_correct and english_for_translation: english_for_translation = normalize_english_caption(english_for_translation) - english_for_translation = apply_glossary(english_for_translation, args.glossary_pair) + english_for_translation = apply_glossary(english_for_translation, merged_glossary_pairs) english_for_caption = english_for_translation if args.smart_correct and english_for_caption: