From 3fad171905e31cb849bccfe3d1d8c3114843ccf7 Mon Sep 17 00:00:00 2001 From: Adolfo Reyna Date: Fri, 7 Aug 2026 18:37:07 -0400 Subject: [PATCH] Fix Gemma chat template system prompt compatibility in apple_llm.py --- apple_llm.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/apple_llm.py b/apple_llm.py index e362069..81d2ed3 100644 --- a/apple_llm.py +++ b/apple_llm.py @@ -195,12 +195,27 @@ class MacOSLLM(FrameProcessor): def _gen(): from mlx_lm import generate from mlx_lm.sample_utils import make_sampler + # Keep history concise to avoid context drift and repetition - recent_history = self._history[-8:] - messages = [{"role": "system", "content": self._system_prompt}] + recent_history - prompt = self._mlx_tokenizer.apply_chat_template( - messages, add_generation_prompt=True, tokenize=False - ) + recent_history = [dict(m) for m in self._history[-8:]] + if recent_history: + if self._system_prompt: + recent_history[0] = { + "role": recent_history[0]["role"], + "content": f"{self._system_prompt}\n\n{recent_history[0]['content']}" + } + messages = recent_history + else: + messages = [{"role": "user", "content": f"{self._system_prompt}\n\n{utterance}"}] + + try: + prompt = self._mlx_tokenizer.apply_chat_template( + messages, add_generation_prompt=True, tokenize=False + ) + except Exception: + # Fallback for models without chat template support + prompt = f"{self._system_prompt}\n\nUser: {utterance}\nAssistant:" + sampler = make_sampler(temp=0.7) return generate( self._mlx_model,