Fix Gemma chat template system prompt compatibility in apple_llm.py

This commit is contained in:
Adolfo Reyna
2026-08-07 18:37:07 -04:00
parent c50da6f29c
commit 3fad171905
+20 -5
View File
@@ -195,12 +195,27 @@ class MacOSLLM(FrameProcessor):
def _gen(): def _gen():
from mlx_lm import generate from mlx_lm import generate
from mlx_lm.sample_utils import make_sampler from mlx_lm.sample_utils import make_sampler
# Keep history concise to avoid context drift and repetition # Keep history concise to avoid context drift and repetition
recent_history = self._history[-8:] recent_history = [dict(m) for m in self._history[-8:]]
messages = [{"role": "system", "content": self._system_prompt}] + recent_history if recent_history:
prompt = self._mlx_tokenizer.apply_chat_template( if self._system_prompt:
messages, add_generation_prompt=True, tokenize=False recent_history[0] = {
) "role": recent_history[0]["role"],
"content": f"{self._system_prompt}\n\n{recent_history[0]['content']}"
}
messages = recent_history
else:
messages = [{"role": "user", "content": f"{self._system_prompt}\n\n{utterance}"}]
try:
prompt = self._mlx_tokenizer.apply_chat_template(
messages, add_generation_prompt=True, tokenize=False
)
except Exception:
# Fallback for models without chat template support
prompt = f"{self._system_prompt}\n\nUser: {utterance}\nAssistant:"
sampler = make_sampler(temp=0.7) sampler = make_sampler(temp=0.7)
return generate( return generate(
self._mlx_model, self._mlx_model,