Fix Gemma chat template system prompt compatibility in apple_llm.py
This commit is contained in:
+20
-5
@@ -195,12 +195,27 @@ class MacOSLLM(FrameProcessor):
|
||||
def _gen():
|
||||
from mlx_lm import generate
|
||||
from mlx_lm.sample_utils import make_sampler
|
||||
|
||||
# Keep history concise to avoid context drift and repetition
|
||||
recent_history = self._history[-8:]
|
||||
messages = [{"role": "system", "content": self._system_prompt}] + recent_history
|
||||
prompt = self._mlx_tokenizer.apply_chat_template(
|
||||
messages, add_generation_prompt=True, tokenize=False
|
||||
)
|
||||
recent_history = [dict(m) for m in self._history[-8:]]
|
||||
if recent_history:
|
||||
if self._system_prompt:
|
||||
recent_history[0] = {
|
||||
"role": recent_history[0]["role"],
|
||||
"content": f"{self._system_prompt}\n\n{recent_history[0]['content']}"
|
||||
}
|
||||
messages = recent_history
|
||||
else:
|
||||
messages = [{"role": "user", "content": f"{self._system_prompt}\n\n{utterance}"}]
|
||||
|
||||
try:
|
||||
prompt = self._mlx_tokenizer.apply_chat_template(
|
||||
messages, add_generation_prompt=True, tokenize=False
|
||||
)
|
||||
except Exception:
|
||||
# Fallback for models without chat template support
|
||||
prompt = f"{self._system_prompt}\n\nUser: {utterance}\nAssistant:"
|
||||
|
||||
sampler = make_sampler(temp=0.7)
|
||||
return generate(
|
||||
self._mlx_model,
|
||||
|
||||
Reference in New Issue
Block a user