Fix Gemma chat template system prompt compatibility in apple_llm.py
This commit is contained in:
+20
-5
@@ -195,12 +195,27 @@ class MacOSLLM(FrameProcessor):
|
|||||||
def _gen():
|
def _gen():
|
||||||
from mlx_lm import generate
|
from mlx_lm import generate
|
||||||
from mlx_lm.sample_utils import make_sampler
|
from mlx_lm.sample_utils import make_sampler
|
||||||
|
|
||||||
# Keep history concise to avoid context drift and repetition
|
# Keep history concise to avoid context drift and repetition
|
||||||
recent_history = self._history[-8:]
|
recent_history = [dict(m) for m in self._history[-8:]]
|
||||||
messages = [{"role": "system", "content": self._system_prompt}] + recent_history
|
if recent_history:
|
||||||
prompt = self._mlx_tokenizer.apply_chat_template(
|
if self._system_prompt:
|
||||||
messages, add_generation_prompt=True, tokenize=False
|
recent_history[0] = {
|
||||||
)
|
"role": recent_history[0]["role"],
|
||||||
|
"content": f"{self._system_prompt}\n\n{recent_history[0]['content']}"
|
||||||
|
}
|
||||||
|
messages = recent_history
|
||||||
|
else:
|
||||||
|
messages = [{"role": "user", "content": f"{self._system_prompt}\n\n{utterance}"}]
|
||||||
|
|
||||||
|
try:
|
||||||
|
prompt = self._mlx_tokenizer.apply_chat_template(
|
||||||
|
messages, add_generation_prompt=True, tokenize=False
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
# Fallback for models without chat template support
|
||||||
|
prompt = f"{self._system_prompt}\n\nUser: {utterance}\nAssistant:"
|
||||||
|
|
||||||
sampler = make_sampler(temp=0.7)
|
sampler = make_sampler(temp=0.7)
|
||||||
return generate(
|
return generate(
|
||||||
self._mlx_model,
|
self._mlx_model,
|
||||||
|
|||||||
Reference in New Issue
Block a user