adolforeyna/VoiceAgent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add AGENTS.md template and fix repetition penalty for local macOS LLM

Browse Source
This commit is contained in:
Adolfo Reyna
2026-08-07 18:27:51 -04:00
parent 598d55e383
commit f579a26cc0
3 changed files with 20 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apple_llm.py
+12 -5
View File
@@ -92,7 +92,7 @@ class MacOSLLM(FrameProcessor):
def __init__(
self,
*,
model: str = "mlx-community/Qwen2.5-0.5B-Instruct-4bit",
model: str = "mlx-community/Llama-3.2-1B-Instruct-4bit",
system_prompt: str | None = None,
observer=None,
**kwargs,
@@ -194,7 +194,9 @@ class MacOSLLM(FrameProcessor):
loop = asyncio.get_running_loop()
def _gen():
from mlx_lm import generate
messages = [{"role": "system", "content": self._system_prompt}] + self._history
# Keep history concise to avoid context drift and repetition
recent_history = self._history[-8:]
messages = [{"role": "system", "content": self._system_prompt}] + recent_history
prompt = self._mlx_tokenizer.apply_chat_template(
messages, add_generation_prompt=True, tokenize=False
)
@@ -202,13 +204,18 @@ class MacOSLLM(FrameProcessor):
self._mlx_model,
self._mlx_tokenizer,
prompt=prompt,
max_tokens=256,
max_tokens=150,
temp=0.7,
repetition_penalty=1.2,
repetition_context_size=30,
verbose=False,
)
response_text = await loop.run_in_executor(None, _gen)
chunks.append(response_text)
await self.push_frame(LLMTextFrame(response_text))
# Clean up any potential repetitive sentence tails
cleaned_text = response_text.strip()
chunks.append(cleaned_text)
await self.push_frame(LLMTextFrame(cleaned_text))
except asyncio.CancelledError:
logger.info("Turn cancelled mid-response.")