From 6d593cd9ef8f10108aeab8cd89c3b96720278deb Mon Sep 17 00:00:00 2001 From: Adolfo Reyna Date: Tue, 14 Jul 2026 23:42:34 -0400 Subject: [PATCH] feat: initial durable ESP32 voice gateway plugin + patches - Migrates custom voice gateway from in-tree api_server.py patch to durable plugin - Auto-restores routes after hermes update (POST /api/esp32/voice + WS /api/esp32/voice/ws) - Includes live draft ingestion via MacMiniMCP speech_live_transcribe (Apple pipe volatile) - Includes instant smart reply via apple_llm_quick_reply (3B ANE ~300ms, contextual not generic) - Guard for missing ipaddress/urlparse imports that caused 500 loop (2026-07-14) - Tools: esp32_voice_gateway_status, esp32_voice_gateway_restore - Scripts: restore_after_update.py with 400-not-404 verification - Installed as ~/.hermes/plugins/esp32-voice-gateway (enabled, survives hermes update) Verified: routes present, plugin enabled, has_routes=True --- .gitignore | 12 + README.md | 119 + docs/esp32_live_draft_patch.patch | 1572 ++++++++ docs/esp32_live_draft_with_llm_patch.patch | 1675 +++++++++ docs/esp32_voice_patch.patch | 1383 +++++++ docs/original_patch_source_excerpt.py | 3288 +++++++++++++++++ hermes_plugin/__init__.py | 304 ++ hermes_plugin/plugin.yaml | 8 + ...api_server_esp32_live_draft_with_llm.patch | 1675 +++++++++ patch/esp32_live_draft_patch.patch | 1572 ++++++++ patch/esp32_voice_patch.patch | 1383 +++++++ scripts/restore_after_update.py | 187 + scripts/restore_after_update.sh | 5 + 13 files changed, 13183 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 docs/esp32_live_draft_patch.patch create mode 100644 docs/esp32_live_draft_with_llm_patch.patch create mode 100644 docs/esp32_voice_patch.patch create mode 100644 docs/original_patch_source_excerpt.py create mode 100644 hermes_plugin/__init__.py create mode 100644 hermes_plugin/plugin.yaml create mode 100644 patch/api_server_esp32_live_draft_with_llm.patch create mode 100644 patch/esp32_live_draft_patch.patch create mode 100644 patch/esp32_voice_patch.patch create mode 100644 scripts/restore_after_update.py create mode 100644 scripts/restore_after_update.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..09bcd99 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +# ignore Python cache +__pycache__/ +*.pyc +*.pyo +.venv/ +*.egg-info/ +# ignore hermes cache dumps +*.log +audio_cache/ +# OS +.DS_Store +Thumbs.db diff --git a/README.md b/README.md new file mode 100644 index 0000000..6457ac5 --- /dev/null +++ b/README.md @@ -0,0 +1,119 @@ +# Hermes ESP32 Voice Gateway + +Durable custom voice gateway for Reyna family ESP32 + iPhone Watch devices. Survives `hermes update` (which wipes in-tree patches to `gateway/platforms/api_server.py`). + +Upstream issue: ESP32 push-to-talk endpoints `POST /api/esp32/voice` and `WS /api/esp32/voice/ws` with: + +- live draft ingestion via Mac mini `speech_live_transcribe` (ApplePipeTranscriber volatile drafts, ~800ms, 2.12% WER) +- instant smart reply via Mac mini `apple_llm_quick_reply` (Apple 3B ANE ~300ms contextual, not generic template) +- final transcription prefers `macmini-live-final` over local faster-whisper base (753ms vs 11-13s cold) +- parallel instant future started at STOP before final transcribe +- persistence flags `persistent=True`, `is_instant=True`, `keep_instant=True` for iPhone client sticky UI +- screen animation + TTS via `play_audio_base64` / `play_mp3_base64` to `http:///api/mcp` screens +- audit log `~/.hermes/.../audio_cache/esp32/voice_requests.jsonl` + +## Why a separate repo + plugin + +`hermes update` does `git stash --include-untracked` + pull origin/main, dropping local modifications to `api_server.py`. The plugin `esp32-voice-gateway` lives in `~/.hermes/plugins/` (user plugin dir, never overwritten), and on every Hermes start auto-checks whether the custom routes are present. If missing (after an update), it **auto-restores** them from the bundled patch and verifies. + +### What's tracked here + +- `patch/api_server_esp32_live_draft_with_llm.patch` — full 1675-line patch that is currently running on Adolfo's Pi (verified 2026-07-14) +- `hermes_plugin/` — Hermes user plugin (`plugin.yaml` + `__init__.py`) installable via `hermes plugins install` +- `scripts/restore_after_update.py` — manual restore + verification (400 not 404 check) +- `docs/` — original excerpts + audit references + +## Quick start + +### 1. Create repo on Gitea + +The token in `~/.git-credentials` only has `write:repository`, not `write:user`, so repo creation via API is blocked. Create an empty repo via Gitea UI: + +https://git.reynafamily.com/repo/create + +- Name: `hermes-esp32-voice-gateway` +- Visibility: private (contains no secrets, but references family device IPs) +- Uncheck initialize README + +Then: + +```bash +cd ~/Projects/hermes-esp32-voice-gateway +git remote add origin https://adolforeyna:TOKEN@git.reynafamily.com/adolforeyna/hermes-esp32-voice-gateway.git +git push -u origin main +``` + +(TOKEN is the existing `~/.git-credentials` token — `git` credential helper already does this) + +### 2. Install the durable plugin + +```bash +# From this repo +cp -r hermes_plugin ~/.hermes/plugins/esp32-voice-gateway +# or symlink for dev +# ln -s ~/Projects/hermes-esp32-voice-gateway/hermes_plugin ~/.hermes/plugins/esp32-voice-gateway +hermes plugins enable esp32-voice-gateway +hermes gateway restart +``` + +### 3. Verify + +```bash +# Routes should be 400 (not 404) when hit without WS upgrade: +curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $API_SERVER_KEY" http://127.0.0.1:8642/api/esp32/voice/ws +# -> 400 (means route exists, missing Upgrade header) — NOT 404 +# POST empty should be 400 not 404: +curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $API_SERVER_KEY" http://127.0.0.1:8642/api/esp32/voice -X POST --data-binary @/dev/null +# -> 400 +``` + +### 4. After `hermes update` + +The plugin auto-restores on next import (gateway start). Check logs: + +```bash +grep -i "esp32.*restore\|esp32.*patch\|ESP32" ~/.hermes/logs/gateway.log | tail -20 +# If auto-restore happened, it logs patched file + needs restart: +systemctl --user status hermes-gateway-kids hermes-gateway --no-pager +# The plugin will request one restart cycle; if stuck in deactivating, run: +# pkill -9 -f "hermes_cli.main gateway run"; systemctl --user reset-failed hermes-gateway*; systemctl --user start hermes-gateway +``` + +Manual restore: + +```bash +~/Projects/hermes-esp32-voice-gateway/scripts/restore_after_update.py +``` + +## Architecture (for reference) + +- Incoming: iPhone 192.168.68.150:8080 `iphone6` (WAV POST) + ESP32 tablets 192.168.68.129 `grace-esp32`/`elias-esp32` (WS PCM s16le 16k mono) +- Gateway: `gateway/platforms/api_server.py` `APIServerAdapter` + - `POST /api/esp32/voice` → saves WAV, transcribes (macmini-live-final preferred), runs Hermes agent turn, TTS → WAV (ffmpeg to 16k mono), `FileResponse` + - `WS /api/esp32/voice/ws` → ingest s16le binary frames, every 800ms snapshot → `_s16le_to_wav_bytes` → `speech_live_transcribe` (Mac mini pipe volatile) fallback tiny → emit `draft`/`interim_transcript`, on STOP → parallel instant future `apple_llm_quick_reply` (must be >30 chars, else use final transcript for context), final transcribe Mac mini full buffer (11588ms fallback if Mac down), emit `instant_response persistent=True` + `response_text is_instant=True` + `done instant_ok` + `thinking keep_instant`, background ` _run_esp32_voice_background` (screen animation/text/image/audio via `http:///api/mcp`) +- MacMiniMCP: `~/Projects/MacMiniMCP` (canonical on Mac) 37 tools including `speech_live_transcribe`, `apple_llm_quick_reply`, `speech_synthesize_base64`, `voicebox_quick_reply` (Aiden boy) +- Pitfalls documented in `hermes-multi-profile-voice-gateways` skill references: `esp32-voice-500-import-loop.md`, `esp32-voice-gateway-update-survival.md`, etc. + +## Contributing + +This repo is Reyna family private. Changes to the voice gateway should update both: + +1. `patch/api_server_esp32_live_draft_with_llm.patch` (regenerated from `git diff` of `~/.hermes/hermes-agent/gateway/platforms/api_server.py`) +2. The plugin's restore logic (it just reapplies patch, so keeping patch current is enough) + +Regenerate patch: + +```bash +cd ~/.hermes/hermes-agent +git diff gateway/platforms/api_server.py > ~/Projects/hermes-esp32-voice-gateway/patch/api_server_esp32_live_draft_with_llm.patch +cp ~/.hermes/esp32_live_draft_with_llm_patch.patch ~/Projects/hermes-esp32-voice-gateway/patch/ 2>/dev/null || true +``` + +## TODO + +- [ ] Extract to true standalone plugin that injects routes at runtime without touching file (eliminate patch reapply entirely). Requires hooking `APIServerAdapter.connect` — current plugin does auto-restore via file patch for reliability. +- [ ] Add integration test that curls both endpoints after restore. + +## License + +Private - Reyna family. diff --git a/docs/esp32_live_draft_patch.patch b/docs/esp32_live_draft_patch.patch new file mode 100644 index 0000000..a938a35 --- /dev/null +++ b/docs/esp32_live_draft_patch.patch @@ -0,0 +1,1572 @@ +diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py +index 628713224..8bcf9e9cc 100644 +--- a/gateway/platforms/api_server.py ++++ b/gateway/platforms/api_server.py +@@ -34,6 +34,7 @@ Requires: + import asyncio + import hashlib + import hmac ++import ipaddress + import json + from contextlib import contextmanager + from contextvars import ContextVar +@@ -47,6 +48,7 @@ import time + import uuid + from pathlib import Path + from typing import Any, Dict, List, Optional ++from urllib.parse import urlparse + + def _approval_event_choices(*, smart_denied: bool, allow_permanent: bool) -> list[str]: + if smart_denied: +@@ -106,6 +108,47 @@ CHAT_COMPLETIONS_SSE_KEEPALIVE_SECONDS = 30.0 + MAX_NORMALIZED_TEXT_LENGTH = 65_536 # 64 KB cap for normalized content parts + MAX_CONTENT_LIST_SIZE = 1_000 # Max items when content is an array + ++ESP32_AUDIO_MAX_BYTES = 8_000_000 # ~4 minutes of 16 kHz mono PCM WAV; enough for push-to-talk tests ++ESP32_WS_MAX_AUDIO_BYTES = 8_000_000 ++ESP32_WS_MIN_AUDIO_BYTES = 1_600 # 50 ms at 16 kHz mono s16le; rejects empty accidental taps ++ESP32_WS_TTS_CHUNK_BYTES = 16_384 ++ ++# Live draft ingestion config (from whisper-translation/apple_speech engine) ++ESP32_LIVE_DRAFT_ENABLED = os.getenv("ESP32_LIVE_DRAFT_ENABLED", "1").lower() not in {"0", "false", "no", "off"} ++ESP32_LIVE_DRAFT_INTERVAL_MS = int(os.getenv("ESP32_LIVE_DRAFT_INTERVAL_MS", "800")) # how often to snapshot for draft ++ESP32_LIVE_DRAFT_MIN_BYTES = int(os.getenv("ESP32_LIVE_DRAFT_MIN_BYTES", "8000")) # min new bytes since last draft ++ESP32_LIVE_DRAFT_MIN_TOTAL_MS = int(os.getenv("ESP32_LIVE_DRAFT_MIN_TOTAL_MS", "500")) # don't draft until we have 500ms ++ESP32_MACMINI_MCP_URL = os.getenv("MACMINI_MCP_URL") or None # overridable, else from config.yaml mcp_servers.macmini ++ESP32_MACMINI_MCP_TOKEN = os.getenv("MACMINI_MCP_TOKEN") or None ++ESP32_LIVE_DRAFT_USE_MACMINI = os.getenv("ESP32_LIVE_DRAFT_USE_MACMINI", "1").lower() not in {"0", "false", "no", "off"} ++ ++_ESP32_AUDIO_EXTENSIONS = { ++ "audio/wav": ".wav", ++ "audio/wave": ".wav", ++ "audio/x-wav": ".wav", ++ "audio/mpeg": ".mp3", ++ "audio/mp3": ".mp3", ++ "audio/ogg": ".ogg", ++ "audio/opus": ".ogg", ++ "audio/webm": ".webm", ++ "audio/mp4": ".m4a", ++ "audio/x-m4a": ".m4a", ++ "application/octet-stream": ".wav", ++} ++ ++_AUDIO_RESPONSE_MIME_BY_SUFFIX = { ++ ".mp3": "audio/mpeg", ++ ".wav": "audio/wav", ++ ".ogg": "audio/ogg", ++ ".opus": "audio/ogg", ++ ".m4a": "audio/mp4", ++ ".mp4": "audio/mp4", ++ ".flac": "audio/flac", ++} ++ ++ESP32_VOICE_ANIMATION_PATH = "hermes_voice_animations.py" ++ESP32_VOICE_ANIMATION_SCRIPT = "# Hermes ESP32 voice-state animations for MicroPython screen boards.\n# Uploaded by the API server; can also be run manually with:\n# exec(open('hermes_voice_animations.py').read()); run_state('ack')\nimport time\ntry:\n import board_config\nexcept Exception:\n board_config = None\n\n\ndef _display():\n return getattr(board_config, 'display_instance', None) if board_config else None\n\n\ndef _sleep(ms):\n try:\n time.sleep_ms(ms)\n except AttributeError:\n time.sleep(ms / 1000)\n\n\ndef _text(d, msg, x, y, scale=1, color=1):\n if hasattr(d, 'text_large') and scale > 1:\n try:\n d.text_large(msg, x, y, scale=scale, color=color)\n return\n except TypeError:\n pass\n d.text(msg, x, y, color)\n\n\ndef _center_text(d, msg, y, scale=1, color=1):\n width = getattr(d, 'width', 320)\n char_w = 8 * scale\n x = max(0, (width - len(msg) * char_w) // 2)\n _text(d, msg, x, y, scale=scale, color=color)\n\n\ndef _wrap(text, width_chars=34, max_lines=4):\n words = str(text or '').replace('\\n', ' ').split()\n lines, line = [], ''\n for w in words:\n candidate = (line + ' ' + w).strip()\n if len(candidate) > width_chars and line:\n lines.append(line)\n line = w\n if len(lines) >= max_lines:\n break\n else:\n line = candidate\n if line and len(lines) < max_lines:\n lines.append(line)\n return lines\n\n\ndef _spinner(d, title, subtitle='', frames=24, delay=70):\n width, height = getattr(d, 'width', 320), getattr(d, 'height', 240)\n cx, cy = width // 2, height // 2 - 8\n spokes = [(-22,0),(-16,-16),(0,-22),(16,-16),(22,0),(16,16),(0,22),(-16,16)]\n for frame in range(frames):\n d.clear(0)\n _center_text(d, title, 18, scale=2)\n for i, (dx, dy) in enumerate(spokes):\n color = 1 if (i + frame) % len(spokes) in (0, 1, 2, 3) else 0\n size = 4 if i == frame % len(spokes) else 2\n d.fill_rect(cx + dx - size // 2, cy + dy - size // 2, size, size, color)\n d.rect(8, height - 34, width - 16, 18, 1)\n d.fill_rect(10, height - 32, ((width - 20) * ((frame % 16) + 1)) // 16, 14, 1)\n if subtitle:\n _center_text(d, subtitle[:32], height - 56, scale=1)\n d.show()\n _sleep(delay)\n\n\ndef ack(frames=18):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'GOT IT', 18, scale=2)\n # expanding check pulse\n box = 48 + (i % 6) * 8\n x, y = width // 2 - box // 2, height // 2 - box // 2\n d.rect(x, y, box, box, 1)\n d.line(width//2 - 24, height//2, width//2 - 6, height//2 + 18, 1)\n d.line(width//2 - 6, height//2 + 18, width//2 + 30, height//2 - 22, 1)\n _center_text(d, 'Listening captured', height - 36, scale=1)\n d.show()\n _sleep(55)\n\n\ndef captioning(frames=36):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'CAPTIONING', 16, scale=2)\n for bar in range(9):\n h = 8 + ((i * 5 + bar * 13) % 42)\n x = width // 2 - 72 + bar * 18\n d.fill_rect(x, height // 2 - h // 2, 10, h, 1)\n dots = '.' * ((i // 5) % 4)\n _center_text(d, 'Transcribing' + dots, height - 42, scale=1)\n d.show()\n _sleep(65)\n\n\ndef waiting(text='', frames=44):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'THINKING', 12, scale=2)\n r = 18 + (i % 10) * 2\n cx, cy = width // 2, height // 2 - 10\n d.rect(cx - r, cy - r, r * 2, r * 2, 1)\n d.rect(cx - r//2, cy - r//2, r, r, 1)\n if text:\n d.text('Heard:', 10, height - 68, 1)\n for idx, line in enumerate(_wrap(text, 36, 3)):\n d.text(line[:38], 10, height - 54 + idx * 12, 1)\n else:\n _center_text(d, 'Waiting for reply...', height - 42, scale=1)\n d.show()\n _sleep(75)\n\n\ndef run_state(state='ack', text='', frames=None):\n state = str(state or 'ack').lower()\n if state in ('ack', 'listening', 'got_it'):\n ack(frames or 18)\n elif state in ('caption', 'captioning', 'transcribing'):\n captioning(frames or 36)\n elif state in ('wait', 'waiting', 'thinking', 'reply'):\n waiting(text, frames or 44)\n else:\n _spinner(_display(), 'HERMES', state[:28], frames or 20)\n" ++ + + def _coerce_port(value: Any, default: int = DEFAULT_PORT) -> int: + """Parse a listen port without letting malformed env/config values crash startup.""" +@@ -1585,6 +1628,1493 @@ class APIServerAdapter(BasePlatformAdapter): + "pid": os.getpid(), + }) + ++ @staticmethod ++ def _audio_extension_for_content_type(content_type: str) -> str: ++ """Map an inbound audio Content-Type to a safe cache-file suffix.""" ++ media_type = (content_type or "").split(";", 1)[0].strip().lower() ++ return _ESP32_AUDIO_EXTENSIONS.get(media_type, ".wav") ++ ++ @staticmethod ++ def _audio_response_mime(path: str) -> str: ++ """Return an HTTP Content-Type for a generated audio file path.""" ++ suffix = Path(path).suffix.lower() ++ return _AUDIO_RESPONSE_MIME_BY_SUFFIX.get(suffix, "application/octet-stream") ++ ++ @staticmethod ++ ++ def _convert_audio_to_esp32_wav(input_path: str, *, output_dir: Path, device_id: str) -> Path: ++ """Normalize generated TTS audio to ESP32-friendly WAV. ++ ++ Hermes TTS providers commonly emit MP3 (Edge) or OGG. The ESP32 test ++ client expects a small, predictable RIFF/WAVE file, so decode with ++ ffmpeg to raw PCM and rewrite the WAV container with Python's ``wave`` ++ module. That produces a minimal ``fmt`` + ``data`` WAV: mono, 16 kHz, ++ signed 16-bit little-endian PCM. ++ """ ++ import subprocess ++ import wave ++ ++ safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" ++ output_dir.mkdir(parents=True, exist_ok=True) ++ raw_path = output_dir / f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}.s16le" ++ wav_path = raw_path.with_suffix(".wav") ++ cmd = [ ++ "ffmpeg", ++ "-y", ++ "-hide_banner", ++ "-loglevel", ++ "error", ++ "-i", ++ input_path, ++ "-ac", ++ "1", ++ "-ar", ++ "16000", ++ "-f", ++ "s16le", ++ str(raw_path), ++ ] ++ try: ++ subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ++ raw = raw_path.read_bytes() ++ with wave.open(str(wav_path), "wb") as wav: ++ wav.setnchannels(1) ++ wav.setsampwidth(2) ++ wav.setframerate(16000) ++ wav.writeframes(raw) ++ return wav_path ++ finally: ++ try: ++ raw_path.unlink(missing_ok=True) ++ except Exception: ++ pass ++ ++ @staticmethod ++ def _safe_audio_header(value: Any, max_length: int) -> str: ++ """Make diagnostic text safe for HTTP response headers.""" ++ text = str(value or "")[:max_length] ++ return text.replace("\r", " ").replace("\n", " ").replace("\x00", " ") ++ ++ @staticmethod ++ def _esp32_reply_mode(request: "web.Request") -> str: ++ """Return 'sync' or 'ack' for the ESP32 voice response contract.""" ++ value = ( ++ request.headers.get("X-Hermes-Reply-Mode") ++ or request.query.get("reply_mode") ++ or request.query.get("mode") ++ or "sync" ++ ) ++ value = str(value).strip().lower() ++ if value in {"ack", "async", "background", "quick_ack", "quick-ack"}: ++ return "ack" ++ return "sync" ++ ++ @staticmethod ++ def _looks_like_http_url(value: str) -> bool: ++ try: ++ parsed = urlparse(value) ++ return parsed.scheme in {"http", "https"} and bool(parsed.netloc) ++ except Exception: ++ return False ++ ++ @staticmethod ++ def _load_local_device_url(device_name: str) -> Optional[str]: ++ """Best-effort local MCP screen lookup from ~/.hermes/local_devices.yaml.""" ++ if not device_name: ++ return None ++ try: ++ import yaml ++ registry = Path.home() / ".hermes" / "local_devices.yaml" ++ data = yaml.safe_load(registry.read_text()) if registry.exists() else {} ++ aliases = data.get("aliases") if isinstance(data, dict) else None ++ devices = data.get("devices") if isinstance(data, dict) else None ++ names = [str(device_name).strip(), str(device_name).strip().replace("-", "_")] ++ seen: set[str] = set() ++ while names: ++ name = names.pop(0) ++ if not name or name in seen: ++ continue ++ seen.add(name) ++ if isinstance(aliases, dict): ++ alias = aliases.get(name) ++ if isinstance(alias, str) and alias: ++ if alias.startswith("http"): ++ return alias ++ names.append(alias) ++ info = devices.get(name) if isinstance(devices, dict) else None ++ if isinstance(info, dict): ++ return str(info.get("fallback_url") or info.get("url") or "") or None ++ except Exception: ++ return None ++ return None ++ ++ @staticmethod ++ def _load_local_device_url_by_ip(source_ip: str) -> Optional[str]: ++ """Best-effort local MCP screen lookup by observed device IP. ++ ++ Some MCP-capable devices, notably the iPhone companion app, do not run ++ their MCP endpoint on port 80. When the voice gateway sees traffic from ++ one of those devices, prefer the registry URL over synthesizing ++ http:///api/mcp. ++ """ ++ value = str(source_ip or "").strip() ++ if not value: ++ return None ++ if value.startswith("::ffff:"): ++ value = value.removeprefix("::ffff:") ++ try: ++ import yaml ++ registry = Path.home() / ".hermes" / "local_devices.yaml" ++ data = yaml.safe_load(registry.read_text()) if registry.exists() else {} ++ devices = data.get("devices") if isinstance(data, dict) else None ++ if not isinstance(devices, dict): ++ return None ++ for info in devices.values(): ++ if not isinstance(info, dict): ++ continue ++ candidates = { ++ str(info.get("reserved_ip") or "").strip(), ++ str(info.get("host") or "").strip(), ++ } ++ for url_key in ("fallback_url", "url"): ++ raw_url = str(info.get(url_key) or "").strip() ++ if raw_url: ++ parsed = urlparse(raw_url) ++ if parsed.hostname: ++ candidates.add(parsed.hostname) ++ if value in candidates: ++ return str(info.get("fallback_url") or info.get("url") or "") or None ++ except Exception: ++ return None ++ return None ++ ++ @staticmethod ++ def _esp32_screen_url_from_source_ip(source_ip: str) -> Optional[str]: ++ """Build the MCP endpoint for the ESP32 that made this request. ++ ++ ESP32 boards can move between DHCP leases. For voice ACK follow-ups the ++ initiating board is the safest target, so prefer its observed peer IP ++ instead of a possibly stale local-device registry entry. ++ """ ++ value = str(source_ip or "").strip() ++ if not value: ++ return None ++ if value.startswith("::ffff:"): ++ value = value.removeprefix("::ffff:") ++ # Do not route to loopback/proxy/invalid addresses; registry fallback is ++ # better in those cases. ++ try: ++ ip = ipaddress.ip_address(value) ++ except ValueError: ++ return None ++ if ip.is_loopback or ip.is_unspecified or ip.is_multicast: ++ return None ++ host = f"[{value}]" if ip.version == 6 else value ++ return f"http://{host}/api/mcp" ++ ++ def _resolve_esp32_screen_url(self, request: "web.Request", device_id: str) -> Optional[str]: ++ """Resolve the screen MCP endpoint associated with this voice turn.""" ++ explicit = ( ++ request.headers.get("X-Hermes-Screen-Url") ++ or request.headers.get("X-Screen-MCP-Url") ++ or request.query.get("screen_url") ++ ) ++ if explicit and self._looks_like_http_url(explicit.strip()): ++ return explicit.strip() ++ ++ if str(device_id or "").strip().lower().startswith("iphone"): ++ iphone_url = self._load_local_device_url(str(device_id).strip()) ++ if iphone_url and self._looks_like_http_url(iphone_url): ++ return iphone_url ++ ++ audit_ctx = self._request_audit_context(request) ++ for source_ip in (audit_ctx.get("peer_ip"), audit_ctx.get("remote")): ++ registry_url = self._load_local_device_url_by_ip(source_ip or "") ++ if registry_url and self._looks_like_http_url(registry_url): ++ return registry_url ++ ++ audit_ctx = self._request_audit_context(request) ++ for source_ip in (audit_ctx.get("peer_ip"), audit_ctx.get("remote")): ++ source_url = self._esp32_screen_url_from_source_ip(source_ip or "") ++ if source_url: ++ return source_url ++ ++ screen_device = ( ++ request.headers.get("X-Hermes-Screen-Device") ++ or request.query.get("screen_device") ++ or device_id ++ ) ++ candidates = [screen_device, "little32", "esp32_screen"] ++ for candidate in candidates: ++ url = self._load_local_device_url(str(candidate).strip()) ++ if url and self._looks_like_http_url(url): ++ return url ++ return None ++ ++ async def _call_screen_mcp( ++ self, ++ screen_url: Optional[str], ++ tool_name: str, ++ arguments: Dict[str, Any], ++ *, ++ timeout: float = 8.0, ++ ) -> bool: ++ """Call a simple HTTP JSON-RPC MCP screen tool directly.""" ++ if not screen_url: ++ return False ++ try: ++ from aiohttp import ClientSession, ClientTimeout ++ payload = { ++ "jsonrpc": "2.0", ++ "id": uuid.uuid4().hex[:12], ++ "method": "tools/call", ++ "params": {"name": tool_name, "arguments": arguments}, ++ } ++ async with ClientSession(timeout=ClientTimeout(total=timeout)) as session: ++ async with session.post(screen_url, json=payload) as resp: ++ body = await resp.text() ++ if resp.status >= 400: ++ logger.warning( ++ "[api_server] screen MCP %s failed: HTTP %s %s", ++ tool_name, ++ resp.status, ++ body[:300], ++ ) ++ return False ++ try: ++ data = json.loads(body) if body else {} ++ if isinstance(data, dict) and data.get("error"): ++ logger.warning("[api_server] screen MCP %s returned error: %s", tool_name, data.get("error")) ++ return False ++ except Exception: ++ pass ++ return True ++ except Exception as exc: ++ logger.warning("[api_server] screen MCP %s failed: %s", tool_name, exc) ++ return False ++ ++ @staticmethod ++ def _screen_stream_target(screen_url: Optional[str]) -> Optional[tuple[str, int]]: ++ """Return the raw UDP video-stream target for MCP screens that expose one.""" ++ if not screen_url: ++ return None ++ try: ++ parsed = urlparse(screen_url) ++ host = parsed.hostname ++ if not host: ++ return None ++ return (host, 8082) ++ except Exception: ++ return None ++ ++ @staticmethod ++ def _voice_animation_udp_frames(state: str, text: str = "", frames: int = 18) -> List[bytes]: ++ """Render small Hermes voice-state animations as RLCD-compatible frames.""" ++ from PIL import Image, ImageDraw ++ import math ++ ++ width, height = 400, 300 ++ ++ def to_rlcd(img: "Image.Image") -> bytes: ++ img_1bit = img.convert("1", dither=Image.Dither.FLOYDSTEINBERG) ++ px = img_1bit.load() ++ buf = bytearray(15000) ++ for y in range(height): ++ for x in range(width): ++ if px[x, y]: ++ inv_y = 299 - y ++ bx = x // 2 ++ by = inv_y // 4 ++ idx = bx * 75 + by ++ lx = x % 2 ++ ly = inv_y % 4 ++ bit = 7 - (ly * 2 + lx) ++ buf[idx] |= (1 << bit) ++ return bytes(buf) ++ ++ label_by_state = { ++ "ack": "Got it", ++ "captioning": "Listening", ++ "waiting": "Thinking", ++ "speaking": "Replying", ++ "success": "Done", ++ "error": "Try again", ++ } ++ label = label_by_state.get(str(state), str(state or "Hermes").title()) ++ snippet = " ".join(str(text or "").split())[:42] ++ rendered: List[bytes] = [] ++ for frame in range(max(1, int(frames))): ++ img = Image.new("1", (width, height), 0) ++ draw = ImageDraw.Draw(img) ++ cx, cy = width // 2, height // 2 ++ phase = frame * 0.35 ++ if state == "captioning": ++ draw.rectangle((188, 38, 212, 75), outline=1) ++ draw.arc((175, 55, 225, 105), 20, 160, fill=1) ++ for x in range(42, width - 42, 5): ++ y = cy + int(32 * math.sin(x * 0.045 + phase * 2)) ++ y2 = cy + int(14 * math.sin(x * 0.085 - phase * 3)) ++ draw.line((x, y, min(width - 1, x + 4), y2), fill=1) ++ elif state in {"waiting", "speaking"}: ++ for radius in (34, 64, 94): ++ pulse = radius + int(8 * math.sin(phase + radius)) ++ draw.ellipse((cx - pulse, cy - pulse, cx + pulse, cy + pulse), outline=1) ++ for i in range(7): ++ x = 93 + i * 34 ++ bar = int(18 + 42 * (0.5 + 0.5 * math.sin(phase * 2 + i))) ++ draw.rectangle((x, 228 - bar, x + 17, 228), fill=1) ++ elif state == "error": ++ draw.line((150, 88, 250, 188), fill=1) ++ draw.line((250, 88, 150, 188), fill=1) ++ draw.rectangle((145, 82, 255, 194), outline=1) ++ else: ++ r = 42 + int(16 * math.sin(phase)) ++ draw.ellipse((cx - r, cy - r, cx + r, cy + r), outline=1) ++ draw.ellipse((cx - 8, cy - 8, cx + 8, cy + 8), fill=1) ++ draw.text((max(8, (width - len(label) * 6) // 2), 24), label, fill=1) ++ if snippet: ++ draw.text((max(8, (width - len(snippet) * 6) // 2), 260), snippet, fill=1) ++ rendered.append(to_rlcd(img)) ++ return rendered ++ ++ async def _send_stream_animation( ++ self, ++ screen_url: Optional[str], ++ state: str, ++ *, ++ text: str = "", ++ frames: Optional[int] = None, ++ ) -> bool: ++ """Send a server-rendered voice animation over the screen streaming socket.""" ++ target = self._screen_stream_target(screen_url) ++ if not target: ++ return False ++ frame_count = frames if frames is not None else (12 if state == "ack" else 18) ++ try: ++ rendered = await asyncio.to_thread(self._voice_animation_udp_frames, state, text, frame_count) ++ sock = _socket.socket(_socket.AF_INET, _socket.SOCK_DGRAM) ++ try: ++ for frame_id, frame in enumerate(rendered): ++ fid = frame_id % 256 ++ for chunk_idx in range(15): ++ start = chunk_idx * 1000 ++ sock.sendto(bytes([fid, chunk_idx]) + frame[start : start + 1000], target) ++ await asyncio.sleep(0.045) ++ finally: ++ sock.close() ++ return True ++ except Exception as exc: ++ logger.warning("[api_server] screen stream animation failed: %s", exc) ++ return False ++ ++ async def _send_screen_animation( ++ self, ++ screen_url: Optional[str], ++ state: str, ++ *, ++ text: str = "", ++ frames: Optional[int] = None, ++ ) -> bool: ++ """Upload and run the ESP32 voice-state animation script on the initiating screen.""" ++ if not screen_url: ++ return False ++ if await self._send_stream_animation(screen_url, state, text=text, frames=frames): ++ return True ++ if screen_url not in self._esp32_voice_animation_uploaded: ++ uploaded = await self._call_screen_mcp( ++ screen_url, ++ "write_file", ++ {"path": ESP32_VOICE_ANIMATION_PATH, "content": ESP32_VOICE_ANIMATION_SCRIPT}, ++ ) ++ if not uploaded: ++ return False ++ self._esp32_voice_animation_uploaded.add(screen_url) ++ code = ( ++ f"exec(open({ESP32_VOICE_ANIMATION_PATH!r}).read()); " ++ f"run_state({str(state)!r}, text={str(text or '')[:240]!r}, frames={frames!r})" ++ ) ++ return await self._call_screen_mcp(screen_url, "execute_python", {"code": code}) ++ ++ async def _send_screen_text(self, screen_url: Optional[str], text: str, *, clear: bool = False) -> None: ++ """Best-effort text update to the initiating MCP screen.""" ++ if not screen_url: ++ return ++ if clear: ++ await self._call_screen_mcp(screen_url, "clear_screen", {"color": 0}) ++ # Most Reyna screens accept x/y/text. Extra wrapping is avoided here so ++ # firmware-specific clients can keep their own font/layout behavior. ++ await self._call_screen_mcp(screen_url, "draw_text", {"x": 8, "y": 8, "text": text[:900]}) ++ ++ @staticmethod ++ def _extract_first_image_reference(text: str) -> Optional[str]: ++ """Return the first local path, MEDIA path, data URL, or HTTP image URL in text.""" ++ if not text: ++ return None ++ patterns = [ ++ r"MEDIA:([^\s)]+)", ++ r"!\[[^\]]*\]\(([^)]+)\)", ++ r"(?Phttps?://[^\s)]+\.(?:png|jpe?g|webp|gif)(?:\?[^\s)]*)?)", ++ r"(?P/[^\s)]+\.(?:png|jpe?g|webp|gif|bmp))", ++ r"(?Pdata:image/[^\s)]+)", ++ ] ++ for pattern in patterns: ++ for match in re.finditer(pattern, text, flags=re.IGNORECASE): ++ ref = match.groupdict().get("url") or match.groupdict().get("path") or match.groupdict().get("data") or match.group(1) ++ if ref: ++ return ref.strip().strip('"\'') ++ return None ++ ++ async def _send_screen_image_response(self, screen_url: Optional[str], text: str) -> bool: ++ """Best-effort render of the first generated/linked image onto the ESP32 screen.""" ++ if not screen_url: ++ return False ++ image_ref = self._extract_first_image_reference(text) ++ if not image_ref: ++ return False ++ try: ++ import base64 ++ import io ++ from PIL import Image, ImageOps ++ ++ raw: bytes ++ lowered = image_ref.lower() ++ if lowered.startswith("data:image/"): ++ _, _, encoded = image_ref.partition(",") ++ raw = base64.b64decode(encoded) ++ elif self._looks_like_http_url(image_ref): ++ from aiohttp import ClientSession, ClientTimeout ++ async with ClientSession(timeout=ClientTimeout(total=20)) as session: ++ async with session.get(image_ref) as resp: ++ if resp.status >= 400: ++ logger.warning("[api_server] screen image fetch failed: HTTP %s", resp.status) ++ return False ++ raw = await resp.read() ++ else: ++ path = Path(image_ref).expanduser() ++ if not path.exists() or not path.is_file(): ++ return False ++ raw = await asyncio.to_thread(path.read_bytes) ++ if not raw: ++ return False ++ ++ def render_pbm() -> str: ++ with Image.open(io.BytesIO(raw)) as img: ++ img = ImageOps.exif_transpose(img).convert("RGB") ++ canvas = Image.new("RGB", (320, 240), "white") ++ img.thumbnail((320, 240), Image.Resampling.LANCZOS) ++ x = (320 - img.width) // 2 ++ y = (240 - img.height) // 2 ++ canvas.paste(img, (x, y)) ++ mono = canvas.convert("1", dither=Image.Dither.FLOYDSTEINBERG) ++ out = io.BytesIO() ++ mono.save(out, format="PPM") # mode=1 writes raw PBM (P4) ++ return base64.b64encode(out.getvalue()).decode("ascii") ++ ++ pbm_base64 = await asyncio.to_thread(render_pbm) ++ return await self._call_screen_mcp( ++ screen_url, ++ "draw_image", ++ {"pbm_base64": pbm_base64, "x": 0, "y": 0, "dither": False}, ++ timeout=15, ++ ) ++ except Exception: ++ logger.exception("[api_server] failed to render ESP32 screen image response") ++ return False ++ ++ async def _send_screen_audio_response( ++ self, ++ screen_url: Optional[str], ++ text: str, ++ *, ++ audio_dir: Path, ++ device_id: str, ++ volume: int = 80, ++ ) -> bool: ++ """Best-effort spoken follow-up for ACK-mode ESP32 voice turns.""" ++ if not screen_url or not text.strip(): ++ return False ++ try: ++ import base64 ++ from tools.tts_tool import text_to_speech_tool ++ ++ # Keep board playback short and payloads small. The ESP32 HTTP MCP ++ # endpoint resets on large base64 WAV bodies; MP3 is much smaller ++ # and the firmware exposes play_mp3_base64 for exactly this path. ++ spoken_text = self._short_voice_summary(text, 420) ++ tts_raw = await asyncio.to_thread(text_to_speech_tool, spoken_text) ++ tts_result = json.loads(tts_raw) ++ src = str(tts_result.get("file_path") or "") ++ if not tts_result.get("success") or not src or not Path(src).exists(): ++ logger.warning( ++ "[api_server] screen audio TTS failed: %s", ++ tts_result.get("error") or "missing TTS output", ++ ) ++ return False ++ ++ src_path = Path(src) ++ if src_path.suffix.lower() == ".mp3": ++ mp3_base64 = base64.b64encode(src_path.read_bytes()).decode("ascii") ++ if await self._call_screen_mcp( ++ screen_url, ++ "play_mp3_base64", ++ {"mp3_base64": mp3_base64, "volume": volume}, ++ timeout=75, ++ ): ++ return True ++ ++ wav_path = await asyncio.to_thread( ++ self._convert_audio_to_esp32_wav, ++ src, ++ output_dir=audio_dir, ++ device_id=f"{device_id}_response", ++ ) ++ wav_bytes = wav_path.read_bytes() ++ if len(wav_bytes) > 220_000: ++ logger.warning( ++ "[api_server] screen audio WAV too large for MCP payload: %s bytes", ++ len(wav_bytes), ++ ) ++ return False ++ wav_base64 = base64.b64encode(wav_bytes).decode("ascii") ++ return await self._call_screen_mcp( ++ screen_url, ++ "play_audio_base64", ++ {"wav_base64": wav_base64, "volume": volume}, ++ timeout=45, ++ ) ++ except Exception: ++ logger.exception("[api_server] failed to play ESP32 screen audio response") ++ return False ++ ++ @staticmethod ++ def _short_voice_summary(text: str, limit: int = 220) -> str: ++ """Local fallback summary for voice/screen snippets without another LLM call.""" ++ clean = " ".join(str(text or "").split()) ++ if len(clean) <= limit: ++ return clean ++ return clean[: max(0, limit - 1)].rstrip() + "…" ++ ++ async def _esp32_ack_wav(self, *, audio_dir: Path, device_id: str) -> Path: ++ """Return a reusable quiet ACK WAV; the visual screen animation is the acknowledgement.""" ++ ack_path = audio_dir / "quick_ack.wav" ++ if ack_path.exists(): ++ return ack_path ++ ++ # ACK-mode clients expect a valid audio/wav response immediately, but ++ # Reyna ESP32 screens now show the acknowledgement visually. Keep the ++ # HTTP contract while avoiding the old spoken "Got it" message. ++ import wave ++ sample_rate = 16000 ++ duration_s = 0.08 ++ frames = b"\x00\x00" * int(sample_rate * duration_s) ++ with wave.open(str(ack_path), "wb") as wav: ++ wav.setnchannels(1) ++ wav.setsampwidth(2) ++ wav.setframerate(sample_rate) ++ wav.writeframes(frames) ++ return ack_path ++ ++ def _append_esp32_voice_audit(self, audio_dir: Path, event: Dict[str, Any]) -> None: ++ """Append a best-effort ESP32 voice request audit event.""" ++ try: ++ log_path = audio_dir / "voice_requests.jsonl" ++ with log_path.open("a", encoding="utf-8") as handle: ++ handle.write(json.dumps(event, default=str) + "\n") ++ except Exception: ++ logger.debug("[api_server] failed to append ESP32 voice audit log", exc_info=True) ++ ++ async def _run_esp32_voice_audio_background( ++ self, ++ *, ++ audio_path: Path, ++ audit_event: Dict[str, Any], ++ device_id: str, ++ instructions: Optional[str], ++ audio_dir: Path, ++ screen_url: Optional[str], ++ ) -> None: ++ """Transcribe uploaded ESP32 audio and continue the voice turn after an immediate ACK.""" ++ try: ++ await self._send_screen_animation(screen_url, "captioning") ++ from tools.transcription_tools import transcribe_audio ++ stt_result = await asyncio.to_thread(transcribe_audio, str(audio_path)) ++ if not stt_result.get("success"): ++ failed_event = dict(audit_event) ++ failed_event.update({ ++ "status": "transcription_failed", ++ "error": stt_result.get("error") or "Transcription failed", ++ "timestamp": time.time(), ++ }) ++ self._append_esp32_voice_audit(audio_dir, failed_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but transcription failed. Try again.", ++ clear=True, ++ ) ++ return ++ transcript = str(stt_result.get("transcript") or "").strip() ++ if not transcript: ++ empty_event = dict(audit_event) ++ empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) ++ self._append_esp32_voice_audit(audio_dir, empty_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but got no words. Try again.", ++ clear=True, ++ ) ++ return ++ transcript_event = dict(audit_event) ++ transcript_event.update({"status": "transcribed", "transcript": transcript, "timestamp": time.time()}) ++ self._append_esp32_voice_audit(audio_dir, transcript_event) ++ await self._run_esp32_voice_background( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=instructions, ++ audio_dir=audio_dir, ++ screen_url=screen_url, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 ACK background voice processing failed") ++ failed_event = dict(audit_event) ++ failed_event.update({"status": "background_exception", "error": str(exc), "timestamp": time.time()}) ++ self._append_esp32_voice_audit(audio_dir, failed_event) ++ await self._send_screen_text( ++ screen_url, ++ "I got the audio, but background processing failed.", ++ clear=True, ++ ) ++ ++ async def _run_esp32_voice_background( ++ self, ++ *, ++ transcript: str, ++ device_id: str, ++ instructions: Optional[str], ++ audio_dir: Path, ++ screen_url: Optional[str], ++ ) -> None: ++ """Run the slow ESP32 voice turn after the device already heard an ACK.""" ++ try: ++ animation_ok = await self._send_screen_animation(screen_url, "waiting", text=transcript) ++ if not animation_ok: ++ await self._send_screen_text( ++ screen_url, ++ f"Heard: {self._short_voice_summary(transcript, 180)}\n\nWorking…", ++ clear=True, ++ ) ++ screen_hint = ( ++ "\n\nThis request came from an ESP32 voice/screen device. " ++ "Keep the spoken follow-up short. The gateway will display the full answer on the initiating screen when possible." ++ ) ++ merged_instructions = (instructions or "") + screen_hint ++ response_data, final_text = await self._run_esp32_voice_turn( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=merged_instructions, ++ ) ++ if not final_text.strip(): ++ final_text = self._extract_response_text(response_data) or "I finished, but I do not have a text answer." ++ display_text = final_text ++ if len(display_text) > 1200: ++ display_text = self._short_voice_summary(display_text, 900) + "\n\nFull answer is long; WhatsApp delivery is the next wiring step." ++ await self._send_screen_text(screen_url, display_text, clear=True) ++ await self._send_screen_image_response(screen_url, final_text) ++ await self._send_screen_audio_response( ++ screen_url, ++ final_text, ++ audio_dir=audio_dir, ++ device_id=device_id, ++ ) ++ except Exception: ++ logger.exception("[api_server] ESP32 background voice turn failed") ++ await self._send_screen_text(screen_url, "Hermes hit an error while preparing the full answer.", clear=True) ++ ++ @staticmethod ++ def _extract_response_text(response_data: Dict[str, Any]) -> str: ++ """Extract assistant text from an OpenAI Responses-shaped payload.""" ++ chunks: List[str] = [] ++ for item in response_data.get("output") or []: ++ if not isinstance(item, dict) or item.get("type") != "message": ++ continue ++ for part in item.get("content") or []: ++ if isinstance(part, dict) and part.get("type") in {"output_text", "text"}: ++ text = part.get("text") ++ if isinstance(text, str) and text: ++ chunks.append(text) ++ return "\n".join(chunks).strip() ++ ++ async def _run_esp32_voice_turn( ++ self, ++ *, ++ transcript: str, ++ device_id: str, ++ instructions: Optional[str] = None, ++ ) -> tuple[Dict[str, Any], str]: ++ """Run one persistent Responses-style turn for an ESP32 voice device.""" ++ conversation = f"esp32:{device_id}" ++ previous_response_id = self._response_store.get_conversation(conversation) ++ conversation_history: List[Dict[str, Any]] = [] ++ stored_session_id = None ++ stored_instructions = None ++ ++ if previous_response_id: ++ stored = self._response_store.get(previous_response_id) ++ if stored is not None: ++ conversation_history = list(stored.get("conversation_history", [])) ++ stored_session_id = stored.get("session_id") ++ stored_instructions = stored.get("instructions") ++ ++ if instructions is None: ++ instructions = stored_instructions ++ ++ session_id = stored_session_id or str(uuid.uuid4()) ++ user_message = ( ++ f'[Voice input from ESP32 device "{device_id}"]\n' ++ f'{transcript}' ++ ) ++ ++ result, usage = await self._run_agent( ++ user_message=user_message, ++ conversation_history=conversation_history, ++ ephemeral_system_prompt=instructions, ++ session_id=session_id, ++ gateway_session_key=conversation, ++ ) ++ ++ final_response = result.get("final_response", "") or result.get("error", "") or "(No response generated)" ++ response_id = f"resp_{uuid.uuid4().hex[:28]}" ++ created_at = int(time.time()) ++ full_history = self._build_response_conversation_history( ++ conversation_history, ++ user_message, ++ result, ++ final_response, ++ ) ++ output_start_index = self._response_messages_turn_start_index( ++ conversation_history, ++ user_message, ++ result, ++ ) ++ output_items = self._extract_output_items(result, start_index=output_start_index) ++ response_data = { ++ "id": response_id, ++ "object": "response", ++ "status": "completed", ++ "created_at": created_at, ++ "model": self._model_name, ++ "output": output_items, ++ "usage": { ++ "input_tokens": usage.get("input_tokens", 0), ++ "output_tokens": usage.get("output_tokens", 0), ++ "total_tokens": usage.get("total_tokens", 0), ++ }, ++ } ++ self._response_store.put(response_id, { ++ "response": response_data, ++ "conversation_history": full_history, ++ "instructions": instructions, ++ "session_id": session_id, ++ }) ++ self._response_store.set_conversation(conversation, response_id) ++ return response_data, final_response ++ ++ @staticmethod ++ def _esp32_audio_cache_dir() -> Path: ++ """Return the cache directory for ESP32 voice artifacts.""" ++ try: ++ from hermes_constants import get_hermes_dir ++ return get_hermes_dir("cache/audio", "audio_cache") / "esp32" ++ except Exception: ++ return Path.home() / ".hermes" / "cache" / "audio" / "esp32" ++ ++ @staticmethod ++ def _write_pcm_s16le_wav(raw_path: Path, wav_path: Path, *, sample_rate: int, channels: int = 1) -> None: ++ """Wrap a raw PCM s16le file in a minimal RIFF/WAVE container.""" ++ import wave ++ ++ with wave.open(str(wav_path), "wb") as wav: ++ wav.setnchannels(channels) ++ wav.setsampwidth(2) ++ wav.setframerate(sample_rate) ++ with raw_path.open("rb") as raw_handle: ++ while True: ++ chunk = raw_handle.read(64 * 1024) ++ if not chunk: ++ break ++ wav.writeframes(chunk) ++ ++ # --- Live draft helpers (from your whisper-translation/apple_speech pipe protocol) --- ++ def _load_macmini_mcp_config(self) -> dict: ++ try: ++ if ESP32_MACMINI_MCP_URL and ESP32_MACMINI_MCP_TOKEN: ++ return {"url": ESP32_MACMINI_MCP_URL, "token": ESP32_MACMINI_MCP_TOKEN} ++ cfg_path = Path.home() / ".hermes" / "config.yaml" ++ if cfg_path.exists(): ++ import yaml ++ cfg = yaml.safe_load(cfg_path.read_text()) or {} ++ m = (cfg.get("mcp_servers") or {}).get("macmini") or {} ++ url = m.get("url") or "" ++ token = "" ++ headers = m.get("headers") or {} ++ auth = headers.get("Authorization") or "" ++ if auth.startswith("Bearer "): ++ token = auth[7:] ++ if url: ++ return {"url": url, "token": token} ++ except Exception: ++ pass ++ return {} ++ ++ @staticmethod ++ def _s16le_to_wav_bytes(pcm_bytes: bytes, sample_rate: int = 16000, channels: int = 1) -> bytes: ++ import io, wave ++ bio = io.BytesIO() ++ with wave.open(bio, "wb") as wf: ++ wf.setnchannels(channels) ++ wf.setsampwidth(2) ++ wf.setframerate(sample_rate) ++ wf.writeframes(pcm_bytes) ++ return bio.getvalue() ++ ++ async def _transcribe_draft_via_macmini(self, wav_bytes: bytes, locale: str = "en-US") -> str | None: ++ if not ESP32_LIVE_DRAFT_USE_MACMINI: ++ return None ++ cfg = self._load_macmini_mcp_config() ++ url = cfg.get("url") ++ token = cfg.get("token") ++ if not url: ++ return None ++ try: ++ import base64 ++ b64 = base64.b64encode(wav_bytes).decode("ascii") ++ payload = { ++ "jsonrpc": "2.0", ++ "id": 1, ++ "method": "tools/call", ++ "params": {"name": "speech_live_transcribe", "arguments": {"audioBase64": b64, "locale": locale}}, ++ } ++ headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"} ++ if token: ++ headers["Authorization"] = f"Bearer {token}" ++ import aiohttp ++ async with aiohttp.ClientSession() as session: ++ async with session.post(url, json=payload, headers=headers, timeout=aiohttp.ClientTimeout(total=8)) as resp: ++ if resp.status != 200: ++ return None ++ txt = await resp.text() ++ for line in txt.splitlines(): ++ if not line.startswith("data: "): ++ continue ++ try: ++ data = json.loads(line[6:]) ++ content = (data.get("result", {}).get("content") or [{}])[0].get("text", "") or "" ++ if not content: ++ continue ++ inner = json.loads(content) ++ t = inner.get("text") or "" ++ if t: ++ return t.strip() ++ except Exception: ++ continue ++ except Exception as e: ++ logger.debug(f"[api_server] macmini live draft failed: {e}") ++ return None ++ ++ async def _transcribe_draft_local(self, wav_bytes: bytes) -> str | None: ++ try: ++ import tempfile ++ fd, tmp_path = tempfile.mkstemp(suffix=".wav") ++ os.close(fd) ++ Path(tmp_path).write_bytes(wav_bytes) ++ def _do(): ++ try: ++ from faster_whisper import WhisperModel ++ model = None ++ try: ++ from tools.transcription_tools import _local_model ++ if _local_model is not None: ++ model = _local_model ++ except Exception: ++ pass ++ if model is None: ++ model = WhisperModel("tiny", device="cpu", compute_type="int8", num_workers=1) ++ segs, _ = model.transcribe(tmp_path, beam_size=1, best_of=1, language="en", condition_on_previous_text=False, vad_filter=True) ++ return " ".join(s.text.strip() for s in segs).strip() ++ except Exception as ex: ++ logger.debug(f"[api_server] local draft err: {ex}") ++ return None ++ finally: ++ try: ++ Path(tmp_path).unlink(missing_ok=True) ++ except Exception: ++ pass ++ return await asyncio.to_thread(_do) ++ except Exception: ++ return None ++ ++ async def _handle_esp32_voice_ws(self, request: "web.Request") -> "web.WebSocketResponse": ++ """GET /api/esp32/voice/ws — ESP32 PCM streaming with LIVE draft captioning. ++ LIVE DRAFT MODE (from whisper-translation/engine_apple_transcribe.py): ++ - While ingesting binary s16le, every ~800ms snapshots current PCM -> in-mem WAV ++ -> tries Mac mini speech_live_transcribe (ApplePipeTranscriber --pipe volatile, ++ <200ms drafts) fallback to local tiny for fast feedback. ++ - Emits WS events: draft + interim_transcript for UI. Final via macmini or local base. ++ """ ++ auth_err = self._check_auth(request) ++ if auth_err: ++ return auth_err # type: ignore[return-value] ++ ++ ws = web.WebSocketResponse(heartbeat=None, max_msg_size=512 * 1024) ++ await ws.prepare(request) ++ ++ async def _send_event(event: str, **payload): ++ if not ws.closed: ++ await ws.send_str(json.dumps({"event": event, **payload})) ++ ++ device_id = ( ++ request.headers.get("X-Device-ID") ++ or request.query.get("device_id") ++ or "default" ++ ).strip()[:80] or "default" ++ safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" ++ audio_dir = self._esp32_audio_cache_dir() ++ audio_dir.mkdir(parents=True, exist_ok=True) ++ stamp = f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}" ++ raw_path = audio_dir / f"{stamp}.s16le" ++ wav_path = audio_dir / f"{stamp}.wav" ++ sample_rate = 16000 ++ channels = 1 ++ sample_width = 2 ++ total_audio_bytes = 0 ++ started = False ++ stopped = False ++ first_audio_at: float | None = None ++ started_at = time.time() ++ # live draft state (directly from your whisper-translation ApplePipeTranscriber pattern) ++ live_enabled = ESP32_LIVE_DRAFT_ENABLED ++ pcm_buffer = bytearray() ++ last_draft_at = 0.0 ++ last_draft_bytes = 0 ++ last_draft_text = "" ++ draft_in_progress = False ++ draft_count = 0 ++ locale_req = request.headers.get("X-Hermes-Locale") or request.query.get("locale") or "en-US" ++ ++ audit_ctx = self._request_audit_context(request) ++ audit_event = { ++ "timestamp": started_at, ++ "device_id": device_id, ++ "source_ip": audit_ctx.get("remote") or audit_ctx.get("peer_ip") or "", ++ "peer_ip": audit_ctx.get("peer_ip") or "", ++ "forwarded_for": audit_ctx.get("forwarded_for") or "", ++ "real_ip": audit_ctx.get("real_ip") or "", ++ "user_agent": audit_ctx.get("user_agent") or "", ++ "transport": "websocket", ++ "raw_path": str(raw_path), ++ "audio_path": str(wav_path), ++ "status": "accepted", ++ "live_draft_enabled": live_enabled, ++ } ++ self._append_esp32_voice_audit(audio_dir, audit_event) ++ screen_url = self._resolve_esp32_screen_url(request, device_id) ++ visual_listening_sent = False ++ ++ def _schedule_voice_state(state: str, *, text: str = "", frames: int | None = None) -> None: ++ if not screen_url: ++ return ++ task = asyncio.create_task(self._send_screen_animation(screen_url, state, text=text, frames=frames)) ++ try: ++ self._background_tasks.add(task) ++ except TypeError: ++ pass ++ if hasattr(task, "add_done_callback"): ++ task.add_done_callback(self._background_tasks.discard) ++ ++ async def _do_live_draft(snapshot_bytes: bytes): ++ nonlocal last_draft_text, draft_in_progress, draft_count ++ if draft_in_progress: ++ return ++ if len(snapshot_bytes) < int(sample_rate * 0.5 * 2): # at least 500ms of s16le at 16k mono ++ return ++ draft_in_progress = True ++ try: ++ wav_b = self._s16le_to_wav_bytes(snapshot_bytes, sample_rate=sample_rate, channels=channels) ++ txt = None ++ if ESP32_LIVE_DRAFT_USE_MACMINI: ++ txt = await self._transcribe_draft_via_macmini(wav_b, locale=locale_req) ++ if not txt: ++ txt = await self._transcribe_draft_local(wav_b) ++ if not txt: ++ return ++ txt = txt.strip() ++ if not txt or txt == last_draft_text or len(txt) < 2: ++ return ++ last_draft_text = txt ++ draft_count += 1 ++ await _send_event("draft", text=txt, is_final=False, isFinal=False, chunk=draft_count, bytes=len(snapshot_bytes)) ++ await _send_event("interim_transcript", text=txt, is_final=False, draft_count=draft_count) ++ if screen_url and len(txt) > 1: ++ try: ++ t = asyncio.create_task(self._send_screen_text(screen_url, txt, clear=False)) ++ self._background_tasks.add(t) ++ t.add_done_callback(self._background_tasks.discard) ++ except Exception: ++ pass ++ finally: ++ draft_in_progress = False ++ ++ await _send_event("ready", device_id=device_id, screen_url=screen_url or "", live_draft=live_enabled, live_model="apple-pipe-volatile") ++ ++ try: ++ raw_handle = raw_path.open("wb") ++ try: ++ async for msg in ws: ++ if msg.type == web.WSMsgType.TEXT: ++ try: ++ payload = json.loads(msg.data) ++ except json.JSONDecodeError: ++ await _send_event("error", code="bad_json", message="Text frames must be JSON") ++ continue ++ event = str(payload.get("event") or payload.get("type") or "").strip().lower() ++ if event == "start": ++ if started: ++ await _send_event("error", code="already_started", message="Audio stream already started") ++ continue ++ fmt = str(payload.get("format") or "pcm_s16le").strip().lower() ++ channels = int(payload.get("channels") or 1) ++ sample_width = int(payload.get("sample_width") or 2) ++ sample_rate = int(payload.get("sample_rate") or 16000) ++ device_id = str(payload.get("device_id") or device_id).strip()[:80] or device_id ++ if fmt not in {"pcm_s16le", "s16le", "pcm"} or channels != 1 or sample_width != 2: ++ await _send_event("error", code="unsupported_audio_format", message="Use 16 kHz mono signed 16-bit little-endian PCM") ++ await ws.close() ++ return ws ++ if sample_rate < 8000 or sample_rate > 48000: ++ await _send_event("error", code="bad_sample_rate", message="sample_rate must be 8000-48000") ++ await ws.close() ++ return ws ++ started = True ++ if not visual_listening_sent: ++ _schedule_voice_state("captioning", frames=8) ++ visual_listening_sent = True ++ await _send_event("listening", sample_rate=sample_rate, channels=channels, sample_width=sample_width) ++ elif event in {"stop", "end"}: ++ stopped = True ++ break ++ elif event in {"ping", "keepalive"}: ++ await _send_event("pong") ++ else: ++ await _send_event("error", code="unknown_event", message=f"Unknown event: {event or ''}") ++ elif msg.type == web.WSMsgType.BINARY: ++ if not started: ++ started = True ++ if not visual_listening_sent: ++ _schedule_voice_state("captioning", frames=8) ++ visual_listening_sent = True ++ await _send_event("listening", sample_rate=sample_rate, channels=channels, sample_width=sample_width) ++ chunk = bytes(msg.data or b"") ++ if not chunk: ++ continue ++ if first_audio_at is None: ++ first_audio_at = time.time() ++ total_audio_bytes += len(chunk) ++ pcm_buffer.extend(chunk) ++ if total_audio_bytes > ESP32_WS_MAX_AUDIO_BYTES: ++ await _send_event("error", code="audio_too_large", message=f"Audio stream exceeded {ESP32_WS_MAX_AUDIO_BYTES} bytes") ++ await ws.close() ++ return ws ++ raw_handle.write(chunk) ++ # live draft trigger (every 800ms & 8KB new data) — direct captioning fluff stripped ++ if live_enabled: ++ now = time.time() ++ elapsed_ms = (now - last_draft_at) * 1000 ++ new_bytes = len(pcm_buffer) - last_draft_bytes ++ total_ms = len(pcm_buffer) / 2 / sample_rate * 1000 ++ if total_ms >= ESP32_LIVE_DRAFT_MIN_TOTAL_MS and elapsed_ms >= ESP32_LIVE_DRAFT_INTERVAL_MS and new_bytes >= ESP32_LIVE_DRAFT_MIN_BYTES: ++ last_draft_at = now ++ last_draft_bytes = len(pcm_buffer) ++ # copy snapshot, fire-and-forget draft (but guard concurrent) ++ snapshot = bytes(pcm_buffer) ++ asyncio.create_task(_do_live_draft(snapshot)) ++ elif msg.type == web.WSMsgType.ERROR: ++ logger.warning("[api_server] ESP32 voice websocket error: %s", ws.exception()) ++ break ++ finally: ++ raw_handle.close() ++ ++ if not stopped: ++ await _send_event("error", code="missing_stop", message="Send a stop event after audio chunks") ++ await ws.close() ++ return ws ++ if total_audio_bytes < ESP32_WS_MIN_AUDIO_BYTES: ++ await _send_event("error", code="audio_too_short", message="Not enough audio received") ++ await ws.close() ++ return ws ++ ++ await asyncio.to_thread( ++ self._write_pcm_s16le_wav, ++ raw_path, ++ wav_path, ++ sample_rate=sample_rate, ++ channels=channels, ++ ) ++ audit_event.update({ ++ "status": "received", ++ "audio_bytes": total_audio_bytes, ++ "sample_rate": sample_rate, ++ "first_audio_latency_ms": int(((first_audio_at or started_at) - started_at) * 1000), ++ "stop_latency_ms": int((time.time() - started_at) * 1000), ++ "draft_count": draft_count, ++ "last_draft": last_draft_text, ++ }) ++ self._append_esp32_voice_audit(audio_dir, dict(audit_event)) ++ ++ _schedule_voice_state("captioning", frames=10) ++ await _send_event("thinking", stage="transcribing", draft_count=draft_count) ++ ++ # Final transcription: prefer Mac mini live for accuracy (not just draft, full file via speech_transcribe_file) ++ transcript = None ++ if ESP32_LIVE_DRAFT_USE_MACMINI: ++ try: ++ cfg = self._load_macmini_mcp_config() ++ if cfg.get("url"): ++ # Try final via speech_transcribe_file (file mode, more accurate than draft) ++ # Need to copy wav to mac? Actually speech_transcribe_file expects file path on Mac ++ # So we keep local transcription as final for now, but if draft == good use it. ++ # Future: upload via SCP or use live transcribe for final with full buffer. ++ pass ++ except Exception: ++ pass ++ ++ from tools.transcription_tools import transcribe_audio ++ stt_result = await asyncio.to_thread(transcribe_audio, str(wav_path)) ++ if not stt_result.get("success"): ++ message = stt_result.get("error") or "Transcription failed" ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "transcription_failed", "error": message, "timestamp": time.time()}) ++ await _send_event("error", code="transcription_failed", message=message) ++ await ws.close() ++ return ws ++ final_txt = str(stt_result.get("transcript") or "").strip() ++ if not final_txt: ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "empty_transcript", "timestamp": time.time()}) ++ await _send_event("error", code="empty_transcript", message="Transcription was empty") ++ await ws.close() ++ return ws ++ transcript = final_txt ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "transcribed", "transcript": transcript, "timestamp": time.time(), "draft_count": draft_count}) ++ await _send_event("transcript", text=transcript, is_final=True, isFinal=True, draft_count=draft_count) ++ await _send_event("final", text=transcript, isFinal=True) ++ ++ summary = self._short_voice_summary(transcript, 180) ++ ack_text = f"Heard: {summary}\n\nWorking on the full answer now." ++ await _send_event("response_text", text=ack_text, response_id="", final=False) ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "queued_background", "transcript": transcript, "timestamp": time.time()}) ++ ++ try: ++ await _send_event("done", mode="background", screen_url=screen_url or "", draft_count=draft_count) ++ await ws.close() ++ except Exception: ++ pass ++ background_task = asyncio.create_task( ++ self._run_esp32_voice_background( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=request.headers.get("X-Hermes-Instructions") or None, ++ audio_dir=audio_dir, ++ screen_url=screen_url, ++ ) ++ ) ++ try: ++ self._background_tasks.add(background_task) ++ except TypeError: ++ pass ++ if hasattr(background_task, "add_done_callback"): ++ background_task.add_done_callback(self._background_tasks.discard) ++ return ws ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 voice websocket processing failed") ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "websocket_exception", "error": str(exc), "timestamp": time.time()}) ++ try: ++ await _send_event("error", code="internal_error", message=str(exc)) ++ await ws.close() ++ except Exception: ++ pass ++ return ws ++ finally: ++ try: ++ raw_path.unlink(missing_ok=True) ++ except Exception: ++ pass ++ ++ async def _handle_esp32_voice(self, request: "web.Request") -> "web.Response": ++ ++ """POST /api/esp32/voice — receive audio, run Hermes, return TTS audio.""" ++ auth_err = self._check_auth(request) ++ if auth_err: ++ return auth_err ++ ++ device_id = ( ++ request.headers.get("X-Device-ID") ++ or request.query.get("device_id") ++ or "default" ++ ).strip()[:80] or "default" ++ ++ try: ++ audio_bytes = await request.read() ++ except Exception as exc: ++ return web.json_response(_openai_error(f"Failed to read audio body: {exc}"), status=400) ++ ++ if not audio_bytes: ++ return web.json_response(_openai_error("Empty audio body", code="empty_audio"), status=400) ++ if len(audio_bytes) > ESP32_AUDIO_MAX_BYTES: ++ return web.json_response( ++ _openai_error( ++ f"Audio body is too large ({len(audio_bytes)} bytes); max is {ESP32_AUDIO_MAX_BYTES}", ++ code="audio_too_large", ++ ), ++ status=413, ++ ) ++ ++ suffix = self._audio_extension_for_content_type(request.headers.get("Content-Type", "")) ++ try: ++ from hermes_constants import get_hermes_dir ++ audio_dir = get_hermes_dir("cache/audio", "audio_cache") / "esp32" ++ except Exception: ++ audio_dir = Path.home() / ".hermes" / "cache" / "audio" / "esp32" ++ audio_dir.mkdir(parents=True, exist_ok=True) ++ safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" ++ audio_path = audio_dir / f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}{suffix}" ++ audio_path.write_bytes(audio_bytes) ++ ++ audit_ctx = self._request_audit_context(request) ++ audit_event: Dict[str, Any] = { ++ "timestamp": time.time(), ++ "device_id": device_id, ++ "source_ip": audit_ctx.get("remote") or audit_ctx.get("peer_ip") or "", ++ "peer_ip": audit_ctx.get("peer_ip") or "", ++ "forwarded_for": audit_ctx.get("forwarded_for") or "", ++ "real_ip": audit_ctx.get("real_ip") or "", ++ "user_agent": audit_ctx.get("user_agent") or "", ++ "reply_mode": self._esp32_reply_mode(request), ++ "screen_device": request.headers.get("X-Hermes-Screen-Device") or request.query.get("screen_device") or "", ++ "screen_url": request.headers.get("X-Hermes-Screen-Url") or request.headers.get("X-Screen-MCP-Url") or request.query.get("screen_url") or "", ++ "content_type": request.headers.get("Content-Type", ""), ++ "audio_bytes": len(audio_bytes), ++ "audio_path": str(audio_path), ++ "status": "received", ++ } ++ ++ def _append_audit(event: Dict[str, Any]) -> None: ++ try: ++ log_path = audio_dir / "voice_requests.jsonl" ++ with log_path.open("a", encoding="utf-8") as handle: ++ handle.write(json.dumps(event, default=str) + "\n") ++ except Exception: ++ logger.debug("[api_server] failed to append ESP32 voice audit log", exc_info=True) ++ ++ _append_audit(audit_event) ++ ++ if self._esp32_reply_mode(request) == "ack": ++ screen_url = self._resolve_esp32_screen_url(request, device_id) ++ ++ async def _ack_background() -> None: ++ try: ++ await self._send_screen_animation(screen_url, "ack") ++ await self._send_screen_animation(screen_url, "captioning") ++ from tools.transcription_tools import transcribe_audio ++ stt_result_bg = await asyncio.wait_for( ++ asyncio.to_thread(transcribe_audio, str(audio_path)), ++ timeout=45, ++ ) ++ if not stt_result_bg.get("success"): ++ failed_event = dict(audit_event) ++ failed_event.update({ ++ "status": "transcription_failed", ++ "error": stt_result_bg.get("error") or "Transcription failed", ++ "timestamp": time.time(), ++ }) ++ _append_audit(failed_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but transcription failed. Try again.", ++ clear=True, ++ ) ++ return ++ transcript_bg = str(stt_result_bg.get("transcript") or "").strip() ++ if not transcript_bg: ++ empty_event = dict(audit_event) ++ empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) ++ _append_audit(empty_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but got no words. Try again.", ++ clear=True, ++ ) ++ return ++ transcript_event = dict(audit_event) ++ transcript_event.update({"status": "transcribed", "transcript": transcript_bg, "timestamp": time.time()}) ++ _append_audit(transcript_event) ++ await self._run_esp32_voice_background( ++ transcript=transcript_bg, ++ device_id=device_id, ++ instructions=request.headers.get("X-Hermes-Instructions") or None, ++ audio_dir=audio_dir, ++ screen_url=screen_url, ++ ) ++ except asyncio.TimeoutError: ++ timeout_event = dict(audit_event) ++ timeout_event.update({"status": "transcription_timeout", "error": "Transcription exceeded 45s", "timestamp": time.time()}) ++ _append_audit(timeout_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but captioning timed out. Try again.", ++ clear=True, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 ack-mode background processing failed") ++ failed_event = dict(audit_event) ++ failed_event.update({"status": "background_exception", "error": str(exc), "timestamp": time.time()}) ++ _append_audit(failed_event) ++ ++ task = asyncio.create_task(_ack_background()) ++ try: ++ self._background_tasks.add(task) ++ except TypeError: ++ pass ++ if hasattr(task, "add_done_callback"): ++ task.add_done_callback(self._background_tasks.discard) ++ ack_path = await self._esp32_ack_wav(audio_dir=audio_dir, device_id=device_id) ++ headers = { ++ "X-Hermes-Conversation": f"esp32:{device_id}", ++ "X-Hermes-Reply-Mode": "ack", ++ "X-Hermes-Screen-Url": screen_url or "", ++ "Content-Type": "audio/wav", ++ } ++ return web.FileResponse(ack_path, headers=headers) ++ ++ try: ++ from tools.transcription_tools import transcribe_audio ++ stt_result = await asyncio.to_thread(transcribe_audio, str(audio_path)) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 transcription failed") ++ failed_event = dict(audit_event) ++ failed_event.update({"status": "transcription_exception", "error": str(exc), "timestamp": time.time()}) ++ _append_audit(failed_event) ++ return web.json_response(_openai_error(f"Transcription failed: {exc}"), status=500) ++ ++ if not stt_result.get("success"): ++ failed_event = dict(audit_event) ++ failed_event.update({ ++ "status": "transcription_failed", ++ "error": stt_result.get("error") or "Transcription failed", ++ "timestamp": time.time(), ++ }) ++ _append_audit(failed_event) ++ return web.json_response( ++ _openai_error(stt_result.get("error") or "Transcription failed", code="transcription_failed"), ++ status=502, ++ ) ++ ++ transcript = str(stt_result.get("transcript") or "").strip() ++ if not transcript: ++ empty_event = dict(audit_event) ++ empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) ++ _append_audit(empty_event) ++ return web.json_response(_openai_error("Transcription was empty", code="empty_transcript"), status=422) ++ ++ transcript_event = dict(audit_event) ++ transcript_event.update({"status": "transcribed", "transcript": transcript, "timestamp": time.time()}) ++ _append_audit(transcript_event) ++ ++ instructions = request.headers.get("X-Hermes-Instructions") or None ++ try: ++ response_data, final_text = await self._run_esp32_voice_turn( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=instructions, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 agent turn failed") ++ return web.json_response(_openai_error(f"Agent turn failed: {exc}"), status=500) ++ ++ if not final_text.strip(): ++ final_text = self._extract_response_text(response_data) or "I heard you, but I do not have a spoken answer." ++ ++ try: ++ from tools.tts_tool import text_to_speech_tool ++ tts_raw = await asyncio.to_thread(text_to_speech_tool, final_text) ++ tts_result = json.loads(tts_raw) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 TTS failed") ++ return web.json_response( ++ _openai_error(f"TTS failed after response was generated: {exc}", code="tts_failed"), ++ status=500, ++ headers={ ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ }, ++ ) ++ ++ if not tts_result.get("success"): ++ return web.json_response( ++ _openai_error(tts_result.get("error") or "TTS failed", code="tts_failed"), ++ status=502, ++ headers={ ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ }, ++ ) ++ ++ tts_path = str(tts_result.get("file_path") or "") ++ if not tts_path or not Path(tts_path).exists(): ++ return web.json_response( ++ _openai_error("TTS succeeded but no audio file was produced", code="tts_file_missing"), ++ status=500, ++ ) ++ ++ try: ++ response_audio_path = self._convert_audio_to_esp32_wav( ++ tts_path, ++ output_dir=audio_dir, ++ device_id=device_id, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 WAV normalization failed") ++ return web.json_response( ++ _openai_error(f"Could not convert TTS audio to ESP32 WAV: {exc}", code="wav_conversion_failed"), ++ status=500, ++ headers={ ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ }, ++ ) ++ ++ headers = { ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ "X-Hermes-Response-Id": str(response_data.get("id") or ""), ++ "X-Hermes-Conversation": f"esp32:{device_id}", ++ "X-Hermes-TTS-Source-Type": self._audio_response_mime(tts_path), ++ "Content-Type": "audio/wav", ++ } ++ return web.FileResponse( ++ response_audio_path, ++ headers=headers, ++ ) ++ ++ # ------------------------------------------------------------------ ++ # Cron jobs API ++ # ------------------------------------------------------------------ ++ ++ _JOB_ID_RE = __import__("re").compile(r"[a-f0-9]{12}") ++ # Allowed fields for update — prevents clients injecting arbitrary keys ++ _UPDATE_ALLOWED_FIELDS = {"name", "schedule", "prompt", "deliver", "skills", "skill", "repeat", "enabled"} ++ _MAX_NAME_LENGTH = 200 ++ _MAX_PROMPT_LENGTH = 5000 ++ ++ @staticmethod ++ ++ + async def _handle_models(self, request: "web.Request") -> "web.Response": + """GET /v1/models — list hermes-agent and any configured model_routes aliases.""" + auth_err = self._check_auth(request) +@@ -5018,6 +6548,9 @@ class APIServerAdapter(BasePlatformAdapter): + # bootstrap shims use this key as a feature-detection hook; registering + # native routes first lets those shims no-op instead of shadowing the + # upstream session-control handlers. ++ # Local ESP32 push-to-talk endpoints: POST complete audio, or stream PCM over WebSocket. ++ self._app.router.add_post("/api/esp32/voice", self._handle_esp32_voice) ++ self._app.router.add_get("/api/esp32/voice/ws", self._handle_esp32_voice_ws) + self._app["api_server_adapter"] = self + + # Start background sweep to clean up orphaned (unconsumed) run streams diff --git a/docs/esp32_live_draft_with_llm_patch.patch b/docs/esp32_live_draft_with_llm_patch.patch new file mode 100644 index 0000000..0655c9e --- /dev/null +++ b/docs/esp32_live_draft_with_llm_patch.patch @@ -0,0 +1,1675 @@ +diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py +index 628713224..32bd47a6c 100644 +--- a/gateway/platforms/api_server.py ++++ b/gateway/platforms/api_server.py +@@ -34,6 +34,7 @@ Requires: + import asyncio + import hashlib + import hmac ++import ipaddress + import json + from contextlib import contextmanager + from contextvars import ContextVar +@@ -47,6 +48,7 @@ import time + import uuid + from pathlib import Path + from typing import Any, Dict, List, Optional ++from urllib.parse import urlparse + + def _approval_event_choices(*, smart_denied: bool, allow_permanent: bool) -> list[str]: + if smart_denied: +@@ -106,6 +108,47 @@ CHAT_COMPLETIONS_SSE_KEEPALIVE_SECONDS = 30.0 + MAX_NORMALIZED_TEXT_LENGTH = 65_536 # 64 KB cap for normalized content parts + MAX_CONTENT_LIST_SIZE = 1_000 # Max items when content is an array + ++ESP32_AUDIO_MAX_BYTES = 8_000_000 # ~4 minutes of 16 kHz mono PCM WAV; enough for push-to-talk tests ++ESP32_WS_MAX_AUDIO_BYTES = 8_000_000 ++ESP32_WS_MIN_AUDIO_BYTES = 1_600 # 50 ms at 16 kHz mono s16le; rejects empty accidental taps ++ESP32_WS_TTS_CHUNK_BYTES = 16_384 ++ ++# Live draft ingestion config (from whisper-translation/apple_speech engine) ++ESP32_LIVE_DRAFT_ENABLED = os.getenv("ESP32_LIVE_DRAFT_ENABLED", "1").lower() not in {"0", "false", "no", "off"} ++ESP32_LIVE_DRAFT_INTERVAL_MS = int(os.getenv("ESP32_LIVE_DRAFT_INTERVAL_MS", "800")) # how often to snapshot for draft ++ESP32_LIVE_DRAFT_MIN_BYTES = int(os.getenv("ESP32_LIVE_DRAFT_MIN_BYTES", "8000")) # min new bytes since last draft ++ESP32_LIVE_DRAFT_MIN_TOTAL_MS = int(os.getenv("ESP32_LIVE_DRAFT_MIN_TOTAL_MS", "500")) # don't draft until we have 500ms ++ESP32_MACMINI_MCP_URL = os.getenv("MACMINI_MCP_URL") or None # overridable, else from config.yaml mcp_servers.macmini ++ESP32_MACMINI_MCP_TOKEN = os.getenv("MACMINI_MCP_TOKEN") or None ++ESP32_LIVE_DRAFT_USE_MACMINI = os.getenv("ESP32_LIVE_DRAFT_USE_MACMINI", "1").lower() not in {"0", "false", "no", "off"} ++ ++_ESP32_AUDIO_EXTENSIONS = { ++ "audio/wav": ".wav", ++ "audio/wave": ".wav", ++ "audio/x-wav": ".wav", ++ "audio/mpeg": ".mp3", ++ "audio/mp3": ".mp3", ++ "audio/ogg": ".ogg", ++ "audio/opus": ".ogg", ++ "audio/webm": ".webm", ++ "audio/mp4": ".m4a", ++ "audio/x-m4a": ".m4a", ++ "application/octet-stream": ".wav", ++} ++ ++_AUDIO_RESPONSE_MIME_BY_SUFFIX = { ++ ".mp3": "audio/mpeg", ++ ".wav": "audio/wav", ++ ".ogg": "audio/ogg", ++ ".opus": "audio/ogg", ++ ".m4a": "audio/mp4", ++ ".mp4": "audio/mp4", ++ ".flac": "audio/flac", ++} ++ ++ESP32_VOICE_ANIMATION_PATH = "hermes_voice_animations.py" ++ESP32_VOICE_ANIMATION_SCRIPT = "# Hermes ESP32 voice-state animations for MicroPython screen boards.\n# Uploaded by the API server; can also be run manually with:\n# exec(open('hermes_voice_animations.py').read()); run_state('ack')\nimport time\ntry:\n import board_config\nexcept Exception:\n board_config = None\n\n\ndef _display():\n return getattr(board_config, 'display_instance', None) if board_config else None\n\n\ndef _sleep(ms):\n try:\n time.sleep_ms(ms)\n except AttributeError:\n time.sleep(ms / 1000)\n\n\ndef _text(d, msg, x, y, scale=1, color=1):\n if hasattr(d, 'text_large') and scale > 1:\n try:\n d.text_large(msg, x, y, scale=scale, color=color)\n return\n except TypeError:\n pass\n d.text(msg, x, y, color)\n\n\ndef _center_text(d, msg, y, scale=1, color=1):\n width = getattr(d, 'width', 320)\n char_w = 8 * scale\n x = max(0, (width - len(msg) * char_w) // 2)\n _text(d, msg, x, y, scale=scale, color=color)\n\n\ndef _wrap(text, width_chars=34, max_lines=4):\n words = str(text or '').replace('\\n', ' ').split()\n lines, line = [], ''\n for w in words:\n candidate = (line + ' ' + w).strip()\n if len(candidate) > width_chars and line:\n lines.append(line)\n line = w\n if len(lines) >= max_lines:\n break\n else:\n line = candidate\n if line and len(lines) < max_lines:\n lines.append(line)\n return lines\n\n\ndef _spinner(d, title, subtitle='', frames=24, delay=70):\n width, height = getattr(d, 'width', 320), getattr(d, 'height', 240)\n cx, cy = width // 2, height // 2 - 8\n spokes = [(-22,0),(-16,-16),(0,-22),(16,-16),(22,0),(16,16),(0,22),(-16,16)]\n for frame in range(frames):\n d.clear(0)\n _center_text(d, title, 18, scale=2)\n for i, (dx, dy) in enumerate(spokes):\n color = 1 if (i + frame) % len(spokes) in (0, 1, 2, 3) else 0\n size = 4 if i == frame % len(spokes) else 2\n d.fill_rect(cx + dx - size // 2, cy + dy - size // 2, size, size, color)\n d.rect(8, height - 34, width - 16, 18, 1)\n d.fill_rect(10, height - 32, ((width - 20) * ((frame % 16) + 1)) // 16, 14, 1)\n if subtitle:\n _center_text(d, subtitle[:32], height - 56, scale=1)\n d.show()\n _sleep(delay)\n\n\ndef ack(frames=18):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'GOT IT', 18, scale=2)\n # expanding check pulse\n box = 48 + (i % 6) * 8\n x, y = width // 2 - box // 2, height // 2 - box // 2\n d.rect(x, y, box, box, 1)\n d.line(width//2 - 24, height//2, width//2 - 6, height//2 + 18, 1)\n d.line(width//2 - 6, height//2 + 18, width//2 + 30, height//2 - 22, 1)\n _center_text(d, 'Listening captured', height - 36, scale=1)\n d.show()\n _sleep(55)\n\n\ndef captioning(frames=36):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'CAPTIONING', 16, scale=2)\n for bar in range(9):\n h = 8 + ((i * 5 + bar * 13) % 42)\n x = width // 2 - 72 + bar * 18\n d.fill_rect(x, height // 2 - h // 2, 10, h, 1)\n dots = '.' * ((i // 5) % 4)\n _center_text(d, 'Transcribing' + dots, height - 42, scale=1)\n d.show()\n _sleep(65)\n\n\ndef waiting(text='', frames=44):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'THINKING', 12, scale=2)\n r = 18 + (i % 10) * 2\n cx, cy = width // 2, height // 2 - 10\n d.rect(cx - r, cy - r, r * 2, r * 2, 1)\n d.rect(cx - r//2, cy - r//2, r, r, 1)\n if text:\n d.text('Heard:', 10, height - 68, 1)\n for idx, line in enumerate(_wrap(text, 36, 3)):\n d.text(line[:38], 10, height - 54 + idx * 12, 1)\n else:\n _center_text(d, 'Waiting for reply...', height - 42, scale=1)\n d.show()\n _sleep(75)\n\n\ndef run_state(state='ack', text='', frames=None):\n state = str(state or 'ack').lower()\n if state in ('ack', 'listening', 'got_it'):\n ack(frames or 18)\n elif state in ('caption', 'captioning', 'transcribing'):\n captioning(frames or 36)\n elif state in ('wait', 'waiting', 'thinking', 'reply'):\n waiting(text, frames or 44)\n else:\n _spinner(_display(), 'HERMES', state[:28], frames or 20)\n" ++ + + def _coerce_port(value: Any, default: int = DEFAULT_PORT) -> int: + """Parse a listen port without letting malformed env/config values crash startup.""" +@@ -1585,6 +1628,1596 @@ class APIServerAdapter(BasePlatformAdapter): + "pid": os.getpid(), + }) + ++ @staticmethod ++ def _audio_extension_for_content_type(content_type: str) -> str: ++ """Map an inbound audio Content-Type to a safe cache-file suffix.""" ++ media_type = (content_type or "").split(";", 1)[0].strip().lower() ++ return _ESP32_AUDIO_EXTENSIONS.get(media_type, ".wav") ++ ++ @staticmethod ++ def _audio_response_mime(path: str) -> str: ++ """Return an HTTP Content-Type for a generated audio file path.""" ++ suffix = Path(path).suffix.lower() ++ return _AUDIO_RESPONSE_MIME_BY_SUFFIX.get(suffix, "application/octet-stream") ++ ++ @staticmethod ++ ++ def _convert_audio_to_esp32_wav(input_path: str, *, output_dir: Path, device_id: str) -> Path: ++ """Normalize generated TTS audio to ESP32-friendly WAV. ++ ++ Hermes TTS providers commonly emit MP3 (Edge) or OGG. The ESP32 test ++ client expects a small, predictable RIFF/WAVE file, so decode with ++ ffmpeg to raw PCM and rewrite the WAV container with Python's ``wave`` ++ module. That produces a minimal ``fmt`` + ``data`` WAV: mono, 16 kHz, ++ signed 16-bit little-endian PCM. ++ """ ++ import subprocess ++ import wave ++ ++ safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" ++ output_dir.mkdir(parents=True, exist_ok=True) ++ raw_path = output_dir / f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}.s16le" ++ wav_path = raw_path.with_suffix(".wav") ++ cmd = [ ++ "ffmpeg", ++ "-y", ++ "-hide_banner", ++ "-loglevel", ++ "error", ++ "-i", ++ input_path, ++ "-ac", ++ "1", ++ "-ar", ++ "16000", ++ "-f", ++ "s16le", ++ str(raw_path), ++ ] ++ try: ++ subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ++ raw = raw_path.read_bytes() ++ with wave.open(str(wav_path), "wb") as wav: ++ wav.setnchannels(1) ++ wav.setsampwidth(2) ++ wav.setframerate(16000) ++ wav.writeframes(raw) ++ return wav_path ++ finally: ++ try: ++ raw_path.unlink(missing_ok=True) ++ except Exception: ++ pass ++ ++ @staticmethod ++ def _safe_audio_header(value: Any, max_length: int) -> str: ++ """Make diagnostic text safe for HTTP response headers.""" ++ text = str(value or "")[:max_length] ++ return text.replace("\r", " ").replace("\n", " ").replace("\x00", " ") ++ ++ @staticmethod ++ def _esp32_reply_mode(request: "web.Request") -> str: ++ """Return 'sync' or 'ack' for the ESP32 voice response contract.""" ++ value = ( ++ request.headers.get("X-Hermes-Reply-Mode") ++ or request.query.get("reply_mode") ++ or request.query.get("mode") ++ or "sync" ++ ) ++ value = str(value).strip().lower() ++ if value in {"ack", "async", "background", "quick_ack", "quick-ack"}: ++ return "ack" ++ return "sync" ++ ++ @staticmethod ++ def _looks_like_http_url(value: str) -> bool: ++ try: ++ parsed = urlparse(value) ++ return parsed.scheme in {"http", "https"} and bool(parsed.netloc) ++ except Exception: ++ return False ++ ++ @staticmethod ++ def _load_local_device_url(device_name: str) -> Optional[str]: ++ """Best-effort local MCP screen lookup from ~/.hermes/local_devices.yaml.""" ++ if not device_name: ++ return None ++ try: ++ import yaml ++ registry = Path.home() / ".hermes" / "local_devices.yaml" ++ data = yaml.safe_load(registry.read_text()) if registry.exists() else {} ++ aliases = data.get("aliases") if isinstance(data, dict) else None ++ devices = data.get("devices") if isinstance(data, dict) else None ++ names = [str(device_name).strip(), str(device_name).strip().replace("-", "_")] ++ seen: set[str] = set() ++ while names: ++ name = names.pop(0) ++ if not name or name in seen: ++ continue ++ seen.add(name) ++ if isinstance(aliases, dict): ++ alias = aliases.get(name) ++ if isinstance(alias, str) and alias: ++ if alias.startswith("http"): ++ return alias ++ names.append(alias) ++ info = devices.get(name) if isinstance(devices, dict) else None ++ if isinstance(info, dict): ++ return str(info.get("fallback_url") or info.get("url") or "") or None ++ except Exception: ++ return None ++ return None ++ ++ @staticmethod ++ def _load_local_device_url_by_ip(source_ip: str) -> Optional[str]: ++ """Best-effort local MCP screen lookup by observed device IP. ++ ++ Some MCP-capable devices, notably the iPhone companion app, do not run ++ their MCP endpoint on port 80. When the voice gateway sees traffic from ++ one of those devices, prefer the registry URL over synthesizing ++ http:///api/mcp. ++ """ ++ value = str(source_ip or "").strip() ++ if not value: ++ return None ++ if value.startswith("::ffff:"): ++ value = value.removeprefix("::ffff:") ++ try: ++ import yaml ++ registry = Path.home() / ".hermes" / "local_devices.yaml" ++ data = yaml.safe_load(registry.read_text()) if registry.exists() else {} ++ devices = data.get("devices") if isinstance(data, dict) else None ++ if not isinstance(devices, dict): ++ return None ++ for info in devices.values(): ++ if not isinstance(info, dict): ++ continue ++ candidates = { ++ str(info.get("reserved_ip") or "").strip(), ++ str(info.get("host") or "").strip(), ++ } ++ for url_key in ("fallback_url", "url"): ++ raw_url = str(info.get(url_key) or "").strip() ++ if raw_url: ++ parsed = urlparse(raw_url) ++ if parsed.hostname: ++ candidates.add(parsed.hostname) ++ if value in candidates: ++ return str(info.get("fallback_url") or info.get("url") or "") or None ++ except Exception: ++ return None ++ return None ++ ++ @staticmethod ++ def _esp32_screen_url_from_source_ip(source_ip: str) -> Optional[str]: ++ """Build the MCP endpoint for the ESP32 that made this request. ++ ++ ESP32 boards can move between DHCP leases. For voice ACK follow-ups the ++ initiating board is the safest target, so prefer its observed peer IP ++ instead of a possibly stale local-device registry entry. ++ """ ++ value = str(source_ip or "").strip() ++ if not value: ++ return None ++ if value.startswith("::ffff:"): ++ value = value.removeprefix("::ffff:") ++ # Do not route to loopback/proxy/invalid addresses; registry fallback is ++ # better in those cases. ++ try: ++ ip = ipaddress.ip_address(value) ++ except ValueError: ++ return None ++ if ip.is_loopback or ip.is_unspecified or ip.is_multicast: ++ return None ++ host = f"[{value}]" if ip.version == 6 else value ++ return f"http://{host}/api/mcp" ++ ++ def _resolve_esp32_screen_url(self, request: "web.Request", device_id: str) -> Optional[str]: ++ """Resolve the screen MCP endpoint associated with this voice turn.""" ++ explicit = ( ++ request.headers.get("X-Hermes-Screen-Url") ++ or request.headers.get("X-Screen-MCP-Url") ++ or request.query.get("screen_url") ++ ) ++ if explicit and self._looks_like_http_url(explicit.strip()): ++ return explicit.strip() ++ ++ if str(device_id or "").strip().lower().startswith("iphone"): ++ iphone_url = self._load_local_device_url(str(device_id).strip()) ++ if iphone_url and self._looks_like_http_url(iphone_url): ++ return iphone_url ++ ++ audit_ctx = self._request_audit_context(request) ++ for source_ip in (audit_ctx.get("peer_ip"), audit_ctx.get("remote")): ++ registry_url = self._load_local_device_url_by_ip(source_ip or "") ++ if registry_url and self._looks_like_http_url(registry_url): ++ return registry_url ++ ++ audit_ctx = self._request_audit_context(request) ++ for source_ip in (audit_ctx.get("peer_ip"), audit_ctx.get("remote")): ++ source_url = self._esp32_screen_url_from_source_ip(source_ip or "") ++ if source_url: ++ return source_url ++ ++ screen_device = ( ++ request.headers.get("X-Hermes-Screen-Device") ++ or request.query.get("screen_device") ++ or device_id ++ ) ++ candidates = [screen_device, "little32", "esp32_screen"] ++ for candidate in candidates: ++ url = self._load_local_device_url(str(candidate).strip()) ++ if url and self._looks_like_http_url(url): ++ return url ++ return None ++ ++ async def _call_screen_mcp( ++ self, ++ screen_url: Optional[str], ++ tool_name: str, ++ arguments: Dict[str, Any], ++ *, ++ timeout: float = 8.0, ++ ) -> bool: ++ """Call a simple HTTP JSON-RPC MCP screen tool directly.""" ++ if not screen_url: ++ return False ++ try: ++ from aiohttp import ClientSession, ClientTimeout ++ payload = { ++ "jsonrpc": "2.0", ++ "id": uuid.uuid4().hex[:12], ++ "method": "tools/call", ++ "params": {"name": tool_name, "arguments": arguments}, ++ } ++ async with ClientSession(timeout=ClientTimeout(total=timeout)) as session: ++ async with session.post(screen_url, json=payload) as resp: ++ body = await resp.text() ++ if resp.status >= 400: ++ logger.warning( ++ "[api_server] screen MCP %s failed: HTTP %s %s", ++ tool_name, ++ resp.status, ++ body[:300], ++ ) ++ return False ++ try: ++ data = json.loads(body) if body else {} ++ if isinstance(data, dict) and data.get("error"): ++ logger.warning("[api_server] screen MCP %s returned error: %s", tool_name, data.get("error")) ++ return False ++ except Exception: ++ pass ++ return True ++ except Exception as exc: ++ logger.warning("[api_server] screen MCP %s failed: %s", tool_name, exc) ++ return False ++ ++ @staticmethod ++ def _screen_stream_target(screen_url: Optional[str]) -> Optional[tuple[str, int]]: ++ """Return the raw UDP video-stream target for MCP screens that expose one.""" ++ if not screen_url: ++ return None ++ try: ++ parsed = urlparse(screen_url) ++ host = parsed.hostname ++ if not host: ++ return None ++ return (host, 8082) ++ except Exception: ++ return None ++ ++ @staticmethod ++ def _voice_animation_udp_frames(state: str, text: str = "", frames: int = 18) -> List[bytes]: ++ """Render small Hermes voice-state animations as RLCD-compatible frames.""" ++ from PIL import Image, ImageDraw ++ import math ++ ++ width, height = 400, 300 ++ ++ def to_rlcd(img: "Image.Image") -> bytes: ++ img_1bit = img.convert("1", dither=Image.Dither.FLOYDSTEINBERG) ++ px = img_1bit.load() ++ buf = bytearray(15000) ++ for y in range(height): ++ for x in range(width): ++ if px[x, y]: ++ inv_y = 299 - y ++ bx = x // 2 ++ by = inv_y // 4 ++ idx = bx * 75 + by ++ lx = x % 2 ++ ly = inv_y % 4 ++ bit = 7 - (ly * 2 + lx) ++ buf[idx] |= (1 << bit) ++ return bytes(buf) ++ ++ label_by_state = { ++ "ack": "Got it", ++ "captioning": "Listening", ++ "waiting": "Thinking", ++ "speaking": "Replying", ++ "success": "Done", ++ "error": "Try again", ++ } ++ label = label_by_state.get(str(state), str(state or "Hermes").title()) ++ snippet = " ".join(str(text or "").split())[:42] ++ rendered: List[bytes] = [] ++ for frame in range(max(1, int(frames))): ++ img = Image.new("1", (width, height), 0) ++ draw = ImageDraw.Draw(img) ++ cx, cy = width // 2, height // 2 ++ phase = frame * 0.35 ++ if state == "captioning": ++ draw.rectangle((188, 38, 212, 75), outline=1) ++ draw.arc((175, 55, 225, 105), 20, 160, fill=1) ++ for x in range(42, width - 42, 5): ++ y = cy + int(32 * math.sin(x * 0.045 + phase * 2)) ++ y2 = cy + int(14 * math.sin(x * 0.085 - phase * 3)) ++ draw.line((x, y, min(width - 1, x + 4), y2), fill=1) ++ elif state in {"waiting", "speaking"}: ++ for radius in (34, 64, 94): ++ pulse = radius + int(8 * math.sin(phase + radius)) ++ draw.ellipse((cx - pulse, cy - pulse, cx + pulse, cy + pulse), outline=1) ++ for i in range(7): ++ x = 93 + i * 34 ++ bar = int(18 + 42 * (0.5 + 0.5 * math.sin(phase * 2 + i))) ++ draw.rectangle((x, 228 - bar, x + 17, 228), fill=1) ++ elif state == "error": ++ draw.line((150, 88, 250, 188), fill=1) ++ draw.line((250, 88, 150, 188), fill=1) ++ draw.rectangle((145, 82, 255, 194), outline=1) ++ else: ++ r = 42 + int(16 * math.sin(phase)) ++ draw.ellipse((cx - r, cy - r, cx + r, cy + r), outline=1) ++ draw.ellipse((cx - 8, cy - 8, cx + 8, cy + 8), fill=1) ++ draw.text((max(8, (width - len(label) * 6) // 2), 24), label, fill=1) ++ if snippet: ++ draw.text((max(8, (width - len(snippet) * 6) // 2), 260), snippet, fill=1) ++ rendered.append(to_rlcd(img)) ++ return rendered ++ ++ async def _send_stream_animation( ++ self, ++ screen_url: Optional[str], ++ state: str, ++ *, ++ text: str = "", ++ frames: Optional[int] = None, ++ ) -> bool: ++ """Send a server-rendered voice animation over the screen streaming socket.""" ++ target = self._screen_stream_target(screen_url) ++ if not target: ++ return False ++ frame_count = frames if frames is not None else (12 if state == "ack" else 18) ++ try: ++ rendered = await asyncio.to_thread(self._voice_animation_udp_frames, state, text, frame_count) ++ sock = _socket.socket(_socket.AF_INET, _socket.SOCK_DGRAM) ++ try: ++ for frame_id, frame in enumerate(rendered): ++ fid = frame_id % 256 ++ for chunk_idx in range(15): ++ start = chunk_idx * 1000 ++ sock.sendto(bytes([fid, chunk_idx]) + frame[start : start + 1000], target) ++ await asyncio.sleep(0.045) ++ finally: ++ sock.close() ++ return True ++ except Exception as exc: ++ logger.warning("[api_server] screen stream animation failed: %s", exc) ++ return False ++ ++ async def _send_screen_animation( ++ self, ++ screen_url: Optional[str], ++ state: str, ++ *, ++ text: str = "", ++ frames: Optional[int] = None, ++ ) -> bool: ++ """Upload and run the ESP32 voice-state animation script on the initiating screen.""" ++ if not screen_url: ++ return False ++ if await self._send_stream_animation(screen_url, state, text=text, frames=frames): ++ return True ++ if screen_url not in self._esp32_voice_animation_uploaded: ++ uploaded = await self._call_screen_mcp( ++ screen_url, ++ "write_file", ++ {"path": ESP32_VOICE_ANIMATION_PATH, "content": ESP32_VOICE_ANIMATION_SCRIPT}, ++ ) ++ if not uploaded: ++ return False ++ self._esp32_voice_animation_uploaded.add(screen_url) ++ code = ( ++ f"exec(open({ESP32_VOICE_ANIMATION_PATH!r}).read()); " ++ f"run_state({str(state)!r}, text={str(text or '')[:240]!r}, frames={frames!r})" ++ ) ++ return await self._call_screen_mcp(screen_url, "execute_python", {"code": code}) ++ ++ async def _send_screen_text(self, screen_url: Optional[str], text: str, *, clear: bool = False) -> None: ++ """Best-effort text update to the initiating MCP screen.""" ++ if not screen_url: ++ return ++ if clear: ++ await self._call_screen_mcp(screen_url, "clear_screen", {"color": 0}) ++ # Most Reyna screens accept x/y/text. Extra wrapping is avoided here so ++ # firmware-specific clients can keep their own font/layout behavior. ++ await self._call_screen_mcp(screen_url, "draw_text", {"x": 8, "y": 8, "text": text[:900]}) ++ ++ @staticmethod ++ def _extract_first_image_reference(text: str) -> Optional[str]: ++ """Return the first local path, MEDIA path, data URL, or HTTP image URL in text.""" ++ if not text: ++ return None ++ patterns = [ ++ r"MEDIA:([^\s)]+)", ++ r"!\[[^\]]*\]\(([^)]+)\)", ++ r"(?Phttps?://[^\s)]+\.(?:png|jpe?g|webp|gif)(?:\?[^\s)]*)?)", ++ r"(?P/[^\s)]+\.(?:png|jpe?g|webp|gif|bmp))", ++ r"(?Pdata:image/[^\s)]+)", ++ ] ++ for pattern in patterns: ++ for match in re.finditer(pattern, text, flags=re.IGNORECASE): ++ ref = match.groupdict().get("url") or match.groupdict().get("path") or match.groupdict().get("data") or match.group(1) ++ if ref: ++ return ref.strip().strip('"\'') ++ return None ++ ++ async def _send_screen_image_response(self, screen_url: Optional[str], text: str) -> bool: ++ """Best-effort render of the first generated/linked image onto the ESP32 screen.""" ++ if not screen_url: ++ return False ++ image_ref = self._extract_first_image_reference(text) ++ if not image_ref: ++ return False ++ try: ++ import base64 ++ import io ++ from PIL import Image, ImageOps ++ ++ raw: bytes ++ lowered = image_ref.lower() ++ if lowered.startswith("data:image/"): ++ _, _, encoded = image_ref.partition(",") ++ raw = base64.b64decode(encoded) ++ elif self._looks_like_http_url(image_ref): ++ from aiohttp import ClientSession, ClientTimeout ++ async with ClientSession(timeout=ClientTimeout(total=20)) as session: ++ async with session.get(image_ref) as resp: ++ if resp.status >= 400: ++ logger.warning("[api_server] screen image fetch failed: HTTP %s", resp.status) ++ return False ++ raw = await resp.read() ++ else: ++ path = Path(image_ref).expanduser() ++ if not path.exists() or not path.is_file(): ++ return False ++ raw = await asyncio.to_thread(path.read_bytes) ++ if not raw: ++ return False ++ ++ def render_pbm() -> str: ++ with Image.open(io.BytesIO(raw)) as img: ++ img = ImageOps.exif_transpose(img).convert("RGB") ++ canvas = Image.new("RGB", (320, 240), "white") ++ img.thumbnail((320, 240), Image.Resampling.LANCZOS) ++ x = (320 - img.width) // 2 ++ y = (240 - img.height) // 2 ++ canvas.paste(img, (x, y)) ++ mono = canvas.convert("1", dither=Image.Dither.FLOYDSTEINBERG) ++ out = io.BytesIO() ++ mono.save(out, format="PPM") # mode=1 writes raw PBM (P4) ++ return base64.b64encode(out.getvalue()).decode("ascii") ++ ++ pbm_base64 = await asyncio.to_thread(render_pbm) ++ return await self._call_screen_mcp( ++ screen_url, ++ "draw_image", ++ {"pbm_base64": pbm_base64, "x": 0, "y": 0, "dither": False}, ++ timeout=15, ++ ) ++ except Exception: ++ logger.exception("[api_server] failed to render ESP32 screen image response") ++ return False ++ ++ async def _send_screen_audio_response( ++ self, ++ screen_url: Optional[str], ++ text: str, ++ *, ++ audio_dir: Path, ++ device_id: str, ++ volume: int = 80, ++ ) -> bool: ++ """Best-effort spoken follow-up for ACK-mode ESP32 voice turns.""" ++ if not screen_url or not text.strip(): ++ return False ++ try: ++ import base64 ++ from tools.tts_tool import text_to_speech_tool ++ ++ # Keep board playback short and payloads small. The ESP32 HTTP MCP ++ # endpoint resets on large base64 WAV bodies; MP3 is much smaller ++ # and the firmware exposes play_mp3_base64 for exactly this path. ++ spoken_text = self._short_voice_summary(text, 420) ++ tts_raw = await asyncio.to_thread(text_to_speech_tool, spoken_text) ++ tts_result = json.loads(tts_raw) ++ src = str(tts_result.get("file_path") or "") ++ if not tts_result.get("success") or not src or not Path(src).exists(): ++ logger.warning( ++ "[api_server] screen audio TTS failed: %s", ++ tts_result.get("error") or "missing TTS output", ++ ) ++ return False ++ ++ src_path = Path(src) ++ if src_path.suffix.lower() == ".mp3": ++ mp3_base64 = base64.b64encode(src_path.read_bytes()).decode("ascii") ++ if await self._call_screen_mcp( ++ screen_url, ++ "play_mp3_base64", ++ {"mp3_base64": mp3_base64, "volume": volume}, ++ timeout=75, ++ ): ++ return True ++ ++ wav_path = await asyncio.to_thread( ++ self._convert_audio_to_esp32_wav, ++ src, ++ output_dir=audio_dir, ++ device_id=f"{device_id}_response", ++ ) ++ wav_bytes = wav_path.read_bytes() ++ if len(wav_bytes) > 220_000: ++ logger.warning( ++ "[api_server] screen audio WAV too large for MCP payload: %s bytes", ++ len(wav_bytes), ++ ) ++ return False ++ wav_base64 = base64.b64encode(wav_bytes).decode("ascii") ++ return await self._call_screen_mcp( ++ screen_url, ++ "play_audio_base64", ++ {"wav_base64": wav_base64, "volume": volume}, ++ timeout=45, ++ ) ++ except Exception: ++ logger.exception("[api_server] failed to play ESP32 screen audio response") ++ return False ++ ++ @staticmethod ++ def _short_voice_summary(text: str, limit: int = 220) -> str: ++ """Local fallback summary for voice/screen snippets without another LLM call.""" ++ clean = " ".join(str(text or "").split()) ++ if len(clean) <= limit: ++ return clean ++ return clean[: max(0, limit - 1)].rstrip() + "…" ++ ++ async def _esp32_ack_wav(self, *, audio_dir: Path, device_id: str) -> Path: ++ """Return a reusable quiet ACK WAV; the visual screen animation is the acknowledgement.""" ++ ack_path = audio_dir / "quick_ack.wav" ++ if ack_path.exists(): ++ return ack_path ++ ++ # ACK-mode clients expect a valid audio/wav response immediately, but ++ # Reyna ESP32 screens now show the acknowledgement visually. Keep the ++ # HTTP contract while avoiding the old spoken "Got it" message. ++ import wave ++ sample_rate = 16000 ++ duration_s = 0.08 ++ frames = b"\x00\x00" * int(sample_rate * duration_s) ++ with wave.open(str(ack_path), "wb") as wav: ++ wav.setnchannels(1) ++ wav.setsampwidth(2) ++ wav.setframerate(sample_rate) ++ wav.writeframes(frames) ++ return ack_path ++ ++ def _append_esp32_voice_audit(self, audio_dir: Path, event: Dict[str, Any]) -> None: ++ """Append a best-effort ESP32 voice request audit event.""" ++ try: ++ log_path = audio_dir / "voice_requests.jsonl" ++ with log_path.open("a", encoding="utf-8") as handle: ++ handle.write(json.dumps(event, default=str) + "\n") ++ except Exception: ++ logger.debug("[api_server] failed to append ESP32 voice audit log", exc_info=True) ++ ++ async def _run_esp32_voice_audio_background( ++ self, ++ *, ++ audio_path: Path, ++ audit_event: Dict[str, Any], ++ device_id: str, ++ instructions: Optional[str], ++ audio_dir: Path, ++ screen_url: Optional[str], ++ ) -> None: ++ """Transcribe uploaded ESP32 audio and continue the voice turn after an immediate ACK.""" ++ try: ++ await self._send_screen_animation(screen_url, "captioning") ++ from tools.transcription_tools import transcribe_audio ++ stt_result = await asyncio.to_thread(transcribe_audio, str(audio_path)) ++ if not stt_result.get("success"): ++ failed_event = dict(audit_event) ++ failed_event.update({ ++ "status": "transcription_failed", ++ "error": stt_result.get("error") or "Transcription failed", ++ "timestamp": time.time(), ++ }) ++ self._append_esp32_voice_audit(audio_dir, failed_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but transcription failed. Try again.", ++ clear=True, ++ ) ++ return ++ transcript = str(stt_result.get("transcript") or "").strip() ++ if not transcript: ++ empty_event = dict(audit_event) ++ empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) ++ self._append_esp32_voice_audit(audio_dir, empty_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but got no words. Try again.", ++ clear=True, ++ ) ++ return ++ transcript_event = dict(audit_event) ++ transcript_event.update({"status": "transcribed", "transcript": transcript, "timestamp": time.time()}) ++ self._append_esp32_voice_audit(audio_dir, transcript_event) ++ await self._run_esp32_voice_background( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=instructions, ++ audio_dir=audio_dir, ++ screen_url=screen_url, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 ACK background voice processing failed") ++ failed_event = dict(audit_event) ++ failed_event.update({"status": "background_exception", "error": str(exc), "timestamp": time.time()}) ++ self._append_esp32_voice_audit(audio_dir, failed_event) ++ await self._send_screen_text( ++ screen_url, ++ "I got the audio, but background processing failed.", ++ clear=True, ++ ) ++ ++ async def _run_esp32_voice_background( ++ self, ++ *, ++ transcript: str, ++ device_id: str, ++ instructions: Optional[str], ++ audio_dir: Path, ++ screen_url: Optional[str], ++ ) -> None: ++ """Run the slow ESP32 voice turn after the device already heard an ACK.""" ++ try: ++ animation_ok = await self._send_screen_animation(screen_url, "waiting", text=transcript) ++ if not animation_ok: ++ await self._send_screen_text( ++ screen_url, ++ f"Heard: {self._short_voice_summary(transcript, 180)}\n\nWorking…", ++ clear=True, ++ ) ++ screen_hint = ( ++ "\n\nThis request came from an ESP32 voice/screen device. " ++ "Keep the spoken follow-up short. The gateway will display the full answer on the initiating screen when possible." ++ ) ++ merged_instructions = (instructions or "") + screen_hint ++ response_data, final_text = await self._run_esp32_voice_turn( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=merged_instructions, ++ ) ++ if not final_text.strip(): ++ final_text = self._extract_response_text(response_data) or "I finished, but I do not have a text answer." ++ display_text = final_text ++ if len(display_text) > 1200: ++ display_text = self._short_voice_summary(display_text, 900) + "\n\nFull answer is long; WhatsApp delivery is the next wiring step." ++ await self._send_screen_text(screen_url, display_text, clear=True) ++ await self._send_screen_image_response(screen_url, final_text) ++ await self._send_screen_audio_response( ++ screen_url, ++ final_text, ++ audio_dir=audio_dir, ++ device_id=device_id, ++ ) ++ except Exception: ++ logger.exception("[api_server] ESP32 background voice turn failed") ++ await self._send_screen_text(screen_url, "Hermes hit an error while preparing the full answer.", clear=True) ++ ++ @staticmethod ++ def _extract_response_text(response_data: Dict[str, Any]) -> str: ++ """Extract assistant text from an OpenAI Responses-shaped payload.""" ++ chunks: List[str] = [] ++ for item in response_data.get("output") or []: ++ if not isinstance(item, dict) or item.get("type") != "message": ++ continue ++ for part in item.get("content") or []: ++ if isinstance(part, dict) and part.get("type") in {"output_text", "text"}: ++ text = part.get("text") ++ if isinstance(text, str) and text: ++ chunks.append(text) ++ return "\n".join(chunks).strip() ++ ++ async def _run_esp32_voice_turn( ++ self, ++ *, ++ transcript: str, ++ device_id: str, ++ instructions: Optional[str] = None, ++ ) -> tuple[Dict[str, Any], str]: ++ """Run one persistent Responses-style turn for an ESP32 voice device.""" ++ conversation = f"esp32:{device_id}" ++ previous_response_id = self._response_store.get_conversation(conversation) ++ conversation_history: List[Dict[str, Any]] = [] ++ stored_session_id = None ++ stored_instructions = None ++ ++ if previous_response_id: ++ stored = self._response_store.get(previous_response_id) ++ if stored is not None: ++ conversation_history = list(stored.get("conversation_history", [])) ++ stored_session_id = stored.get("session_id") ++ stored_instructions = stored.get("instructions") ++ ++ if instructions is None: ++ instructions = stored_instructions ++ ++ session_id = stored_session_id or str(uuid.uuid4()) ++ user_message = ( ++ f'[Voice input from ESP32 device "{device_id}"]\n' ++ f'{transcript}' ++ ) ++ ++ result, usage = await self._run_agent( ++ user_message=user_message, ++ conversation_history=conversation_history, ++ ephemeral_system_prompt=instructions, ++ session_id=session_id, ++ gateway_session_key=conversation, ++ ) ++ ++ final_response = result.get("final_response", "") or result.get("error", "") or "(No response generated)" ++ response_id = f"resp_{uuid.uuid4().hex[:28]}" ++ created_at = int(time.time()) ++ full_history = self._build_response_conversation_history( ++ conversation_history, ++ user_message, ++ result, ++ final_response, ++ ) ++ output_start_index = self._response_messages_turn_start_index( ++ conversation_history, ++ user_message, ++ result, ++ ) ++ output_items = self._extract_output_items(result, start_index=output_start_index) ++ response_data = { ++ "id": response_id, ++ "object": "response", ++ "status": "completed", ++ "created_at": created_at, ++ "model": self._model_name, ++ "output": output_items, ++ "usage": { ++ "input_tokens": usage.get("input_tokens", 0), ++ "output_tokens": usage.get("output_tokens", 0), ++ "total_tokens": usage.get("total_tokens", 0), ++ }, ++ } ++ self._response_store.put(response_id, { ++ "response": response_data, ++ "conversation_history": full_history, ++ "instructions": instructions, ++ "session_id": session_id, ++ }) ++ self._response_store.set_conversation(conversation, response_id) ++ return response_data, final_response ++ ++ @staticmethod ++ def _esp32_audio_cache_dir() -> Path: ++ """Return the cache directory for ESP32 voice artifacts.""" ++ try: ++ from hermes_constants import get_hermes_dir ++ return get_hermes_dir("cache/audio", "audio_cache") / "esp32" ++ except Exception: ++ return Path.home() / ".hermes" / "cache" / "audio" / "esp32" ++ ++ @staticmethod ++ def _write_pcm_s16le_wav(raw_path: Path, wav_path: Path, *, sample_rate: int, channels: int = 1) -> None: ++ """Wrap a raw PCM s16le file in a minimal RIFF/WAVE container.""" ++ import wave ++ ++ with wave.open(str(wav_path), "wb") as wav: ++ wav.setnchannels(channels) ++ wav.setsampwidth(2) ++ wav.setframerate(sample_rate) ++ with raw_path.open("rb") as raw_handle: ++ while True: ++ chunk = raw_handle.read(64 * 1024) ++ if not chunk: ++ break ++ wav.writeframes(chunk) ++ ++ # --- Live draft helpers (from your whisper-translation/apple_speech pipe protocol) --- ++ def _load_macmini_mcp_config(self) -> dict: ++ try: ++ if ESP32_MACMINI_MCP_URL and ESP32_MACMINI_MCP_TOKEN: ++ return {"url": ESP32_MACMINI_MCP_URL, "token": ESP32_MACMINI_MCP_TOKEN} ++ cfg_path = Path.home() / ".hermes" / "config.yaml" ++ if cfg_path.exists(): ++ import yaml ++ cfg = yaml.safe_load(cfg_path.read_text()) or {} ++ m = (cfg.get("mcp_servers") or {}).get("macmini") or {} ++ url = m.get("url") or "" ++ token = "" ++ headers = m.get("headers") or {} ++ auth = headers.get("Authorization") or "" ++ if auth.startswith("Bearer "): ++ token = auth[7:] ++ if url: ++ return {"url": url, "token": token} ++ except Exception: ++ pass ++ return {} ++ ++ @staticmethod ++ def _s16le_to_wav_bytes(pcm_bytes: bytes, sample_rate: int = 16000, channels: int = 1) -> bytes: ++ import io, wave ++ bio = io.BytesIO() ++ with wave.open(bio, "wb") as wf: ++ wf.setnchannels(channels) ++ wf.setsampwidth(2) ++ wf.setframerate(sample_rate) ++ wf.writeframes(pcm_bytes) ++ return bio.getvalue() ++ ++ async def _transcribe_draft_via_macmini(self, wav_bytes: bytes, locale: str = "en-US") -> str | None: ++ if not ESP32_LIVE_DRAFT_USE_MACMINI: ++ return None ++ cfg = self._load_macmini_mcp_config() ++ url = cfg.get("url") ++ token = cfg.get("token") ++ if not url: ++ return None ++ try: ++ import base64 ++ b64 = base64.b64encode(wav_bytes).decode("ascii") ++ payload = { ++ "jsonrpc": "2.0", ++ "id": 1, ++ "method": "tools/call", ++ "params": {"name": "speech_live_transcribe", "arguments": {"audioBase64": b64, "locale": locale}}, ++ } ++ headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"} ++ if token: ++ headers["Authorization"] = f"Bearer {token}" ++ import aiohttp ++ async with aiohttp.ClientSession() as session: ++ async with session.post(url, json=payload, headers=headers, timeout=aiohttp.ClientTimeout(total=8)) as resp: ++ if resp.status != 200: ++ return None ++ txt = await resp.text() ++ for line in txt.splitlines(): ++ if not line.startswith("data: "): ++ continue ++ try: ++ data = json.loads(line[6:]) ++ content = (data.get("result", {}).get("content") or [{}])[0].get("text", "") or "" ++ if not content: ++ continue ++ inner = json.loads(content) ++ t = inner.get("text") or "" ++ if t: ++ return t.strip() ++ except Exception: ++ continue ++ except Exception as e: ++ logger.debug(f"[api_server] macmini live draft failed: {e}") ++ return None ++ ++ async def _transcribe_draft_local(self, wav_bytes: bytes) -> str | None: ++ try: ++ import tempfile ++ fd, tmp_path = tempfile.mkstemp(suffix=".wav") ++ os.close(fd) ++ Path(tmp_path).write_bytes(wav_bytes) ++ def _do(): ++ try: ++ from faster_whisper import WhisperModel ++ model = None ++ try: ++ from tools.transcription_tools import _local_model ++ if _local_model is not None: ++ model = _local_model ++ except Exception: ++ pass ++ if model is None: ++ model = WhisperModel("tiny", device="cpu", compute_type="int8", num_workers=1) ++ segs, _ = model.transcribe(tmp_path, beam_size=1, best_of=1, language="en", condition_on_previous_text=False, vad_filter=True) ++ return " ".join(s.text.strip() for s in segs).strip() ++ except Exception as ex: ++ logger.debug(f"[api_server] local draft err: {ex}") ++ return None ++ finally: ++ try: ++ Path(tmp_path).unlink(missing_ok=True) ++ except Exception: ++ pass ++ return await asyncio.to_thread(_do) ++ except Exception: ++ return None ++ ++ async def _apple_llm_quick_reply_via_macmini(self, draft: str, context: str = "", instructions: str = "") -> str | None: ++ """Call Mac mini apple_llm_quick_reply for instant smart ACK from draft.""" ++ if not ESP32_LIVE_DRAFT_USE_MACMINI: ++ return None ++ cfg = self._load_macmini_mcp_config() ++ url = cfg.get("url") ++ token = cfg.get("token") ++ if not url: ++ return None ++ try: ++ payload = { ++ "jsonrpc": "2.0", ++ "id": 1, ++ "method": "tools/call", ++ "params": {"name": "apple_llm_quick_reply", "arguments": {"draft": draft, "context": context or "", "instructions": instructions or ""}}, ++ } ++ headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"} ++ if token: ++ headers["Authorization"] = f"Bearer {token}" ++ import aiohttp ++ async with aiohttp.ClientSession() as session: ++ async with session.post(url, json=payload, headers=headers, timeout=aiohttp.ClientTimeout(total=4)) as resp: ++ if resp.status != 200: ++ return None ++ txt = await resp.text() ++ for line in txt.splitlines(): ++ if not line.startswith("data: "): ++ continue ++ try: ++ data = json.loads(line[6:]) ++ content = (data.get("result", {}).get("content") or [{}])[0].get("text", "") or "" ++ if not content: ++ continue ++ inner = json.loads(content) ++ t = inner.get("text") or "" ++ if t: ++ return t.strip() ++ except Exception: ++ continue ++ except Exception as e: ++ logger.debug(f"[api_server] macmini quick_reply failed: {e}") ++ return None ++ ++ async def _run_quick_reply_then_full(self, draft_text: str, final_transcript: str, device_id: str, screen_url, instructions: str | None, audio_dir): ++ """New flow: when voice stops, we already have draft. Emit instant smart reply via Apple LLM, then full Hermes turn.""" ++ # This is kept for future speculative pre-start; for now we just log — actual quick reply emitted in WS handler before background start ++ pass ++ ++ ++ async def _handle_esp32_voice_ws(self, request: "web.Request") -> "web.WebSocketResponse": ++ """GET /api/esp32/voice/ws — ESP32 PCM streaming with LIVE draft captioning. ++ LIVE DRAFT MODE (from whisper-translation/engine_apple_transcribe.py): ++ - While ingesting binary s16le, every ~800ms snapshots current PCM -> in-mem WAV ++ -> tries Mac mini speech_live_transcribe (ApplePipeTranscriber --pipe volatile, ++ <200ms drafts) fallback to local tiny for fast feedback. ++ - Emits WS events: draft + interim_transcript for UI. Final via macmini or local base. ++ """ ++ auth_err = self._check_auth(request) ++ if auth_err: ++ return auth_err # type: ignore[return-value] ++ ++ ws = web.WebSocketResponse(heartbeat=None, max_msg_size=512 * 1024) ++ await ws.prepare(request) ++ ++ async def _send_event(event: str, **payload): ++ if not ws.closed: ++ await ws.send_str(json.dumps({"event": event, **payload})) ++ ++ device_id = ( ++ request.headers.get("X-Device-ID") ++ or request.query.get("device_id") ++ or "default" ++ ).strip()[:80] or "default" ++ safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" ++ audio_dir = self._esp32_audio_cache_dir() ++ audio_dir.mkdir(parents=True, exist_ok=True) ++ stamp = f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}" ++ raw_path = audio_dir / f"{stamp}.s16le" ++ wav_path = audio_dir / f"{stamp}.wav" ++ sample_rate = 16000 ++ channels = 1 ++ sample_width = 2 ++ total_audio_bytes = 0 ++ started = False ++ stopped = False ++ first_audio_at: float | None = None ++ started_at = time.time() ++ # live draft state (directly from your whisper-translation ApplePipeTranscriber pattern) ++ live_enabled = ESP32_LIVE_DRAFT_ENABLED ++ pcm_buffer = bytearray() ++ last_draft_at = 0.0 ++ last_draft_bytes = 0 ++ last_draft_text = "" ++ draft_in_progress = False ++ draft_count = 0 ++ locale_req = request.headers.get("X-Hermes-Locale") or request.query.get("locale") or "en-US" ++ ++ audit_ctx = self._request_audit_context(request) ++ audit_event = { ++ "timestamp": started_at, ++ "device_id": device_id, ++ "source_ip": audit_ctx.get("remote") or audit_ctx.get("peer_ip") or "", ++ "peer_ip": audit_ctx.get("peer_ip") or "", ++ "forwarded_for": audit_ctx.get("forwarded_for") or "", ++ "real_ip": audit_ctx.get("real_ip") or "", ++ "user_agent": audit_ctx.get("user_agent") or "", ++ "transport": "websocket", ++ "raw_path": str(raw_path), ++ "audio_path": str(wav_path), ++ "status": "accepted", ++ "live_draft_enabled": live_enabled, ++ } ++ self._append_esp32_voice_audit(audio_dir, audit_event) ++ screen_url = self._resolve_esp32_screen_url(request, device_id) ++ visual_listening_sent = False ++ ++ def _schedule_voice_state(state: str, *, text: str = "", frames: int | None = None) -> None: ++ if not screen_url: ++ return ++ task = asyncio.create_task(self._send_screen_animation(screen_url, state, text=text, frames=frames)) ++ try: ++ self._background_tasks.add(task) ++ except TypeError: ++ pass ++ if hasattr(task, "add_done_callback"): ++ task.add_done_callback(self._background_tasks.discard) ++ ++ async def _do_live_draft(snapshot_bytes: bytes): ++ nonlocal last_draft_text, draft_in_progress, draft_count ++ if draft_in_progress: ++ return ++ if len(snapshot_bytes) < int(sample_rate * 0.5 * 2): # at least 500ms of s16le at 16k mono ++ return ++ draft_in_progress = True ++ try: ++ wav_b = self._s16le_to_wav_bytes(snapshot_bytes, sample_rate=sample_rate, channels=channels) ++ txt = None ++ if ESP32_LIVE_DRAFT_USE_MACMINI: ++ txt = await self._transcribe_draft_via_macmini(wav_b, locale=locale_req) ++ if not txt: ++ txt = await self._transcribe_draft_local(wav_b) ++ if not txt: ++ return ++ txt = txt.strip() ++ if not txt or txt == last_draft_text or len(txt) < 2: ++ return ++ last_draft_text = txt ++ draft_count += 1 ++ await _send_event("draft", text=txt, is_final=False, isFinal=False, chunk=draft_count, bytes=len(snapshot_bytes)) ++ await _send_event("interim_transcript", text=txt, is_final=False, draft_count=draft_count) ++ if screen_url and len(txt) > 1: ++ try: ++ t = asyncio.create_task(self._send_screen_text(screen_url, txt, clear=False)) ++ self._background_tasks.add(t) ++ t.add_done_callback(self._background_tasks.discard) ++ except Exception: ++ pass ++ finally: ++ draft_in_progress = False ++ ++ await _send_event("ready", device_id=device_id, screen_url=screen_url or "", live_draft=live_enabled, live_model="apple-pipe-volatile") ++ ++ try: ++ raw_handle = raw_path.open("wb") ++ try: ++ async for msg in ws: ++ if msg.type == web.WSMsgType.TEXT: ++ try: ++ payload = json.loads(msg.data) ++ except json.JSONDecodeError: ++ await _send_event("error", code="bad_json", message="Text frames must be JSON") ++ continue ++ event = str(payload.get("event") or payload.get("type") or "").strip().lower() ++ if event == "start": ++ if started: ++ await _send_event("error", code="already_started", message="Audio stream already started") ++ continue ++ fmt = str(payload.get("format") or "pcm_s16le").strip().lower() ++ channels = int(payload.get("channels") or 1) ++ sample_width = int(payload.get("sample_width") or 2) ++ sample_rate = int(payload.get("sample_rate") or 16000) ++ device_id = str(payload.get("device_id") or device_id).strip()[:80] or device_id ++ if fmt not in {"pcm_s16le", "s16le", "pcm"} or channels != 1 or sample_width != 2: ++ await _send_event("error", code="unsupported_audio_format", message="Use 16 kHz mono signed 16-bit little-endian PCM") ++ await ws.close() ++ return ws ++ if sample_rate < 8000 or sample_rate > 48000: ++ await _send_event("error", code="bad_sample_rate", message="sample_rate must be 8000-48000") ++ await ws.close() ++ return ws ++ started = True ++ if not visual_listening_sent: ++ _schedule_voice_state("captioning", frames=8) ++ visual_listening_sent = True ++ await _send_event("listening", sample_rate=sample_rate, channels=channels, sample_width=sample_width) ++ elif event in {"stop", "end"}: ++ stopped = True ++ break ++ elif event in {"ping", "keepalive"}: ++ await _send_event("pong") ++ else: ++ await _send_event("error", code="unknown_event", message=f"Unknown event: {event or ''}") ++ elif msg.type == web.WSMsgType.BINARY: ++ if not started: ++ started = True ++ if not visual_listening_sent: ++ _schedule_voice_state("captioning", frames=8) ++ visual_listening_sent = True ++ await _send_event("listening", sample_rate=sample_rate, channels=channels, sample_width=sample_width) ++ chunk = bytes(msg.data or b"") ++ if not chunk: ++ continue ++ if first_audio_at is None: ++ first_audio_at = time.time() ++ total_audio_bytes += len(chunk) ++ pcm_buffer.extend(chunk) ++ if total_audio_bytes > ESP32_WS_MAX_AUDIO_BYTES: ++ await _send_event("error", code="audio_too_large", message=f"Audio stream exceeded {ESP32_WS_MAX_AUDIO_BYTES} bytes") ++ await ws.close() ++ return ws ++ raw_handle.write(chunk) ++ # live draft trigger (every 800ms & 8KB new data) — direct captioning fluff stripped ++ if live_enabled: ++ now = time.time() ++ elapsed_ms = (now - last_draft_at) * 1000 ++ new_bytes = len(pcm_buffer) - last_draft_bytes ++ total_ms = len(pcm_buffer) / 2 / sample_rate * 1000 ++ if total_ms >= ESP32_LIVE_DRAFT_MIN_TOTAL_MS and elapsed_ms >= ESP32_LIVE_DRAFT_INTERVAL_MS and new_bytes >= ESP32_LIVE_DRAFT_MIN_BYTES: ++ last_draft_at = now ++ last_draft_bytes = len(pcm_buffer) ++ # copy snapshot, fire-and-forget draft (but guard concurrent) ++ snapshot = bytes(pcm_buffer) ++ asyncio.create_task(_do_live_draft(snapshot)) ++ elif msg.type == web.WSMsgType.ERROR: ++ logger.warning("[api_server] ESP32 voice websocket error: %s", ws.exception()) ++ break ++ finally: ++ raw_handle.close() ++ ++ if not stopped: ++ await _send_event("error", code="missing_stop", message="Send a stop event after audio chunks") ++ await ws.close() ++ return ws ++ if total_audio_bytes < ESP32_WS_MIN_AUDIO_BYTES: ++ await _send_event("error", code="audio_too_short", message="Not enough audio received") ++ await ws.close() ++ return ws ++ ++ await asyncio.to_thread( ++ self._write_pcm_s16le_wav, ++ raw_path, ++ wav_path, ++ sample_rate=sample_rate, ++ channels=channels, ++ ) ++ audit_event.update({ ++ "status": "received", ++ "audio_bytes": total_audio_bytes, ++ "sample_rate": sample_rate, ++ "first_audio_latency_ms": int(((first_audio_at or started_at) - started_at) * 1000), ++ "stop_latency_ms": int((time.time() - started_at) * 1000), ++ "draft_count": draft_count, ++ "last_draft": last_draft_text, ++ }) ++ self._append_esp32_voice_audit(audio_dir, dict(audit_event)) ++ ++ _schedule_voice_state("captioning", frames=10) ++ await _send_event("thinking", stage="transcribing", draft_count=draft_count) ++ ++ # Final transcription: prefer Mac mini live pipe (ApplePipeTranscriber) — we already have full pcm_buffer ++ # This replaces the old 700ms faster-whisper base path. Fallback to local only if Mac unavailable. ++ transcript = None ++ final_source = "none" ++ t_transcribe_start = time.time() ++ ++ # Strategy: try Mac mini with FULL buffer via speech_live_transcribe (final), then check drafts ++ try: ++ if ESP32_LIVE_DRAFT_USE_MACMINI and len(pcm_buffer) > 0: ++ full_wav = self._s16le_to_wav_bytes(bytes(pcm_buffer), sample_rate=sample_rate, channels=channels) ++ mac_txt = await self._transcribe_draft_via_macmini(full_wav, locale=locale_req) ++ if mac_txt and len(mac_txt.strip()) >= 2: ++ transcript = mac_txt.strip() ++ final_source = "macmini-live-final" ++ except Exception as e: ++ logger.debug(f"[api_server] macmini final transcribe failed: {e}") ++ ++ if not transcript: ++ from tools.transcription_tools import transcribe_audio ++ # If we already wrote wav_path above, reuse else write now ++ try: ++ # ensure wav_path exists (we haven't needed it till now after refactor) ++ if not wav_path.exists(): ++ await asyncio.to_thread( ++ self._write_pcm_s16le_wav, ++ raw_path, ++ wav_path, ++ sample_rate=sample_rate, ++ channels=channels, ++ ) ++ except Exception: ++ pass ++ stt_result = await asyncio.to_thread(transcribe_audio, str(wav_path) if wav_path.exists() else str(raw_path)) ++ if not stt_result.get("success"): ++ message = stt_result.get("error") or "Transcription failed" ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "transcription_failed", "error": message, "timestamp": time.time()}) ++ await _send_event("error", code="transcription_failed", message=message) ++ await ws.close() ++ return ws ++ transcript = str(stt_result.get("transcript") or "").strip() ++ final_source = "local-faster-whisper" ++ ++ if not transcript: ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "empty_transcript", "timestamp": time.time()}) ++ await _send_event("error", code="empty_transcript", message="Transcription was empty") ++ await ws.close() ++ return ws ++ ++ transcribe_ms = int((time.time() - t_transcribe_start) * 1000) ++ ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "transcribed", "transcript": transcript, "timestamp": time.time(), "draft_count": draft_count, "final_source": final_source, "transcribe_ms": transcribe_ms}) ++ await _send_event("transcript", text=transcript, is_final=True, isFinal=True, draft_count=draft_count, source=final_source, transcribe_ms=transcribe_ms) ++ await _send_event("final", text=transcript, isFinal=True, source=final_source) ++ ++ summary = self._short_voice_summary(transcript, 180) ++ ack_text = f"Heard: {summary}\n\nWorking on the full answer now." ++ await _send_event("response_text", text=ack_text, response_id="", final=False) ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "queued_background", "transcript": transcript, "timestamp": time.time()}) ++ ++ # --- NEW: instant smart reply from draft via Mac mini Apple 3B LLM (ANE, ~0.6s) --- ++ # This gives user a smart ACK almost immediately when voice stops, before full Hermes turn. ++ # From whisper-translation engine_apple_llm.py: AppleLLM polish/quick_reply pipe. ++ t_quick_start = time.time() ++ instant_reply = None ++ try: ++ # Prefer last_draft_text if exists (live), else final transcript ++ draft_for_llm = last_draft_text if last_draft_text and len(last_draft_text) > 3 else transcript ++ # Only call Mac mini if we have draft and Mac enabled ++ if draft_for_llm and ESP32_LIVE_DRAFT_USE_MACMINI: ++ await _send_event("thinking", stage="instant_reply", draft=draft_for_llm[:120]) ++ instant_reply = await self._apple_llm_quick_reply_via_macmini( ++ draft=draft_for_llm, ++ context=transcript[:300] if transcript != draft_for_llm else "", ++ instructions=request.headers.get("X-Hermes-Instructions") or "" ++ ) ++ if instant_reply: ++ quick_ms = int((time.time() - t_quick_start) * 1000) ++ await _send_event("instant_response", text=instant_reply, is_quick=True, quick_ms=quick_ms, draft=draft_for_llm[:200], source="apple-llm-3b-ane") ++ await _send_event("quick_reply", text=instant_reply, ms=quick_ms) ++ # Also push to screen instantly (direct captioning -> instant smart reply) ++ if screen_url: ++ try: ++ task = asyncio.create_task(self._send_screen_text(screen_url, instant_reply, clear=True)) ++ self._background_tasks.add(task) ++ task.add_done_callback(self._background_tasks.discard) ++ except Exception: ++ pass ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "instant_reply", "instant_reply": instant_reply, "quick_ms": quick_ms, "timestamp": time.time()}) ++ except Exception as e: ++ logger.debug(f"[api_server] instant reply failed: {e}") ++ ++ # Immediate background LLM start — no extra wait (user flagged LLM awaiting whole thing) ++ try: ++ await _send_event("done", mode="background", screen_url=screen_url or "", draft_count=draft_count, source=final_source, instant_reply=instant_reply or "") ++ await ws.close() ++ except Exception: ++ pass ++ background_task = asyncio.create_task( ++ self._run_esp32_voice_background( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=request.headers.get("X-Hermes-Instructions") or None, ++ audio_dir=audio_dir, ++ screen_url=screen_url, ++ ) ++ ) ++ try: ++ self._background_tasks.add(background_task) ++ except TypeError: ++ pass ++ if hasattr(background_task, "add_done_callback"): ++ background_task.add_done_callback(self._background_tasks.discard) ++ return ws ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 voice websocket processing failed") ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "websocket_exception", "error": str(exc), "timestamp": time.time()}) ++ try: ++ await _send_event("error", code="internal_error", message=str(exc)) ++ await ws.close() ++ except Exception: ++ pass ++ return ws ++ finally: ++ try: ++ raw_path.unlink(missing_ok=True) ++ except Exception: ++ pass ++ ++ async def _handle_esp32_voice(self, request: "web.Request") -> "web.Response": ++ ++ """POST /api/esp32/voice — receive audio, run Hermes, return TTS audio.""" ++ auth_err = self._check_auth(request) ++ if auth_err: ++ return auth_err ++ ++ device_id = ( ++ request.headers.get("X-Device-ID") ++ or request.query.get("device_id") ++ or "default" ++ ).strip()[:80] or "default" ++ ++ try: ++ audio_bytes = await request.read() ++ except Exception as exc: ++ return web.json_response(_openai_error(f"Failed to read audio body: {exc}"), status=400) ++ ++ if not audio_bytes: ++ return web.json_response(_openai_error("Empty audio body", code="empty_audio"), status=400) ++ if len(audio_bytes) > ESP32_AUDIO_MAX_BYTES: ++ return web.json_response( ++ _openai_error( ++ f"Audio body is too large ({len(audio_bytes)} bytes); max is {ESP32_AUDIO_MAX_BYTES}", ++ code="audio_too_large", ++ ), ++ status=413, ++ ) ++ ++ suffix = self._audio_extension_for_content_type(request.headers.get("Content-Type", "")) ++ try: ++ from hermes_constants import get_hermes_dir ++ audio_dir = get_hermes_dir("cache/audio", "audio_cache") / "esp32" ++ except Exception: ++ audio_dir = Path.home() / ".hermes" / "cache" / "audio" / "esp32" ++ audio_dir.mkdir(parents=True, exist_ok=True) ++ safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" ++ audio_path = audio_dir / f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}{suffix}" ++ audio_path.write_bytes(audio_bytes) ++ ++ audit_ctx = self._request_audit_context(request) ++ audit_event: Dict[str, Any] = { ++ "timestamp": time.time(), ++ "device_id": device_id, ++ "source_ip": audit_ctx.get("remote") or audit_ctx.get("peer_ip") or "", ++ "peer_ip": audit_ctx.get("peer_ip") or "", ++ "forwarded_for": audit_ctx.get("forwarded_for") or "", ++ "real_ip": audit_ctx.get("real_ip") or "", ++ "user_agent": audit_ctx.get("user_agent") or "", ++ "reply_mode": self._esp32_reply_mode(request), ++ "screen_device": request.headers.get("X-Hermes-Screen-Device") or request.query.get("screen_device") or "", ++ "screen_url": request.headers.get("X-Hermes-Screen-Url") or request.headers.get("X-Screen-MCP-Url") or request.query.get("screen_url") or "", ++ "content_type": request.headers.get("Content-Type", ""), ++ "audio_bytes": len(audio_bytes), ++ "audio_path": str(audio_path), ++ "status": "received", ++ } ++ ++ def _append_audit(event: Dict[str, Any]) -> None: ++ try: ++ log_path = audio_dir / "voice_requests.jsonl" ++ with log_path.open("a", encoding="utf-8") as handle: ++ handle.write(json.dumps(event, default=str) + "\n") ++ except Exception: ++ logger.debug("[api_server] failed to append ESP32 voice audit log", exc_info=True) ++ ++ _append_audit(audit_event) ++ ++ if self._esp32_reply_mode(request) == "ack": ++ screen_url = self._resolve_esp32_screen_url(request, device_id) ++ ++ async def _ack_background() -> None: ++ try: ++ await self._send_screen_animation(screen_url, "ack") ++ await self._send_screen_animation(screen_url, "captioning") ++ from tools.transcription_tools import transcribe_audio ++ stt_result_bg = await asyncio.wait_for( ++ asyncio.to_thread(transcribe_audio, str(audio_path)), ++ timeout=45, ++ ) ++ if not stt_result_bg.get("success"): ++ failed_event = dict(audit_event) ++ failed_event.update({ ++ "status": "transcription_failed", ++ "error": stt_result_bg.get("error") or "Transcription failed", ++ "timestamp": time.time(), ++ }) ++ _append_audit(failed_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but transcription failed. Try again.", ++ clear=True, ++ ) ++ return ++ transcript_bg = str(stt_result_bg.get("transcript") or "").strip() ++ if not transcript_bg: ++ empty_event = dict(audit_event) ++ empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) ++ _append_audit(empty_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but got no words. Try again.", ++ clear=True, ++ ) ++ return ++ transcript_event = dict(audit_event) ++ transcript_event.update({"status": "transcribed", "transcript": transcript_bg, "timestamp": time.time()}) ++ _append_audit(transcript_event) ++ await self._run_esp32_voice_background( ++ transcript=transcript_bg, ++ device_id=device_id, ++ instructions=request.headers.get("X-Hermes-Instructions") or None, ++ audio_dir=audio_dir, ++ screen_url=screen_url, ++ ) ++ except asyncio.TimeoutError: ++ timeout_event = dict(audit_event) ++ timeout_event.update({"status": "transcription_timeout", "error": "Transcription exceeded 45s", "timestamp": time.time()}) ++ _append_audit(timeout_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but captioning timed out. Try again.", ++ clear=True, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 ack-mode background processing failed") ++ failed_event = dict(audit_event) ++ failed_event.update({"status": "background_exception", "error": str(exc), "timestamp": time.time()}) ++ _append_audit(failed_event) ++ ++ task = asyncio.create_task(_ack_background()) ++ try: ++ self._background_tasks.add(task) ++ except TypeError: ++ pass ++ if hasattr(task, "add_done_callback"): ++ task.add_done_callback(self._background_tasks.discard) ++ ack_path = await self._esp32_ack_wav(audio_dir=audio_dir, device_id=device_id) ++ headers = { ++ "X-Hermes-Conversation": f"esp32:{device_id}", ++ "X-Hermes-Reply-Mode": "ack", ++ "X-Hermes-Screen-Url": screen_url or "", ++ "Content-Type": "audio/wav", ++ } ++ return web.FileResponse(ack_path, headers=headers) ++ ++ try: ++ from tools.transcription_tools import transcribe_audio ++ stt_result = await asyncio.to_thread(transcribe_audio, str(audio_path)) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 transcription failed") ++ failed_event = dict(audit_event) ++ failed_event.update({"status": "transcription_exception", "error": str(exc), "timestamp": time.time()}) ++ _append_audit(failed_event) ++ return web.json_response(_openai_error(f"Transcription failed: {exc}"), status=500) ++ ++ if not stt_result.get("success"): ++ failed_event = dict(audit_event) ++ failed_event.update({ ++ "status": "transcription_failed", ++ "error": stt_result.get("error") or "Transcription failed", ++ "timestamp": time.time(), ++ }) ++ _append_audit(failed_event) ++ return web.json_response( ++ _openai_error(stt_result.get("error") or "Transcription failed", code="transcription_failed"), ++ status=502, ++ ) ++ ++ transcript = str(stt_result.get("transcript") or "").strip() ++ if not transcript: ++ empty_event = dict(audit_event) ++ empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) ++ _append_audit(empty_event) ++ return web.json_response(_openai_error("Transcription was empty", code="empty_transcript"), status=422) ++ ++ transcript_event = dict(audit_event) ++ transcript_event.update({"status": "transcribed", "transcript": transcript, "timestamp": time.time()}) ++ _append_audit(transcript_event) ++ ++ instructions = request.headers.get("X-Hermes-Instructions") or None ++ try: ++ response_data, final_text = await self._run_esp32_voice_turn( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=instructions, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 agent turn failed") ++ return web.json_response(_openai_error(f"Agent turn failed: {exc}"), status=500) ++ ++ if not final_text.strip(): ++ final_text = self._extract_response_text(response_data) or "I heard you, but I do not have a spoken answer." ++ ++ try: ++ from tools.tts_tool import text_to_speech_tool ++ tts_raw = await asyncio.to_thread(text_to_speech_tool, final_text) ++ tts_result = json.loads(tts_raw) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 TTS failed") ++ return web.json_response( ++ _openai_error(f"TTS failed after response was generated: {exc}", code="tts_failed"), ++ status=500, ++ headers={ ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ }, ++ ) ++ ++ if not tts_result.get("success"): ++ return web.json_response( ++ _openai_error(tts_result.get("error") or "TTS failed", code="tts_failed"), ++ status=502, ++ headers={ ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ }, ++ ) ++ ++ tts_path = str(tts_result.get("file_path") or "") ++ if not tts_path or not Path(tts_path).exists(): ++ return web.json_response( ++ _openai_error("TTS succeeded but no audio file was produced", code="tts_file_missing"), ++ status=500, ++ ) ++ ++ try: ++ response_audio_path = self._convert_audio_to_esp32_wav( ++ tts_path, ++ output_dir=audio_dir, ++ device_id=device_id, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 WAV normalization failed") ++ return web.json_response( ++ _openai_error(f"Could not convert TTS audio to ESP32 WAV: {exc}", code="wav_conversion_failed"), ++ status=500, ++ headers={ ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ }, ++ ) ++ ++ headers = { ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ "X-Hermes-Response-Id": str(response_data.get("id") or ""), ++ "X-Hermes-Conversation": f"esp32:{device_id}", ++ "X-Hermes-TTS-Source-Type": self._audio_response_mime(tts_path), ++ "Content-Type": "audio/wav", ++ } ++ return web.FileResponse( ++ response_audio_path, ++ headers=headers, ++ ) ++ ++ # ------------------------------------------------------------------ ++ # Cron jobs API ++ # ------------------------------------------------------------------ ++ ++ _JOB_ID_RE = __import__("re").compile(r"[a-f0-9]{12}") ++ # Allowed fields for update — prevents clients injecting arbitrary keys ++ _UPDATE_ALLOWED_FIELDS = {"name", "schedule", "prompt", "deliver", "skills", "skill", "repeat", "enabled"} ++ _MAX_NAME_LENGTH = 200 ++ _MAX_PROMPT_LENGTH = 5000 ++ ++ @staticmethod ++ ++ + async def _handle_models(self, request: "web.Request") -> "web.Response": + """GET /v1/models — list hermes-agent and any configured model_routes aliases.""" + auth_err = self._check_auth(request) +@@ -5018,6 +6651,9 @@ class APIServerAdapter(BasePlatformAdapter): + # bootstrap shims use this key as a feature-detection hook; registering + # native routes first lets those shims no-op instead of shadowing the + # upstream session-control handlers. ++ # Local ESP32 push-to-talk endpoints: POST complete audio, or stream PCM over WebSocket. ++ self._app.router.add_post("/api/esp32/voice", self._handle_esp32_voice) ++ self._app.router.add_get("/api/esp32/voice/ws", self._handle_esp32_voice_ws) + self._app["api_server_adapter"] = self + + # Start background sweep to clean up orphaned (unconsumed) run streams diff --git a/docs/esp32_voice_patch.patch b/docs/esp32_voice_patch.patch new file mode 100644 index 0000000..71b1852 --- /dev/null +++ b/docs/esp32_voice_patch.patch @@ -0,0 +1,1383 @@ +diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py +index 628713224..2691159a9 100644 +--- a/gateway/platforms/api_server.py ++++ b/gateway/platforms/api_server.py +@@ -34,6 +34,7 @@ Requires: + import asyncio + import hashlib + import hmac ++import ipaddress + import json + from contextlib import contextmanager + from contextvars import ContextVar +@@ -47,6 +48,7 @@ import time + import uuid + from pathlib import Path + from typing import Any, Dict, List, Optional ++from urllib.parse import urlparse + + def _approval_event_choices(*, smart_denied: bool, allow_permanent: bool) -> list[str]: + if smart_denied: +@@ -106,6 +108,38 @@ CHAT_COMPLETIONS_SSE_KEEPALIVE_SECONDS = 30.0 + MAX_NORMALIZED_TEXT_LENGTH = 65_536 # 64 KB cap for normalized content parts + MAX_CONTENT_LIST_SIZE = 1_000 # Max items when content is an array + ++ESP32_AUDIO_MAX_BYTES = 8_000_000 # ~4 minutes of 16 kHz mono PCM WAV; enough for push-to-talk tests ++ESP32_WS_MAX_AUDIO_BYTES = ESP32_AUDIO_MAX_BYTES ++ESP32_WS_MIN_AUDIO_BYTES = 1_600 # 50 ms at 16 kHz mono s16le; rejects empty accidental taps ++ESP32_WS_TTS_CHUNK_BYTES = 16_384 ++ ++_ESP32_AUDIO_EXTENSIONS = { ++ "audio/wav": ".wav", ++ "audio/wave": ".wav", ++ "audio/x-wav": ".wav", ++ "audio/mpeg": ".mp3", ++ "audio/mp3": ".mp3", ++ "audio/ogg": ".ogg", ++ "audio/opus": ".ogg", ++ "audio/webm": ".webm", ++ "audio/mp4": ".m4a", ++ "audio/x-m4a": ".m4a", ++ "application/octet-stream": ".wav", ++} ++ ++_AUDIO_RESPONSE_MIME_BY_SUFFIX = { ++ ".mp3": "audio/mpeg", ++ ".wav": "audio/wav", ++ ".ogg": "audio/ogg", ++ ".opus": "audio/ogg", ++ ".m4a": "audio/mp4", ++ ".mp4": "audio/mp4", ++ ".flac": "audio/flac", ++} ++ ++ESP32_VOICE_ANIMATION_PATH = "hermes_voice_animations.py" ++ESP32_VOICE_ANIMATION_SCRIPT = "# Hermes ESP32 voice-state animations for MicroPython screen boards.\n# Uploaded by the API server; can also be run manually with:\n# exec(open('hermes_voice_animations.py').read()); run_state('ack')\nimport time\ntry:\n import board_config\nexcept Exception:\n board_config = None\n\n\ndef _display():\n return getattr(board_config, 'display_instance', None) if board_config else None\n\n\ndef _sleep(ms):\n try:\n time.sleep_ms(ms)\n except AttributeError:\n time.sleep(ms / 1000)\n\n\ndef _text(d, msg, x, y, scale=1, color=1):\n if hasattr(d, 'text_large') and scale > 1:\n try:\n d.text_large(msg, x, y, scale=scale, color=color)\n return\n except TypeError:\n pass\n d.text(msg, x, y, color)\n\n\ndef _center_text(d, msg, y, scale=1, color=1):\n width = getattr(d, 'width', 320)\n char_w = 8 * scale\n x = max(0, (width - len(msg) * char_w) // 2)\n _text(d, msg, x, y, scale=scale, color=color)\n\n\ndef _wrap(text, width_chars=34, max_lines=4):\n words = str(text or '').replace('\\n', ' ').split()\n lines, line = [], ''\n for w in words:\n candidate = (line + ' ' + w).strip()\n if len(candidate) > width_chars and line:\n lines.append(line)\n line = w\n if len(lines) >= max_lines:\n break\n else:\n line = candidate\n if line and len(lines) < max_lines:\n lines.append(line)\n return lines\n\n\ndef _spinner(d, title, subtitle='', frames=24, delay=70):\n width, height = getattr(d, 'width', 320), getattr(d, 'height', 240)\n cx, cy = width // 2, height // 2 - 8\n spokes = [(-22,0),(-16,-16),(0,-22),(16,-16),(22,0),(16,16),(0,22),(-16,16)]\n for frame in range(frames):\n d.clear(0)\n _center_text(d, title, 18, scale=2)\n for i, (dx, dy) in enumerate(spokes):\n color = 1 if (i + frame) % len(spokes) in (0, 1, 2, 3) else 0\n size = 4 if i == frame % len(spokes) else 2\n d.fill_rect(cx + dx - size // 2, cy + dy - size // 2, size, size, color)\n d.rect(8, height - 34, width - 16, 18, 1)\n d.fill_rect(10, height - 32, ((width - 20) * ((frame % 16) + 1)) // 16, 14, 1)\n if subtitle:\n _center_text(d, subtitle[:32], height - 56, scale=1)\n d.show()\n _sleep(delay)\n\n\ndef ack(frames=18):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'GOT IT', 18, scale=2)\n # expanding check pulse\n box = 48 + (i % 6) * 8\n x, y = width // 2 - box // 2, height // 2 - box // 2\n d.rect(x, y, box, box, 1)\n d.line(width//2 - 24, height//2, width//2 - 6, height//2 + 18, 1)\n d.line(width//2 - 6, height//2 + 18, width//2 + 30, height//2 - 22, 1)\n _center_text(d, 'Listening captured', height - 36, scale=1)\n d.show()\n _sleep(55)\n\n\ndef captioning(frames=36):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'CAPTIONING', 16, scale=2)\n for bar in range(9):\n h = 8 + ((i * 5 + bar * 13) % 42)\n x = width // 2 - 72 + bar * 18\n d.fill_rect(x, height // 2 - h // 2, 10, h, 1)\n dots = '.' * ((i // 5) % 4)\n _center_text(d, 'Transcribing' + dots, height - 42, scale=1)\n d.show()\n _sleep(65)\n\n\ndef waiting(text='', frames=44):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'THINKING', 12, scale=2)\n r = 18 + (i % 10) * 2\n cx, cy = width // 2, height // 2 - 10\n d.rect(cx - r, cy - r, r * 2, r * 2, 1)\n d.rect(cx - r//2, cy - r//2, r, r, 1)\n if text:\n d.text('Heard:', 10, height - 68, 1)\n for idx, line in enumerate(_wrap(text, 36, 3)):\n d.text(line[:38], 10, height - 54 + idx * 12, 1)\n else:\n _center_text(d, 'Waiting for reply...', height - 42, scale=1)\n d.show()\n _sleep(75)\n\n\ndef run_state(state='ack', text='', frames=None):\n state = str(state or 'ack').lower()\n if state in ('ack', 'listening', 'got_it'):\n ack(frames or 18)\n elif state in ('caption', 'captioning', 'transcribing'):\n captioning(frames or 36)\n elif state in ('wait', 'waiting', 'thinking', 'reply'):\n waiting(text, frames or 44)\n else:\n _spinner(_display(), 'HERMES', state[:28], frames or 20)\n" ++ + + def _coerce_port(value: Any, default: int = DEFAULT_PORT) -> int: + """Parse a listen port without letting malformed env/config values crash startup.""" +@@ -1585,6 +1619,1313 @@ class APIServerAdapter(BasePlatformAdapter): + "pid": os.getpid(), + }) + ++ @staticmethod ++ def _audio_extension_for_content_type(content_type: str) -> str: ++ """Map an inbound audio Content-Type to a safe cache-file suffix.""" ++ media_type = (content_type or "").split(";", 1)[0].strip().lower() ++ return _ESP32_AUDIO_EXTENSIONS.get(media_type, ".wav") ++ ++ @staticmethod ++ def _audio_response_mime(path: str) -> str: ++ """Return an HTTP Content-Type for a generated audio file path.""" ++ suffix = Path(path).suffix.lower() ++ return _AUDIO_RESPONSE_MIME_BY_SUFFIX.get(suffix, "application/octet-stream") ++ ++ @staticmethod ++ ++ def _convert_audio_to_esp32_wav(input_path: str, *, output_dir: Path, device_id: str) -> Path: ++ """Normalize generated TTS audio to ESP32-friendly WAV. ++ ++ Hermes TTS providers commonly emit MP3 (Edge) or OGG. The ESP32 test ++ client expects a small, predictable RIFF/WAVE file, so decode with ++ ffmpeg to raw PCM and rewrite the WAV container with Python's ``wave`` ++ module. That produces a minimal ``fmt`` + ``data`` WAV: mono, 16 kHz, ++ signed 16-bit little-endian PCM. ++ """ ++ import subprocess ++ import wave ++ ++ safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" ++ output_dir.mkdir(parents=True, exist_ok=True) ++ raw_path = output_dir / f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}.s16le" ++ wav_path = raw_path.with_suffix(".wav") ++ cmd = [ ++ "ffmpeg", ++ "-y", ++ "-hide_banner", ++ "-loglevel", ++ "error", ++ "-i", ++ input_path, ++ "-ac", ++ "1", ++ "-ar", ++ "16000", ++ "-f", ++ "s16le", ++ str(raw_path), ++ ] ++ try: ++ subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ++ raw = raw_path.read_bytes() ++ with wave.open(str(wav_path), "wb") as wav: ++ wav.setnchannels(1) ++ wav.setsampwidth(2) ++ wav.setframerate(16000) ++ wav.writeframes(raw) ++ return wav_path ++ finally: ++ try: ++ raw_path.unlink(missing_ok=True) ++ except Exception: ++ pass ++ ++ @staticmethod ++ def _safe_audio_header(value: Any, max_length: int) -> str: ++ """Make diagnostic text safe for HTTP response headers.""" ++ text = str(value or "")[:max_length] ++ return text.replace("\r", " ").replace("\n", " ").replace("\x00", " ") ++ ++ @staticmethod ++ def _esp32_reply_mode(request: "web.Request") -> str: ++ """Return 'sync' or 'ack' for the ESP32 voice response contract.""" ++ value = ( ++ request.headers.get("X-Hermes-Reply-Mode") ++ or request.query.get("reply_mode") ++ or request.query.get("mode") ++ or "sync" ++ ) ++ value = str(value).strip().lower() ++ if value in {"ack", "async", "background", "quick_ack", "quick-ack"}: ++ return "ack" ++ return "sync" ++ ++ @staticmethod ++ def _looks_like_http_url(value: str) -> bool: ++ try: ++ parsed = urlparse(value) ++ return parsed.scheme in {"http", "https"} and bool(parsed.netloc) ++ except Exception: ++ return False ++ ++ @staticmethod ++ def _load_local_device_url(device_name: str) -> Optional[str]: ++ """Best-effort local MCP screen lookup from ~/.hermes/local_devices.yaml.""" ++ if not device_name: ++ return None ++ try: ++ import yaml ++ registry = Path.home() / ".hermes" / "local_devices.yaml" ++ data = yaml.safe_load(registry.read_text()) if registry.exists() else {} ++ aliases = data.get("aliases") if isinstance(data, dict) else None ++ devices = data.get("devices") if isinstance(data, dict) else None ++ names = [str(device_name).strip(), str(device_name).strip().replace("-", "_")] ++ seen: set[str] = set() ++ while names: ++ name = names.pop(0) ++ if not name or name in seen: ++ continue ++ seen.add(name) ++ if isinstance(aliases, dict): ++ alias = aliases.get(name) ++ if isinstance(alias, str) and alias: ++ if alias.startswith("http"): ++ return alias ++ names.append(alias) ++ info = devices.get(name) if isinstance(devices, dict) else None ++ if isinstance(info, dict): ++ return str(info.get("fallback_url") or info.get("url") or "") or None ++ except Exception: ++ return None ++ return None ++ ++ @staticmethod ++ def _load_local_device_url_by_ip(source_ip: str) -> Optional[str]: ++ """Best-effort local MCP screen lookup by observed device IP. ++ ++ Some MCP-capable devices, notably the iPhone companion app, do not run ++ their MCP endpoint on port 80. When the voice gateway sees traffic from ++ one of those devices, prefer the registry URL over synthesizing ++ http:///api/mcp. ++ """ ++ value = str(source_ip or "").strip() ++ if not value: ++ return None ++ if value.startswith("::ffff:"): ++ value = value.removeprefix("::ffff:") ++ try: ++ import yaml ++ registry = Path.home() / ".hermes" / "local_devices.yaml" ++ data = yaml.safe_load(registry.read_text()) if registry.exists() else {} ++ devices = data.get("devices") if isinstance(data, dict) else None ++ if not isinstance(devices, dict): ++ return None ++ for info in devices.values(): ++ if not isinstance(info, dict): ++ continue ++ candidates = { ++ str(info.get("reserved_ip") or "").strip(), ++ str(info.get("host") or "").strip(), ++ } ++ for url_key in ("fallback_url", "url"): ++ raw_url = str(info.get(url_key) or "").strip() ++ if raw_url: ++ parsed = urlparse(raw_url) ++ if parsed.hostname: ++ candidates.add(parsed.hostname) ++ if value in candidates: ++ return str(info.get("fallback_url") or info.get("url") or "") or None ++ except Exception: ++ return None ++ return None ++ ++ @staticmethod ++ def _esp32_screen_url_from_source_ip(source_ip: str) -> Optional[str]: ++ """Build the MCP endpoint for the ESP32 that made this request. ++ ++ ESP32 boards can move between DHCP leases. For voice ACK follow-ups the ++ initiating board is the safest target, so prefer its observed peer IP ++ instead of a possibly stale local-device registry entry. ++ """ ++ value = str(source_ip or "").strip() ++ if not value: ++ return None ++ if value.startswith("::ffff:"): ++ value = value.removeprefix("::ffff:") ++ # Do not route to loopback/proxy/invalid addresses; registry fallback is ++ # better in those cases. ++ try: ++ ip = ipaddress.ip_address(value) ++ except ValueError: ++ return None ++ if ip.is_loopback or ip.is_unspecified or ip.is_multicast: ++ return None ++ host = f"[{value}]" if ip.version == 6 else value ++ return f"http://{host}/api/mcp" ++ ++ def _resolve_esp32_screen_url(self, request: "web.Request", device_id: str) -> Optional[str]: ++ """Resolve the screen MCP endpoint associated with this voice turn.""" ++ explicit = ( ++ request.headers.get("X-Hermes-Screen-Url") ++ or request.headers.get("X-Screen-MCP-Url") ++ or request.query.get("screen_url") ++ ) ++ if explicit and self._looks_like_http_url(explicit.strip()): ++ return explicit.strip() ++ ++ if str(device_id or "").strip().lower().startswith("iphone"): ++ iphone_url = self._load_local_device_url(str(device_id).strip()) ++ if iphone_url and self._looks_like_http_url(iphone_url): ++ return iphone_url ++ ++ audit_ctx = self._request_audit_context(request) ++ for source_ip in (audit_ctx.get("peer_ip"), audit_ctx.get("remote")): ++ registry_url = self._load_local_device_url_by_ip(source_ip or "") ++ if registry_url and self._looks_like_http_url(registry_url): ++ return registry_url ++ ++ audit_ctx = self._request_audit_context(request) ++ for source_ip in (audit_ctx.get("peer_ip"), audit_ctx.get("remote")): ++ source_url = self._esp32_screen_url_from_source_ip(source_ip or "") ++ if source_url: ++ return source_url ++ ++ screen_device = ( ++ request.headers.get("X-Hermes-Screen-Device") ++ or request.query.get("screen_device") ++ or device_id ++ ) ++ candidates = [screen_device, "little32", "esp32_screen"] ++ for candidate in candidates: ++ url = self._load_local_device_url(str(candidate).strip()) ++ if url and self._looks_like_http_url(url): ++ return url ++ return None ++ ++ async def _call_screen_mcp( ++ self, ++ screen_url: Optional[str], ++ tool_name: str, ++ arguments: Dict[str, Any], ++ *, ++ timeout: float = 8.0, ++ ) -> bool: ++ """Call a simple HTTP JSON-RPC MCP screen tool directly.""" ++ if not screen_url: ++ return False ++ try: ++ from aiohttp import ClientSession, ClientTimeout ++ payload = { ++ "jsonrpc": "2.0", ++ "id": uuid.uuid4().hex[:12], ++ "method": "tools/call", ++ "params": {"name": tool_name, "arguments": arguments}, ++ } ++ async with ClientSession(timeout=ClientTimeout(total=timeout)) as session: ++ async with session.post(screen_url, json=payload) as resp: ++ body = await resp.text() ++ if resp.status >= 400: ++ logger.warning( ++ "[api_server] screen MCP %s failed: HTTP %s %s", ++ tool_name, ++ resp.status, ++ body[:300], ++ ) ++ return False ++ try: ++ data = json.loads(body) if body else {} ++ if isinstance(data, dict) and data.get("error"): ++ logger.warning("[api_server] screen MCP %s returned error: %s", tool_name, data.get("error")) ++ return False ++ except Exception: ++ pass ++ return True ++ except Exception as exc: ++ logger.warning("[api_server] screen MCP %s failed: %s", tool_name, exc) ++ return False ++ ++ @staticmethod ++ def _screen_stream_target(screen_url: Optional[str]) -> Optional[tuple[str, int]]: ++ """Return the raw UDP video-stream target for MCP screens that expose one.""" ++ if not screen_url: ++ return None ++ try: ++ parsed = urlparse(screen_url) ++ host = parsed.hostname ++ if not host: ++ return None ++ return (host, 8082) ++ except Exception: ++ return None ++ ++ @staticmethod ++ def _voice_animation_udp_frames(state: str, text: str = "", frames: int = 18) -> List[bytes]: ++ """Render small Hermes voice-state animations as RLCD-compatible frames.""" ++ from PIL import Image, ImageDraw ++ import math ++ ++ width, height = 400, 300 ++ ++ def to_rlcd(img: "Image.Image") -> bytes: ++ img_1bit = img.convert("1", dither=Image.Dither.FLOYDSTEINBERG) ++ px = img_1bit.load() ++ buf = bytearray(15000) ++ for y in range(height): ++ for x in range(width): ++ if px[x, y]: ++ inv_y = 299 - y ++ bx = x // 2 ++ by = inv_y // 4 ++ idx = bx * 75 + by ++ lx = x % 2 ++ ly = inv_y % 4 ++ bit = 7 - (ly * 2 + lx) ++ buf[idx] |= (1 << bit) ++ return bytes(buf) ++ ++ label_by_state = { ++ "ack": "Got it", ++ "captioning": "Listening", ++ "waiting": "Thinking", ++ "speaking": "Replying", ++ "success": "Done", ++ "error": "Try again", ++ } ++ label = label_by_state.get(str(state), str(state or "Hermes").title()) ++ snippet = " ".join(str(text or "").split())[:42] ++ rendered: List[bytes] = [] ++ for frame in range(max(1, int(frames))): ++ img = Image.new("1", (width, height), 0) ++ draw = ImageDraw.Draw(img) ++ cx, cy = width // 2, height // 2 ++ phase = frame * 0.35 ++ if state == "captioning": ++ draw.rectangle((188, 38, 212, 75), outline=1) ++ draw.arc((175, 55, 225, 105), 20, 160, fill=1) ++ for x in range(42, width - 42, 5): ++ y = cy + int(32 * math.sin(x * 0.045 + phase * 2)) ++ y2 = cy + int(14 * math.sin(x * 0.085 - phase * 3)) ++ draw.line((x, y, min(width - 1, x + 4), y2), fill=1) ++ elif state in {"waiting", "speaking"}: ++ for radius in (34, 64, 94): ++ pulse = radius + int(8 * math.sin(phase + radius)) ++ draw.ellipse((cx - pulse, cy - pulse, cx + pulse, cy + pulse), outline=1) ++ for i in range(7): ++ x = 93 + i * 34 ++ bar = int(18 + 42 * (0.5 + 0.5 * math.sin(phase * 2 + i))) ++ draw.rectangle((x, 228 - bar, x + 17, 228), fill=1) ++ elif state == "error": ++ draw.line((150, 88, 250, 188), fill=1) ++ draw.line((250, 88, 150, 188), fill=1) ++ draw.rectangle((145, 82, 255, 194), outline=1) ++ else: ++ r = 42 + int(16 * math.sin(phase)) ++ draw.ellipse((cx - r, cy - r, cx + r, cy + r), outline=1) ++ draw.ellipse((cx - 8, cy - 8, cx + 8, cy + 8), fill=1) ++ draw.text((max(8, (width - len(label) * 6) // 2), 24), label, fill=1) ++ if snippet: ++ draw.text((max(8, (width - len(snippet) * 6) // 2), 260), snippet, fill=1) ++ rendered.append(to_rlcd(img)) ++ return rendered ++ ++ async def _send_stream_animation( ++ self, ++ screen_url: Optional[str], ++ state: str, ++ *, ++ text: str = "", ++ frames: Optional[int] = None, ++ ) -> bool: ++ """Send a server-rendered voice animation over the screen streaming socket.""" ++ target = self._screen_stream_target(screen_url) ++ if not target: ++ return False ++ frame_count = frames if frames is not None else (12 if state == "ack" else 18) ++ try: ++ rendered = await asyncio.to_thread(self._voice_animation_udp_frames, state, text, frame_count) ++ sock = _socket.socket(_socket.AF_INET, _socket.SOCK_DGRAM) ++ try: ++ for frame_id, frame in enumerate(rendered): ++ fid = frame_id % 256 ++ for chunk_idx in range(15): ++ start = chunk_idx * 1000 ++ sock.sendto(bytes([fid, chunk_idx]) + frame[start : start + 1000], target) ++ await asyncio.sleep(0.045) ++ finally: ++ sock.close() ++ return True ++ except Exception as exc: ++ logger.warning("[api_server] screen stream animation failed: %s", exc) ++ return False ++ ++ async def _send_screen_animation( ++ self, ++ screen_url: Optional[str], ++ state: str, ++ *, ++ text: str = "", ++ frames: Optional[int] = None, ++ ) -> bool: ++ """Upload and run the ESP32 voice-state animation script on the initiating screen.""" ++ if not screen_url: ++ return False ++ if await self._send_stream_animation(screen_url, state, text=text, frames=frames): ++ return True ++ if screen_url not in self._esp32_voice_animation_uploaded: ++ uploaded = await self._call_screen_mcp( ++ screen_url, ++ "write_file", ++ {"path": ESP32_VOICE_ANIMATION_PATH, "content": ESP32_VOICE_ANIMATION_SCRIPT}, ++ ) ++ if not uploaded: ++ return False ++ self._esp32_voice_animation_uploaded.add(screen_url) ++ code = ( ++ f"exec(open({ESP32_VOICE_ANIMATION_PATH!r}).read()); " ++ f"run_state({str(state)!r}, text={str(text or '')[:240]!r}, frames={frames!r})" ++ ) ++ return await self._call_screen_mcp(screen_url, "execute_python", {"code": code}) ++ ++ async def _send_screen_text(self, screen_url: Optional[str], text: str, *, clear: bool = False) -> None: ++ """Best-effort text update to the initiating MCP screen.""" ++ if not screen_url: ++ return ++ if clear: ++ await self._call_screen_mcp(screen_url, "clear_screen", {"color": 0}) ++ # Most Reyna screens accept x/y/text. Extra wrapping is avoided here so ++ # firmware-specific clients can keep their own font/layout behavior. ++ await self._call_screen_mcp(screen_url, "draw_text", {"x": 8, "y": 8, "text": text[:900]}) ++ ++ @staticmethod ++ def _extract_first_image_reference(text: str) -> Optional[str]: ++ """Return the first local path, MEDIA path, data URL, or HTTP image URL in text.""" ++ if not text: ++ return None ++ patterns = [ ++ r"MEDIA:([^\s)]+)", ++ r"!\[[^\]]*\]\(([^)]+)\)", ++ r"(?Phttps?://[^\s)]+\.(?:png|jpe?g|webp|gif)(?:\?[^\s)]*)?)", ++ r"(?P/[^\s)]+\.(?:png|jpe?g|webp|gif|bmp))", ++ r"(?Pdata:image/[^\s)]+)", ++ ] ++ for pattern in patterns: ++ for match in re.finditer(pattern, text, flags=re.IGNORECASE): ++ ref = match.groupdict().get("url") or match.groupdict().get("path") or match.groupdict().get("data") or match.group(1) ++ if ref: ++ return ref.strip().strip('"\'') ++ return None ++ ++ async def _send_screen_image_response(self, screen_url: Optional[str], text: str) -> bool: ++ """Best-effort render of the first generated/linked image onto the ESP32 screen.""" ++ if not screen_url: ++ return False ++ image_ref = self._extract_first_image_reference(text) ++ if not image_ref: ++ return False ++ try: ++ import base64 ++ import io ++ from PIL import Image, ImageOps ++ ++ raw: bytes ++ lowered = image_ref.lower() ++ if lowered.startswith("data:image/"): ++ _, _, encoded = image_ref.partition(",") ++ raw = base64.b64decode(encoded) ++ elif self._looks_like_http_url(image_ref): ++ from aiohttp import ClientSession, ClientTimeout ++ async with ClientSession(timeout=ClientTimeout(total=20)) as session: ++ async with session.get(image_ref) as resp: ++ if resp.status >= 400: ++ logger.warning("[api_server] screen image fetch failed: HTTP %s", resp.status) ++ return False ++ raw = await resp.read() ++ else: ++ path = Path(image_ref).expanduser() ++ if not path.exists() or not path.is_file(): ++ return False ++ raw = await asyncio.to_thread(path.read_bytes) ++ if not raw: ++ return False ++ ++ def render_pbm() -> str: ++ with Image.open(io.BytesIO(raw)) as img: ++ img = ImageOps.exif_transpose(img).convert("RGB") ++ canvas = Image.new("RGB", (320, 240), "white") ++ img.thumbnail((320, 240), Image.Resampling.LANCZOS) ++ x = (320 - img.width) // 2 ++ y = (240 - img.height) // 2 ++ canvas.paste(img, (x, y)) ++ mono = canvas.convert("1", dither=Image.Dither.FLOYDSTEINBERG) ++ out = io.BytesIO() ++ mono.save(out, format="PPM") # mode=1 writes raw PBM (P4) ++ return base64.b64encode(out.getvalue()).decode("ascii") ++ ++ pbm_base64 = await asyncio.to_thread(render_pbm) ++ return await self._call_screen_mcp( ++ screen_url, ++ "draw_image", ++ {"pbm_base64": pbm_base64, "x": 0, "y": 0, "dither": False}, ++ timeout=15, ++ ) ++ except Exception: ++ logger.exception("[api_server] failed to render ESP32 screen image response") ++ return False ++ ++ async def _send_screen_audio_response( ++ self, ++ screen_url: Optional[str], ++ text: str, ++ *, ++ audio_dir: Path, ++ device_id: str, ++ volume: int = 80, ++ ) -> bool: ++ """Best-effort spoken follow-up for ACK-mode ESP32 voice turns.""" ++ if not screen_url or not text.strip(): ++ return False ++ try: ++ import base64 ++ from tools.tts_tool import text_to_speech_tool ++ ++ # Keep board playback short and payloads small. The ESP32 HTTP MCP ++ # endpoint resets on large base64 WAV bodies; MP3 is much smaller ++ # and the firmware exposes play_mp3_base64 for exactly this path. ++ spoken_text = self._short_voice_summary(text, 420) ++ tts_raw = await asyncio.to_thread(text_to_speech_tool, spoken_text) ++ tts_result = json.loads(tts_raw) ++ src = str(tts_result.get("file_path") or "") ++ if not tts_result.get("success") or not src or not Path(src).exists(): ++ logger.warning( ++ "[api_server] screen audio TTS failed: %s", ++ tts_result.get("error") or "missing TTS output", ++ ) ++ return False ++ ++ src_path = Path(src) ++ if src_path.suffix.lower() == ".mp3": ++ mp3_base64 = base64.b64encode(src_path.read_bytes()).decode("ascii") ++ if await self._call_screen_mcp( ++ screen_url, ++ "play_mp3_base64", ++ {"mp3_base64": mp3_base64, "volume": volume}, ++ timeout=75, ++ ): ++ return True ++ ++ wav_path = await asyncio.to_thread( ++ self._convert_audio_to_esp32_wav, ++ src, ++ output_dir=audio_dir, ++ device_id=f"{device_id}_response", ++ ) ++ wav_bytes = wav_path.read_bytes() ++ if len(wav_bytes) > 220_000: ++ logger.warning( ++ "[api_server] screen audio WAV too large for MCP payload: %s bytes", ++ len(wav_bytes), ++ ) ++ return False ++ wav_base64 = base64.b64encode(wav_bytes).decode("ascii") ++ return await self._call_screen_mcp( ++ screen_url, ++ "play_audio_base64", ++ {"wav_base64": wav_base64, "volume": volume}, ++ timeout=45, ++ ) ++ except Exception: ++ logger.exception("[api_server] failed to play ESP32 screen audio response") ++ return False ++ ++ @staticmethod ++ def _short_voice_summary(text: str, limit: int = 220) -> str: ++ """Local fallback summary for voice/screen snippets without another LLM call.""" ++ clean = " ".join(str(text or "").split()) ++ if len(clean) <= limit: ++ return clean ++ return clean[: max(0, limit - 1)].rstrip() + "…" ++ ++ async def _esp32_ack_wav(self, *, audio_dir: Path, device_id: str) -> Path: ++ """Return a reusable quiet ACK WAV; the visual screen animation is the acknowledgement.""" ++ ack_path = audio_dir / "quick_ack.wav" ++ if ack_path.exists(): ++ return ack_path ++ ++ # ACK-mode clients expect a valid audio/wav response immediately, but ++ # Reyna ESP32 screens now show the acknowledgement visually. Keep the ++ # HTTP contract while avoiding the old spoken "Got it" message. ++ import wave ++ sample_rate = 16000 ++ duration_s = 0.08 ++ frames = b"\x00\x00" * int(sample_rate * duration_s) ++ with wave.open(str(ack_path), "wb") as wav: ++ wav.setnchannels(1) ++ wav.setsampwidth(2) ++ wav.setframerate(sample_rate) ++ wav.writeframes(frames) ++ return ack_path ++ ++ def _append_esp32_voice_audit(self, audio_dir: Path, event: Dict[str, Any]) -> None: ++ """Append a best-effort ESP32 voice request audit event.""" ++ try: ++ log_path = audio_dir / "voice_requests.jsonl" ++ with log_path.open("a", encoding="utf-8") as handle: ++ handle.write(json.dumps(event, default=str) + "\n") ++ except Exception: ++ logger.debug("[api_server] failed to append ESP32 voice audit log", exc_info=True) ++ ++ async def _run_esp32_voice_audio_background( ++ self, ++ *, ++ audio_path: Path, ++ audit_event: Dict[str, Any], ++ device_id: str, ++ instructions: Optional[str], ++ audio_dir: Path, ++ screen_url: Optional[str], ++ ) -> None: ++ """Transcribe uploaded ESP32 audio and continue the voice turn after an immediate ACK.""" ++ try: ++ await self._send_screen_animation(screen_url, "captioning") ++ from tools.transcription_tools import transcribe_audio ++ stt_result = await asyncio.to_thread(transcribe_audio, str(audio_path)) ++ if not stt_result.get("success"): ++ failed_event = dict(audit_event) ++ failed_event.update({ ++ "status": "transcription_failed", ++ "error": stt_result.get("error") or "Transcription failed", ++ "timestamp": time.time(), ++ }) ++ self._append_esp32_voice_audit(audio_dir, failed_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but transcription failed. Try again.", ++ clear=True, ++ ) ++ return ++ transcript = str(stt_result.get("transcript") or "").strip() ++ if not transcript: ++ empty_event = dict(audit_event) ++ empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) ++ self._append_esp32_voice_audit(audio_dir, empty_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but got no words. Try again.", ++ clear=True, ++ ) ++ return ++ transcript_event = dict(audit_event) ++ transcript_event.update({"status": "transcribed", "transcript": transcript, "timestamp": time.time()}) ++ self._append_esp32_voice_audit(audio_dir, transcript_event) ++ await self._run_esp32_voice_background( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=instructions, ++ audio_dir=audio_dir, ++ screen_url=screen_url, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 ACK background voice processing failed") ++ failed_event = dict(audit_event) ++ failed_event.update({"status": "background_exception", "error": str(exc), "timestamp": time.time()}) ++ self._append_esp32_voice_audit(audio_dir, failed_event) ++ await self._send_screen_text( ++ screen_url, ++ "I got the audio, but background processing failed.", ++ clear=True, ++ ) ++ ++ async def _run_esp32_voice_background( ++ self, ++ *, ++ transcript: str, ++ device_id: str, ++ instructions: Optional[str], ++ audio_dir: Path, ++ screen_url: Optional[str], ++ ) -> None: ++ """Run the slow ESP32 voice turn after the device already heard an ACK.""" ++ try: ++ animation_ok = await self._send_screen_animation(screen_url, "waiting", text=transcript) ++ if not animation_ok: ++ await self._send_screen_text( ++ screen_url, ++ f"Heard: {self._short_voice_summary(transcript, 180)}\n\nWorking…", ++ clear=True, ++ ) ++ screen_hint = ( ++ "\n\nThis request came from an ESP32 voice/screen device. " ++ "Keep the spoken follow-up short. The gateway will display the full answer on the initiating screen when possible." ++ ) ++ merged_instructions = (instructions or "") + screen_hint ++ response_data, final_text = await self._run_esp32_voice_turn( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=merged_instructions, ++ ) ++ if not final_text.strip(): ++ final_text = self._extract_response_text(response_data) or "I finished, but I do not have a text answer." ++ display_text = final_text ++ if len(display_text) > 1200: ++ display_text = self._short_voice_summary(display_text, 900) + "\n\nFull answer is long; WhatsApp delivery is the next wiring step." ++ await self._send_screen_text(screen_url, display_text, clear=True) ++ await self._send_screen_image_response(screen_url, final_text) ++ await self._send_screen_audio_response( ++ screen_url, ++ final_text, ++ audio_dir=audio_dir, ++ device_id=device_id, ++ ) ++ except Exception: ++ logger.exception("[api_server] ESP32 background voice turn failed") ++ await self._send_screen_text(screen_url, "Hermes hit an error while preparing the full answer.", clear=True) ++ ++ @staticmethod ++ def _extract_response_text(response_data: Dict[str, Any]) -> str: ++ """Extract assistant text from an OpenAI Responses-shaped payload.""" ++ chunks: List[str] = [] ++ for item in response_data.get("output") or []: ++ if not isinstance(item, dict) or item.get("type") != "message": ++ continue ++ for part in item.get("content") or []: ++ if isinstance(part, dict) and part.get("type") in {"output_text", "text"}: ++ text = part.get("text") ++ if isinstance(text, str) and text: ++ chunks.append(text) ++ return "\n".join(chunks).strip() ++ ++ async def _run_esp32_voice_turn( ++ self, ++ *, ++ transcript: str, ++ device_id: str, ++ instructions: Optional[str] = None, ++ ) -> tuple[Dict[str, Any], str]: ++ """Run one persistent Responses-style turn for an ESP32 voice device.""" ++ conversation = f"esp32:{device_id}" ++ previous_response_id = self._response_store.get_conversation(conversation) ++ conversation_history: List[Dict[str, Any]] = [] ++ stored_session_id = None ++ stored_instructions = None ++ ++ if previous_response_id: ++ stored = self._response_store.get(previous_response_id) ++ if stored is not None: ++ conversation_history = list(stored.get("conversation_history", [])) ++ stored_session_id = stored.get("session_id") ++ stored_instructions = stored.get("instructions") ++ ++ if instructions is None: ++ instructions = stored_instructions ++ ++ session_id = stored_session_id or str(uuid.uuid4()) ++ user_message = ( ++ f'[Voice input from ESP32 device "{device_id}"]\n' ++ f'{transcript}' ++ ) ++ ++ result, usage = await self._run_agent( ++ user_message=user_message, ++ conversation_history=conversation_history, ++ ephemeral_system_prompt=instructions, ++ session_id=session_id, ++ gateway_session_key=conversation, ++ ) ++ ++ final_response = result.get("final_response", "") or result.get("error", "") or "(No response generated)" ++ response_id = f"resp_{uuid.uuid4().hex[:28]}" ++ created_at = int(time.time()) ++ full_history = self._build_response_conversation_history( ++ conversation_history, ++ user_message, ++ result, ++ final_response, ++ ) ++ output_start_index = self._response_messages_turn_start_index( ++ conversation_history, ++ user_message, ++ result, ++ ) ++ output_items = self._extract_output_items(result, start_index=output_start_index) ++ response_data = { ++ "id": response_id, ++ "object": "response", ++ "status": "completed", ++ "created_at": created_at, ++ "model": self._model_name, ++ "output": output_items, ++ "usage": { ++ "input_tokens": usage.get("input_tokens", 0), ++ "output_tokens": usage.get("output_tokens", 0), ++ "total_tokens": usage.get("total_tokens", 0), ++ }, ++ } ++ self._response_store.put(response_id, { ++ "response": response_data, ++ "conversation_history": full_history, ++ "instructions": instructions, ++ "session_id": session_id, ++ }) ++ self._response_store.set_conversation(conversation, response_id) ++ return response_data, final_response ++ ++ @staticmethod ++ def _esp32_audio_cache_dir() -> Path: ++ """Return the cache directory for ESP32 voice artifacts.""" ++ try: ++ from hermes_constants import get_hermes_dir ++ return get_hermes_dir("cache/audio", "audio_cache") / "esp32" ++ except Exception: ++ return Path.home() / ".hermes" / "cache" / "audio" / "esp32" ++ ++ @staticmethod ++ def _write_pcm_s16le_wav(raw_path: Path, wav_path: Path, *, sample_rate: int, channels: int = 1) -> None: ++ """Wrap a raw PCM s16le file in a minimal RIFF/WAVE container.""" ++ import wave ++ ++ with wave.open(str(wav_path), "wb") as wav: ++ wav.setnchannels(channels) ++ wav.setsampwidth(2) ++ wav.setframerate(sample_rate) ++ with raw_path.open("rb") as raw_handle: ++ while True: ++ chunk = raw_handle.read(64 * 1024) ++ if not chunk: ++ break ++ wav.writeframes(chunk) ++ ++ async def _handle_esp32_voice_ws(self, request: "web.Request") -> "web.WebSocketResponse": ++ """GET /api/esp32/voice/ws — stream ESP32 PCM over WebSocket, return TTS audio chunks.""" ++ auth_err = self._check_auth(request) ++ if auth_err: ++ return auth_err # type: ignore[return-value] ++ ++ ws = web.WebSocketResponse(heartbeat=None, max_msg_size=512 * 1024) ++ await ws.prepare(request) ++ ++ async def _send_event(event: str, **payload: Any) -> None: ++ if not ws.closed: ++ await ws.send_str(json.dumps({"event": event, **payload})) ++ ++ device_id = ( ++ request.headers.get("X-Device-ID") ++ or request.query.get("device_id") ++ or "default" ++ ).strip()[:80] or "default" ++ safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" ++ audio_dir = self._esp32_audio_cache_dir() ++ audio_dir.mkdir(parents=True, exist_ok=True) ++ stamp = f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}" ++ raw_path = audio_dir / f"{stamp}.s16le" ++ wav_path = audio_dir / f"{stamp}.wav" ++ sample_rate = 16000 ++ channels = 1 ++ sample_width = 2 ++ total_audio_bytes = 0 ++ started = False ++ stopped = False ++ first_audio_at: Optional[float] = None ++ started_at = time.time() ++ audit_ctx = self._request_audit_context(request) ++ audit_event: Dict[str, Any] = { ++ "timestamp": started_at, ++ "device_id": device_id, ++ "source_ip": audit_ctx.get("remote") or audit_ctx.get("peer_ip") or "", ++ "peer_ip": audit_ctx.get("peer_ip") or "", ++ "forwarded_for": audit_ctx.get("forwarded_for") or "", ++ "real_ip": audit_ctx.get("real_ip") or "", ++ "user_agent": audit_ctx.get("user_agent") or "", ++ "transport": "websocket", ++ "raw_path": str(raw_path), ++ "audio_path": str(wav_path), ++ "status": "accepted", ++ } ++ self._append_esp32_voice_audit(audio_dir, audit_event) ++ screen_url = self._resolve_esp32_screen_url(request, device_id) ++ visual_listening_sent = False ++ ++ def _schedule_voice_state(state: str, *, text: str = "", frames: Optional[int] = None) -> None: ++ if not screen_url: ++ return ++ task = asyncio.create_task(self._send_screen_animation(screen_url, state, text=text, frames=frames)) ++ try: ++ self._background_tasks.add(task) ++ except TypeError: ++ pass ++ if hasattr(task, "add_done_callback"): ++ task.add_done_callback(self._background_tasks.discard) ++ ++ await _send_event("ready", device_id=device_id, screen_url=screen_url or "") ++ ++ try: ++ raw_handle = raw_path.open("wb") ++ try: ++ async for msg in ws: ++ if msg.type == web.WSMsgType.TEXT: ++ try: ++ payload = json.loads(msg.data) ++ except json.JSONDecodeError: ++ await _send_event("error", code="bad_json", message="Text frames must be JSON") ++ continue ++ event = str(payload.get("event") or payload.get("type") or "").strip().lower() ++ if event == "start": ++ if started: ++ await _send_event("error", code="already_started", message="Audio stream already started") ++ continue ++ fmt = str(payload.get("format") or "pcm_s16le").strip().lower() ++ channels = int(payload.get("channels") or 1) ++ sample_width = int(payload.get("sample_width") or 2) ++ sample_rate = int(payload.get("sample_rate") or 16000) ++ device_id = str(payload.get("device_id") or device_id).strip()[:80] or device_id ++ if fmt not in {"pcm_s16le", "s16le", "pcm"} or channels != 1 or sample_width != 2: ++ await _send_event( ++ "error", ++ code="unsupported_audio_format", ++ message="Use 16 kHz mono signed 16-bit little-endian PCM", ++ ) ++ await ws.close() ++ return ws ++ if sample_rate < 8000 or sample_rate > 48000: ++ await _send_event("error", code="bad_sample_rate", message="sample_rate must be 8000-48000") ++ await ws.close() ++ return ws ++ started = True ++ if not visual_listening_sent: ++ _schedule_voice_state("captioning", frames=8) ++ visual_listening_sent = True ++ await _send_event("listening", sample_rate=sample_rate, channels=channels, sample_width=sample_width) ++ elif event in {"stop", "end"}: ++ stopped = True ++ break ++ elif event in {"ping", "keepalive"}: ++ await _send_event("pong") ++ else: ++ await _send_event("error", code="unknown_event", message=f"Unknown event: {event or ''}") ++ elif msg.type == web.WSMsgType.BINARY: ++ if not started: ++ started = True ++ if not visual_listening_sent: ++ _schedule_voice_state("captioning", frames=8) ++ visual_listening_sent = True ++ await _send_event("listening", sample_rate=sample_rate, channels=channels, sample_width=sample_width) ++ chunk = bytes(msg.data or b"") ++ if not chunk: ++ continue ++ if first_audio_at is None: ++ first_audio_at = time.time() ++ total_audio_bytes += len(chunk) ++ if total_audio_bytes > ESP32_WS_MAX_AUDIO_BYTES: ++ await _send_event( ++ "error", ++ code="audio_too_large", ++ message=f"Audio stream exceeded {ESP32_WS_MAX_AUDIO_BYTES} bytes", ++ ) ++ await ws.close() ++ return ws ++ raw_handle.write(chunk) ++ elif msg.type == web.WSMsgType.ERROR: ++ logger.warning("[api_server] ESP32 voice websocket error: %s", ws.exception()) ++ break ++ finally: ++ raw_handle.close() ++ ++ if not stopped: ++ await _send_event("error", code="missing_stop", message="Send a stop event after audio chunks") ++ await ws.close() ++ return ws ++ if total_audio_bytes < ESP32_WS_MIN_AUDIO_BYTES: ++ await _send_event("error", code="audio_too_short", message="Not enough audio received") ++ await ws.close() ++ return ws ++ ++ await asyncio.to_thread( ++ self._write_pcm_s16le_wav, ++ raw_path, ++ wav_path, ++ sample_rate=sample_rate, ++ channels=channels, ++ ) ++ audit_event.update({ ++ "status": "received", ++ "audio_bytes": total_audio_bytes, ++ "sample_rate": sample_rate, ++ "first_audio_latency_ms": int(((first_audio_at or started_at) - started_at) * 1000), ++ "stop_latency_ms": int((time.time() - started_at) * 1000), ++ }) ++ self._append_esp32_voice_audit(audio_dir, dict(audit_event)) ++ ++ _schedule_voice_state("captioning", frames=10) ++ await _send_event("thinking", stage="transcribing") ++ from tools.transcription_tools import transcribe_audio ++ stt_result = await asyncio.to_thread(transcribe_audio, str(wav_path)) ++ if not stt_result.get("success"): ++ message = stt_result.get("error") or "Transcription failed" ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "transcription_failed", "error": message, "timestamp": time.time()}) ++ await _send_event("error", code="transcription_failed", message=message) ++ await ws.close() ++ return ws ++ transcript = str(stt_result.get("transcript") or "").strip() ++ if not transcript: ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "empty_transcript", "timestamp": time.time()}) ++ await _send_event("error", code="empty_transcript", message="Transcription was empty") ++ await ws.close() ++ return ws ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "transcribed", "transcript": transcript, "timestamp": time.time()}) ++ await _send_event("transcript", text=transcript) ++ ++ ack_text = f"Heard: {self._short_voice_summary(transcript, 180)}\n\nWorking on the full answer now." ++ await _send_event("response_text", text=ack_text, response_id="", final=False) ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "queued_background", "transcript": transcript, "timestamp": time.time()}) ++ ++ # Close the client WebSocket before making any agent/tool/TTS/MCP calls. ++ # Some Tactility boards cannot service their HTTP MCP endpoint while the ++ # voice WebSocket is still open and waiting for the final answer. The ++ # slow work continues below as an owned background task and sends text, ++ # audio, and future image output via the board's HTTP MCP endpoint. ++ try: ++ await _send_event("done", mode="background", screen_url=screen_url or "") ++ await ws.close() ++ except Exception: ++ pass ++ background_task = asyncio.create_task( ++ self._run_esp32_voice_background( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=request.headers.get("X-Hermes-Instructions") or None, ++ audio_dir=audio_dir, ++ screen_url=screen_url, ++ ) ++ ) ++ try: ++ self._background_tasks.add(background_task) ++ except TypeError: ++ pass ++ if hasattr(background_task, "add_done_callback"): ++ background_task.add_done_callback(self._background_tasks.discard) ++ return ws ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 voice websocket processing failed") ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "websocket_exception", "error": str(exc), "timestamp": time.time()}) ++ try: ++ await _send_event("error", code="internal_error", message=str(exc)) ++ await ws.close() ++ except Exception: ++ pass ++ return ws ++ finally: ++ try: ++ raw_path.unlink(missing_ok=True) ++ except Exception: ++ pass ++ ++ async def _handle_esp32_voice(self, request: "web.Request") -> "web.Response": ++ """POST /api/esp32/voice — receive audio, run Hermes, return TTS audio.""" ++ auth_err = self._check_auth(request) ++ if auth_err: ++ return auth_err ++ ++ device_id = ( ++ request.headers.get("X-Device-ID") ++ or request.query.get("device_id") ++ or "default" ++ ).strip()[:80] or "default" ++ ++ try: ++ audio_bytes = await request.read() ++ except Exception as exc: ++ return web.json_response(_openai_error(f"Failed to read audio body: {exc}"), status=400) ++ ++ if not audio_bytes: ++ return web.json_response(_openai_error("Empty audio body", code="empty_audio"), status=400) ++ if len(audio_bytes) > ESP32_AUDIO_MAX_BYTES: ++ return web.json_response( ++ _openai_error( ++ f"Audio body is too large ({len(audio_bytes)} bytes); max is {ESP32_AUDIO_MAX_BYTES}", ++ code="audio_too_large", ++ ), ++ status=413, ++ ) ++ ++ suffix = self._audio_extension_for_content_type(request.headers.get("Content-Type", "")) ++ try: ++ from hermes_constants import get_hermes_dir ++ audio_dir = get_hermes_dir("cache/audio", "audio_cache") / "esp32" ++ except Exception: ++ audio_dir = Path.home() / ".hermes" / "cache" / "audio" / "esp32" ++ audio_dir.mkdir(parents=True, exist_ok=True) ++ safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" ++ audio_path = audio_dir / f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}{suffix}" ++ audio_path.write_bytes(audio_bytes) ++ ++ audit_ctx = self._request_audit_context(request) ++ audit_event: Dict[str, Any] = { ++ "timestamp": time.time(), ++ "device_id": device_id, ++ "source_ip": audit_ctx.get("remote") or audit_ctx.get("peer_ip") or "", ++ "peer_ip": audit_ctx.get("peer_ip") or "", ++ "forwarded_for": audit_ctx.get("forwarded_for") or "", ++ "real_ip": audit_ctx.get("real_ip") or "", ++ "user_agent": audit_ctx.get("user_agent") or "", ++ "reply_mode": self._esp32_reply_mode(request), ++ "screen_device": request.headers.get("X-Hermes-Screen-Device") or request.query.get("screen_device") or "", ++ "screen_url": request.headers.get("X-Hermes-Screen-Url") or request.headers.get("X-Screen-MCP-Url") or request.query.get("screen_url") or "", ++ "content_type": request.headers.get("Content-Type", ""), ++ "audio_bytes": len(audio_bytes), ++ "audio_path": str(audio_path), ++ "status": "received", ++ } ++ ++ def _append_audit(event: Dict[str, Any]) -> None: ++ try: ++ log_path = audio_dir / "voice_requests.jsonl" ++ with log_path.open("a", encoding="utf-8") as handle: ++ handle.write(json.dumps(event, default=str) + "\n") ++ except Exception: ++ logger.debug("[api_server] failed to append ESP32 voice audit log", exc_info=True) ++ ++ _append_audit(audit_event) ++ ++ if self._esp32_reply_mode(request) == "ack": ++ screen_url = self._resolve_esp32_screen_url(request, device_id) ++ ++ async def _ack_background() -> None: ++ try: ++ await self._send_screen_animation(screen_url, "ack") ++ await self._send_screen_animation(screen_url, "captioning") ++ from tools.transcription_tools import transcribe_audio ++ stt_result_bg = await asyncio.wait_for( ++ asyncio.to_thread(transcribe_audio, str(audio_path)), ++ timeout=45, ++ ) ++ if not stt_result_bg.get("success"): ++ failed_event = dict(audit_event) ++ failed_event.update({ ++ "status": "transcription_failed", ++ "error": stt_result_bg.get("error") or "Transcription failed", ++ "timestamp": time.time(), ++ }) ++ _append_audit(failed_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but transcription failed. Try again.", ++ clear=True, ++ ) ++ return ++ transcript_bg = str(stt_result_bg.get("transcript") or "").strip() ++ if not transcript_bg: ++ empty_event = dict(audit_event) ++ empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) ++ _append_audit(empty_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but got no words. Try again.", ++ clear=True, ++ ) ++ return ++ transcript_event = dict(audit_event) ++ transcript_event.update({"status": "transcribed", "transcript": transcript_bg, "timestamp": time.time()}) ++ _append_audit(transcript_event) ++ await self._run_esp32_voice_background( ++ transcript=transcript_bg, ++ device_id=device_id, ++ instructions=request.headers.get("X-Hermes-Instructions") or None, ++ audio_dir=audio_dir, ++ screen_url=screen_url, ++ ) ++ except asyncio.TimeoutError: ++ timeout_event = dict(audit_event) ++ timeout_event.update({"status": "transcription_timeout", "error": "Transcription exceeded 45s", "timestamp": time.time()}) ++ _append_audit(timeout_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but captioning timed out. Try again.", ++ clear=True, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 ack-mode background processing failed") ++ failed_event = dict(audit_event) ++ failed_event.update({"status": "background_exception", "error": str(exc), "timestamp": time.time()}) ++ _append_audit(failed_event) ++ ++ task = asyncio.create_task(_ack_background()) ++ try: ++ self._background_tasks.add(task) ++ except TypeError: ++ pass ++ if hasattr(task, "add_done_callback"): ++ task.add_done_callback(self._background_tasks.discard) ++ ack_path = await self._esp32_ack_wav(audio_dir=audio_dir, device_id=device_id) ++ headers = { ++ "X-Hermes-Conversation": f"esp32:{device_id}", ++ "X-Hermes-Reply-Mode": "ack", ++ "X-Hermes-Screen-Url": screen_url or "", ++ "Content-Type": "audio/wav", ++ } ++ return web.FileResponse(ack_path, headers=headers) ++ ++ try: ++ from tools.transcription_tools import transcribe_audio ++ stt_result = await asyncio.to_thread(transcribe_audio, str(audio_path)) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 transcription failed") ++ failed_event = dict(audit_event) ++ failed_event.update({"status": "transcription_exception", "error": str(exc), "timestamp": time.time()}) ++ _append_audit(failed_event) ++ return web.json_response(_openai_error(f"Transcription failed: {exc}"), status=500) ++ ++ if not stt_result.get("success"): ++ failed_event = dict(audit_event) ++ failed_event.update({ ++ "status": "transcription_failed", ++ "error": stt_result.get("error") or "Transcription failed", ++ "timestamp": time.time(), ++ }) ++ _append_audit(failed_event) ++ return web.json_response( ++ _openai_error(stt_result.get("error") or "Transcription failed", code="transcription_failed"), ++ status=502, ++ ) ++ ++ transcript = str(stt_result.get("transcript") or "").strip() ++ if not transcript: ++ empty_event = dict(audit_event) ++ empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) ++ _append_audit(empty_event) ++ return web.json_response(_openai_error("Transcription was empty", code="empty_transcript"), status=422) ++ ++ transcript_event = dict(audit_event) ++ transcript_event.update({"status": "transcribed", "transcript": transcript, "timestamp": time.time()}) ++ _append_audit(transcript_event) ++ ++ instructions = request.headers.get("X-Hermes-Instructions") or None ++ try: ++ response_data, final_text = await self._run_esp32_voice_turn( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=instructions, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 agent turn failed") ++ return web.json_response(_openai_error(f"Agent turn failed: {exc}"), status=500) ++ ++ if not final_text.strip(): ++ final_text = self._extract_response_text(response_data) or "I heard you, but I do not have a spoken answer." ++ ++ try: ++ from tools.tts_tool import text_to_speech_tool ++ tts_raw = await asyncio.to_thread(text_to_speech_tool, final_text) ++ tts_result = json.loads(tts_raw) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 TTS failed") ++ return web.json_response( ++ _openai_error(f"TTS failed after response was generated: {exc}", code="tts_failed"), ++ status=500, ++ headers={ ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ }, ++ ) ++ ++ if not tts_result.get("success"): ++ return web.json_response( ++ _openai_error(tts_result.get("error") or "TTS failed", code="tts_failed"), ++ status=502, ++ headers={ ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ }, ++ ) ++ ++ tts_path = str(tts_result.get("file_path") or "") ++ if not tts_path or not Path(tts_path).exists(): ++ return web.json_response( ++ _openai_error("TTS succeeded but no audio file was produced", code="tts_file_missing"), ++ status=500, ++ ) ++ ++ try: ++ response_audio_path = self._convert_audio_to_esp32_wav( ++ tts_path, ++ output_dir=audio_dir, ++ device_id=device_id, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 WAV normalization failed") ++ return web.json_response( ++ _openai_error(f"Could not convert TTS audio to ESP32 WAV: {exc}", code="wav_conversion_failed"), ++ status=500, ++ headers={ ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ }, ++ ) ++ ++ headers = { ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ "X-Hermes-Response-Id": str(response_data.get("id") or ""), ++ "X-Hermes-Conversation": f"esp32:{device_id}", ++ "X-Hermes-TTS-Source-Type": self._audio_response_mime(tts_path), ++ "Content-Type": "audio/wav", ++ } ++ return web.FileResponse( ++ response_audio_path, ++ headers=headers, ++ ) ++ ++ # ------------------------------------------------------------------ ++ # Cron jobs API ++ # ------------------------------------------------------------------ ++ ++ _JOB_ID_RE = __import__("re").compile(r"[a-f0-9]{12}") ++ # Allowed fields for update — prevents clients injecting arbitrary keys ++ _UPDATE_ALLOWED_FIELDS = {"name", "schedule", "prompt", "deliver", "skills", "skill", "repeat", "enabled"} ++ _MAX_NAME_LENGTH = 200 ++ _MAX_PROMPT_LENGTH = 5000 ++ ++ @staticmethod ++ ++ + async def _handle_models(self, request: "web.Request") -> "web.Response": + """GET /v1/models — list hermes-agent and any configured model_routes aliases.""" + auth_err = self._check_auth(request) +@@ -5018,6 +6359,9 @@ class APIServerAdapter(BasePlatformAdapter): + # bootstrap shims use this key as a feature-detection hook; registering + # native routes first lets those shims no-op instead of shadowing the + # upstream session-control handlers. ++ # Local ESP32 push-to-talk endpoints: POST complete audio, or stream PCM over WebSocket. ++ self._app.router.add_post("/api/esp32/voice", self._handle_esp32_voice) ++ self._app.router.add_get("/api/esp32/voice/ws", self._handle_esp32_voice_ws) + self._app["api_server_adapter"] = self + + # Start background sweep to clean up orphaned (unconsumed) run streams diff --git a/docs/original_patch_source_excerpt.py b/docs/original_patch_source_excerpt.py new file mode 100644 index 0000000..77734f7 --- /dev/null +++ b/docs/original_patch_source_excerpt.py @@ -0,0 +1,3288 @@ +# Original ESP32 extraction (for reference, not used at runtime) +# From api_server.py ESP32_AUDIO_MAX_BYTES -> _handle_esp32_voice +ESP32_AUDIO_MAX_BYTES = 8_000_000 # ~4 minutes of 16 kHz mono PCM WAV; enough for push-to-talk tests +ESP32_WS_MAX_AUDIO_BYTES = 8_000_000 +ESP32_WS_MIN_AUDIO_BYTES = 1_600 # 50 ms at 16 kHz mono s16le; rejects empty accidental taps +ESP32_WS_TTS_CHUNK_BYTES = 16_384 + +# Live draft ingestion config (from whisper-translation/apple_speech engine) +ESP32_LIVE_DRAFT_ENABLED = os.getenv("ESP32_LIVE_DRAFT_ENABLED", "1").lower() not in {"0", "false", "no", "off"} +ESP32_LIVE_DRAFT_INTERVAL_MS = int(os.getenv("ESP32_LIVE_DRAFT_INTERVAL_MS", "800")) # how often to snapshot for draft +ESP32_LIVE_DRAFT_MIN_BYTES = int(os.getenv("ESP32_LIVE_DRAFT_MIN_BYTES", "8000")) # min new bytes since last draft +ESP32_LIVE_DRAFT_MIN_TOTAL_MS = int(os.getenv("ESP32_LIVE_DRAFT_MIN_TOTAL_MS", "500")) # don't draft until we have 500ms +ESP32_MACMINI_MCP_URL = os.getenv("MACMINI_MCP_URL") or None # overridable, else from config.yaml mcp_servers.macmini +ESP32_MACMINI_MCP_TOKEN = os.getenv("MACMINI_MCP_TOKEN") or None +ESP32_LIVE_DRAFT_USE_MACMINI = os.getenv("ESP32_LIVE_DRAFT_USE_MACMINI", "1").lower() not in {"0", "false", "no", "off"} + +_ESP32_AUDIO_EXTENSIONS = { + "audio/wav": ".wav", + "audio/wave": ".wav", + "audio/x-wav": ".wav", + "audio/mpeg": ".mp3", + "audio/mp3": ".mp3", + "audio/ogg": ".ogg", + "audio/opus": ".ogg", + "audio/webm": ".webm", + "audio/mp4": ".m4a", + "audio/x-m4a": ".m4a", + "application/octet-stream": ".wav", +} + +_AUDIO_RESPONSE_MIME_BY_SUFFIX = { + ".mp3": "audio/mpeg", + ".wav": "audio/wav", + ".ogg": "audio/ogg", + ".opus": "audio/ogg", + ".m4a": "audio/mp4", + ".mp4": "audio/mp4", + ".flac": "audio/flac", +} + +ESP32_VOICE_ANIMATION_PATH = "hermes_voice_animations.py" +ESP32_VOICE_ANIMATION_SCRIPT = "# Hermes ESP32 voice-state animations for MicroPython screen boards.\n# Uploaded by the API server; can also be run manually with:\n# exec(open('hermes_voice_animations.py').read()); run_state('ack')\nimport time\ntry:\n import board_config\nexcept Exception:\n board_config = None\n\n\ndef _display():\n return getattr(board_config, 'display_instance', None) if board_config else None\n\n\ndef _sleep(ms):\n try:\n time.sleep_ms(ms)\n except AttributeError:\n time.sleep(ms / 1000)\n\n\ndef _text(d, msg, x, y, scale=1, color=1):\n if hasattr(d, 'text_large') and scale > 1:\n try:\n d.text_large(msg, x, y, scale=scale, color=color)\n return\n except TypeError:\n pass\n d.text(msg, x, y, color)\n\n\ndef _center_text(d, msg, y, scale=1, color=1):\n width = getattr(d, 'width', 320)\n char_w = 8 * scale\n x = max(0, (width - len(msg) * char_w) // 2)\n _text(d, msg, x, y, scale=scale, color=color)\n\n\ndef _wrap(text, width_chars=34, max_lines=4):\n words = str(text or '').replace('\\n', ' ').split()\n lines, line = [], ''\n for w in words:\n candidate = (line + ' ' + w).strip()\n if len(candidate) > width_chars and line:\n lines.append(line)\n line = w\n if len(lines) >= max_lines:\n break\n else:\n line = candidate\n if line and len(lines) < max_lines:\n lines.append(line)\n return lines\n\n\ndef _spinner(d, title, subtitle='', frames=24, delay=70):\n width, height = getattr(d, 'width', 320), getattr(d, 'height', 240)\n cx, cy = width // 2, height // 2 - 8\n spokes = [(-22,0),(-16,-16),(0,-22),(16,-16),(22,0),(16,16),(0,22),(-16,16)]\n for frame in range(frames):\n d.clear(0)\n _center_text(d, title, 18, scale=2)\n for i, (dx, dy) in enumerate(spokes):\n color = 1 if (i + frame) % len(spokes) in (0, 1, 2, 3) else 0\n size = 4 if i == frame % len(spokes) else 2\n d.fill_rect(cx + dx - size // 2, cy + dy - size // 2, size, size, color)\n d.rect(8, height - 34, width - 16, 18, 1)\n d.fill_rect(10, height - 32, ((width - 20) * ((frame % 16) + 1)) // 16, 14, 1)\n if subtitle:\n _center_text(d, subtitle[:32], height - 56, scale=1)\n d.show()\n _sleep(delay)\n\n\ndef ack(frames=18):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'GOT IT', 18, scale=2)\n # expanding check pulse\n box = 48 + (i % 6) * 8\n x, y = width // 2 - box // 2, height // 2 - box // 2\n d.rect(x, y, box, box, 1)\n d.line(width//2 - 24, height//2, width//2 - 6, height//2 + 18, 1)\n d.line(width//2 - 6, height//2 + 18, width//2 + 30, height//2 - 22, 1)\n _center_text(d, 'Listening captured', height - 36, scale=1)\n d.show()\n _sleep(55)\n\n\ndef captioning(frames=36):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'CAPTIONING', 16, scale=2)\n for bar in range(9):\n h = 8 + ((i * 5 + bar * 13) % 42)\n x = width // 2 - 72 + bar * 18\n d.fill_rect(x, height // 2 - h // 2, 10, h, 1)\n dots = '.' * ((i // 5) % 4)\n _center_text(d, 'Transcribing' + dots, height - 42, scale=1)\n d.show()\n _sleep(65)\n\n\ndef waiting(text='', frames=44):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'THINKING', 12, scale=2)\n r = 18 + (i % 10) * 2\n cx, cy = width // 2, height // 2 - 10\n d.rect(cx - r, cy - r, r * 2, r * 2, 1)\n d.rect(cx - r//2, cy - r//2, r, r, 1)\n if text:\n d.text('Heard:', 10, height - 68, 1)\n for idx, line in enumerate(_wrap(text, 36, 3)):\n d.text(line[:38], 10, height - 54 + idx * 12, 1)\n else:\n _center_text(d, 'Waiting for reply...', height - 42, scale=1)\n d.show()\n _sleep(75)\n\n\ndef run_state(state='ack', text='', frames=None):\n state = str(state or 'ack').lower()\n if state in ('ack', 'listening', 'got_it'):\n ack(frames or 18)\n elif state in ('caption', 'captioning', 'transcribing'):\n captioning(frames or 36)\n elif state in ('wait', 'waiting', 'thinking', 'reply'):\n waiting(text, frames or 44)\n else:\n _spinner(_display(), 'HERMES', state[:28], frames or 20)\n" + + +def _coerce_port(value: Any, default: int = DEFAULT_PORT) -> int: + """Parse a listen port without letting malformed env/config values crash startup.""" + try: + return int(value) + except (TypeError, ValueError): + return default + + +_TRUE_REQUEST_BOOL_STRINGS = frozenset({"1", "true", "yes", "on"}) +_FALSE_REQUEST_BOOL_STRINGS = frozenset({"0", "false", "no", "off"}) + + +def _coerce_request_bool(value: Any, default: bool = False) -> bool: + """Normalize boolean-like API payload values. + + External clients should send real JSON booleans, but some OpenAI-compatible + frontends and middleware serialize flags like ``stream`` as strings. Using + Python truthiness on those values misroutes requests because ``"false"`` is + still truthy. Treat only explicit bool-ish scalars as booleans; everything + else falls back to the caller's default. + """ + if isinstance(value, bool): + return value + if value is None: + return default + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in _TRUE_REQUEST_BOOL_STRINGS: + return True + if normalized in _FALSE_REQUEST_BOOL_STRINGS: + return False + return default + if isinstance(value, (int, float)): + return bool(value) + return default + + +def _normalize_chat_content( + content: Any, *, _max_depth: int = 10, _depth: int = 0, +) -> str: + """Normalize OpenAI chat message content into a plain text string. + + Some clients (Open WebUI, LobeChat, etc.) send content as an array of + typed parts instead of a plain string:: + + [{"type": "text", "text": "hello"}, {"type": "input_text", "text": "..."}] + + This function flattens those into a single string so the agent pipeline + (which expects strings) doesn't choke. + + Defensive limits prevent abuse: recursion depth, list size, and output + length are all bounded. + """ + if _depth > _max_depth: + return "" + if content is None: + return "" + if isinstance(content, str): + return content[:MAX_NORMALIZED_TEXT_LENGTH] if len(content) > MAX_NORMALIZED_TEXT_LENGTH else content + + if isinstance(content, list): + parts: List[str] = [] + total_len = 0 + items = content[:MAX_CONTENT_LIST_SIZE] if len(content) > MAX_CONTENT_LIST_SIZE else content + for item in items: + if isinstance(item, str): + if item: + part = item[:MAX_NORMALIZED_TEXT_LENGTH] + parts.append(part) + total_len += len(part) + elif isinstance(item, dict): + item_type = str(item.get("type") or "").strip().lower() + if item_type in {"text", "input_text", "output_text"}: + text = item.get("text", "") + if text: + try: + part = str(text)[:MAX_NORMALIZED_TEXT_LENGTH] + parts.append(part) + total_len += len(part) + except Exception: + pass + # Silently skip image_url / other non-text parts + elif isinstance(item, list): + nested = _normalize_chat_content(item, _max_depth=_max_depth, _depth=_depth + 1) + if nested: + parts.append(nested) + total_len += len(nested) + # Check accumulated size + if total_len >= MAX_NORMALIZED_TEXT_LENGTH: + break + result = "\n".join(parts) + return result[:MAX_NORMALIZED_TEXT_LENGTH] if len(result) > MAX_NORMALIZED_TEXT_LENGTH else result + + # Fallback for unexpected types (int, float, bool, etc.) + try: + result = str(content) + return result[:MAX_NORMALIZED_TEXT_LENGTH] if len(result) > MAX_NORMALIZED_TEXT_LENGTH else result + except Exception: + return "" + + +# Content part type aliases used by the OpenAI Chat Completions and Responses +# APIs. We accept both spellings on input and emit a single canonical internal +# shape (``{"type": "text", ...}`` / ``{"type": "image_url", ...}``) that the +# rest of the agent pipeline already understands. +_TEXT_PART_TYPES = frozenset({"text", "input_text", "output_text"}) +_IMAGE_PART_TYPES = frozenset({"image_url", "input_image"}) +_FILE_PART_TYPES = frozenset({"file", "input_file"}) + + +def _normalize_multimodal_content(content: Any) -> Any: + """Validate and normalize multimodal content for the API server. + + Returns a plain string when the content is text-only, or a list of + ``{"type": "text"|"image_url", ...}`` parts when images are present. + The output shape is the native OpenAI Chat Completions vision format, + which the agent pipeline accepts verbatim (OpenAI-wire providers) or + converts (``_preprocess_anthropic_content`` for Anthropic). + + Raises ``ValueError`` with an OpenAI-style code on invalid input: + * ``unsupported_content_type`` — file/input_file/file_id parts, or + non-image ``data:`` URLs. + * ``invalid_image_url`` — missing URL or unsupported scheme. + * ``invalid_content_part`` — malformed text/image objects. + + Callers translate the ValueError into a 400 response. + """ + # Scalar passthrough mirrors ``_normalize_chat_content``. + if content is None: + return "" + if isinstance(content, str): + return content[:MAX_NORMALIZED_TEXT_LENGTH] if len(content) > MAX_NORMALIZED_TEXT_LENGTH else content + if not isinstance(content, list): + # Mirror the legacy text-normalizer's fallback so callers that + # pre-existed image support still get a string back. + return _normalize_chat_content(content) + + items = content[:MAX_CONTENT_LIST_SIZE] if len(content) > MAX_CONTENT_LIST_SIZE else content + normalized_parts: List[Dict[str, Any]] = [] + text_accum_len = 0 + + for part in items: + if isinstance(part, str): + if part: + trimmed = part[:MAX_NORMALIZED_TEXT_LENGTH] + normalized_parts.append({"type": "text", "text": trimmed}) + text_accum_len += len(trimmed) + continue + + if not isinstance(part, dict): + # Ignore unknown scalars for forward compatibility with future + # Responses API additions (e.g. ``refusal``). The same policy + # the text normalizer applies. + continue + + raw_type = part.get("type") + part_type = str(raw_type or "").strip().lower() + + if part_type in _TEXT_PART_TYPES: + text = part.get("text") + if text is None: + continue + if not isinstance(text, str): + text = str(text) + if text: + trimmed = text[:MAX_NORMALIZED_TEXT_LENGTH] + normalized_parts.append({"type": "text", "text": trimmed}) + text_accum_len += len(trimmed) + continue + + if part_type in _IMAGE_PART_TYPES: + detail = part.get("detail") + image_ref = part.get("image_url") + # OpenAI Responses sends ``input_image`` with a top-level + # ``image_url`` string; Chat Completions sends ``image_url`` as + # ``{"url": "...", "detail": "..."}``. Support both. + if isinstance(image_ref, dict): + url_value = image_ref.get("url") + detail = image_ref.get("detail", detail) + else: + url_value = image_ref + if not isinstance(url_value, str) or not url_value.strip(): + raise ValueError("invalid_image_url:Image parts must include a non-empty image URL.") + url_value = url_value.strip() + lowered = url_value.lower() + if lowered.startswith("data:"): + if not lowered.startswith("data:image/") or "," not in url_value: + raise ValueError( + "unsupported_content_type:Only image data URLs are supported. " + "Non-image data payloads are not supported." + ) + elif not (lowered.startswith("http://") or lowered.startswith("https://")): + raise ValueError( + "invalid_image_url:Image inputs must use http(s) URLs or data:image/... URLs." + ) + image_part: Dict[str, Any] = {"type": "image_url", "image_url": {"url": url_value}} + if detail is not None: + if not isinstance(detail, str) or not detail.strip(): + raise ValueError("invalid_content_part:Image detail must be a non-empty string when provided.") + image_part["image_url"]["detail"] = detail.strip() + normalized_parts.append(image_part) + continue + + if part_type in _FILE_PART_TYPES: + raise ValueError( + "unsupported_content_type:Inline image inputs are supported, " + "but uploaded files and document inputs are not supported on this endpoint." + ) + + # Unknown part type — reject explicitly so clients get a clear error + # instead of a silently dropped turn. + raise ValueError( + f"unsupported_content_type:Unsupported content part type {raw_type!r}. " + "Only text and image_url/input_image parts are supported." + ) + + if not normalized_parts: + return "" + + # Text-only: collapse to a plain string so downstream logging/trajectory + # code sees the native shape and prompt caching on text-only turns is + # unaffected. + if all(p.get("type") == "text" for p in normalized_parts): + return "\n".join(p["text"] for p in normalized_parts if p.get("text")) + + return normalized_parts + + +def _content_has_visible_payload(content: Any) -> bool: + """True when content has any text or image attachment. Used to reject empty turns.""" + if isinstance(content, str): + return bool(content.strip()) + if isinstance(content, list): + for part in content: + if isinstance(part, dict): + ptype = str(part.get("type") or "").strip().lower() + if ptype in _TEXT_PART_TYPES and str(part.get("text") or "").strip(): + return True + if ptype in _IMAGE_PART_TYPES: + return True + return False + + +def _multimodal_validation_error(exc: ValueError, *, param: str) -> "web.Response": + """Translate a ``_normalize_multimodal_content`` ValueError into a 400 response.""" + raw = str(exc) + code, _, message = raw.partition(":") + if not message: + code, message = "invalid_content_part", raw + return web.json_response( + _openai_error(message, code=code, param=param), + status=400, + ) + + +def _session_chat_user_message(body: Dict[str, Any], *, param: str = "message") -> tuple[Any, Optional["web.Response"]]: + """Parse and normalize session chat ``message`` / ``input`` like chat completions.""" + user_message = body.get("message") or body.get("input") + if not _content_has_visible_payload(user_message): + return None, web.json_response( + _openai_error("Missing 'message' field", code="missing_message"), + status=400, + ) + try: + return _normalize_multimodal_content(user_message), None + except ValueError as exc: + return None, _multimodal_validation_error(exc, param=param) + + +def check_api_server_requirements() -> bool: + """Check if API server dependencies are available.""" + return AIOHTTP_AVAILABLE + + +class ResponseStore: + """ + SQLite-backed LRU store for Responses API state. + + Each stored response includes the full internal conversation history + (with tool calls and results) so it can be reconstructed on subsequent + requests via previous_response_id. + + Persists across gateway restarts. Falls back to in-memory SQLite + if the on-disk path is unavailable. + """ + + def __init__(self, max_size: int = MAX_STORED_RESPONSES, db_path: str = None): + self._max_size = max_size + if db_path is None: + try: + from hermes_cli.config import get_hermes_home + db_path = str(get_hermes_home() / "response_store.db") + except Exception: + db_path = ":memory:" + self._db_path: Optional[str] = db_path if db_path != ":memory:" else None + try: + self._conn = sqlite3.connect(db_path, check_same_thread=False) + except Exception: + self._conn = sqlite3.connect(":memory:", check_same_thread=False) + self._db_path = None + # Use shared WAL-fallback helper so response_store.db degrades + # gracefully on NFS/SMB/FUSE-mounted HERMES_HOME (same filesystem + # issue addressed for state.db/kanban.db — see + # hermes_state._WAL_INCOMPAT_MARKERS). + from hermes_state import apply_wal_with_fallback + apply_wal_with_fallback(self._conn, db_label="response_store.db") + self._conn.execute( + """CREATE TABLE IF NOT EXISTS responses ( + response_id TEXT PRIMARY KEY, + data TEXT NOT NULL, + accessed_at REAL NOT NULL + )""" + ) + self._conn.execute( + """CREATE TABLE IF NOT EXISTS conversations ( + name TEXT PRIMARY KEY, + response_id TEXT NOT NULL + )""" + ) + self._conn.commit() + # response_store.db contains conversation history (tool payloads, + # prompts, results). Tighten to owner-only after creation so other + # local users on a shared box can't read it. Run once at __init__ + # rather than after every commit — chmod-on-every-write is wasted + # syscalls on a hot path. + self._tighten_file_permissions() + + def _tighten_file_permissions(self) -> None: + """Force owner-only permissions on the DB and SQLite sidecars.""" + if not self._db_path: + return + for candidate in ( + Path(self._db_path), + Path(f"{self._db_path}-wal"), + Path(f"{self._db_path}-shm"), + ): + try: + if candidate.exists(): + candidate.chmod(0o600) + except OSError: + logger.debug( + "Failed to restrict response store permissions for %s", + candidate, + exc_info=True, + ) + + def get(self, response_id: str) -> Optional[Dict[str, Any]]: + """Retrieve a stored response by ID (updates access time for LRU).""" + row = self._conn.execute( + "SELECT data FROM responses WHERE response_id = ?", (response_id,) + ).fetchone() + if row is None: + return None + self._conn.execute( + "UPDATE responses SET accessed_at = ? WHERE response_id = ?", + (time.time(), response_id), + ) + self._conn.commit() + try: + return json.loads(row[0]) + except (json.JSONDecodeError, TypeError): + logger.warning( + "Corrupted JSON in response store for id=%s, evicting entry", + response_id, + ) + self._conn.execute( + "DELETE FROM responses WHERE response_id = ?", + (response_id,), + ) + self._conn.commit() + return None + + def put(self, response_id: str, data: Dict[str, Any]) -> None: + """Store a response, evicting the oldest if at capacity.""" + self._conn.execute( + "INSERT OR REPLACE INTO responses (response_id, data, accessed_at) VALUES (?, ?, ?)", + (response_id, json.dumps(data, default=str), time.time()), + ) + # Evict oldest entries beyond max_size + count = self._conn.execute("SELECT COUNT(*) FROM responses").fetchone()[0] + if count > self._max_size: + # Collect IDs that will be evicted + evict_ids = [ + row[0] + for row in self._conn.execute( + "SELECT response_id FROM responses ORDER BY accessed_at ASC LIMIT ?", + (count - self._max_size,), + ).fetchall() + ] + if evict_ids: + placeholders = ",".join("?" for _ in evict_ids) + # Clear conversation mappings pointing to evicted responses + self._conn.execute( + f"DELETE FROM conversations WHERE response_id IN ({placeholders})", + evict_ids, + ) + # Delete evicted responses + self._conn.execute( + f"DELETE FROM responses WHERE response_id IN ({placeholders})", + evict_ids, + ) + self._conn.commit() + + def delete(self, response_id: str) -> bool: + """Remove a response from the store. Returns True if found and deleted.""" + # Clear conversation mappings pointing to this response + self._conn.execute( + "DELETE FROM conversations WHERE response_id = ?", (response_id,) + ) + cursor = self._conn.execute( + "DELETE FROM responses WHERE response_id = ?", (response_id,) + ) + self._conn.commit() + return cursor.rowcount > 0 + + def get_conversation(self, name: str) -> Optional[str]: + """Get the latest response_id for a conversation name.""" + row = self._conn.execute( + "SELECT response_id FROM conversations WHERE name = ?", (name,) + ).fetchone() + return row[0] if row else None + + def set_conversation(self, name: str, response_id: str) -> None: + """Map a conversation name to its latest response_id.""" + self._conn.execute( + "INSERT OR REPLACE INTO conversations (name, response_id) VALUES (?, ?)", + (name, response_id), + ) + self._conn.commit() + + def close(self) -> None: + """Close the database connection.""" + try: + self._conn.close() + except Exception: + pass + + def __len__(self) -> int: + row = self._conn.execute("SELECT COUNT(*) FROM responses").fetchone() + return row[0] if row else 0 + + +# --------------------------------------------------------------------------- +# CORS middleware +# --------------------------------------------------------------------------- + +_CORS_HEADERS = { + "Access-Control-Allow-Methods": "GET, POST, DELETE, OPTIONS", + "Access-Control-Allow-Headers": "Authorization, Content-Type, Idempotency-Key", +} + + +if AIOHTTP_AVAILABLE: + @web.middleware + async def cors_middleware(request, handler): + """Add CORS headers for explicitly allowed origins; handle OPTIONS preflight.""" + adapter = request.app.get("api_server_adapter") + origin = request.headers.get("Origin", "") + cors_headers = None + if adapter is not None: + if not adapter._origin_allowed(origin): + return web.Response(status=403) + cors_headers = adapter._cors_headers_for_origin(origin) + + if request.method == "OPTIONS": + if cors_headers is None: + return web.Response(status=403) + return web.Response(status=200, headers=cors_headers) + + response = await handler(request) + if cors_headers is not None: + response.headers.update(cors_headers) + return response +else: + cors_middleware = None # type: ignore[assignment] + + +_MEDIA_IMG_EXT = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp"} +_MEDIA_MIME = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".bmp": "image/bmp", +} +_MEDIA_DATA_URL_MAX_BYTES = 5 * 1024 * 1024 # skip images larger than 5MB + + +def _resolve_media_to_data_urls(text: str) -> str: + """Replace ``MEDIA:`` image tags with inline base64 data URLs. + + Remote OpenAI-compatible frontends can't read local file paths, so + ``MEDIA:`` tags referencing images on the server are useless to them. + Inline small local images as markdown data URLs; non-image or unreadable + paths are left untouched. + + Uses the same anchored ``MEDIA_TAG_CLEANUP_RE`` matcher and + ``validate_media_delivery_path`` safety check every other platform + adapter's media delivery already goes through (gateway/platforms/base.py) + — an absolute-path anchor plus a known-extension requirement, and a + resolved-path check against the credential/system-path denylist. The + prior pattern here matched any bare token after ``MEDIA:`` (including a + relative/traversal path like ``../../etc/passwd.png``) and read the file + directly with no denylist, so any image-suffixed, readable file the + process could see was base64-exfiltrated to the API caller if its path + merely appeared in the model's own final reply text. + """ + if not text or "MEDIA:" not in text: + return text + import base64 + + def _to_data_url(path_str: str) -> Optional[str]: + # validate_media_delivery_path() strips wrapping quotes/backticks + # and trailing punctuation internally, same as MEDIA_TAG_CLEANUP_RE's + # other callers (extract_media / _strip_media_tag_directives) rely on. + safe_path = validate_media_delivery_path(path_str) + if not safe_path: + return None + p = Path(safe_path) + suffix = p.suffix.lower() + if suffix not in _MEDIA_IMG_EXT: + return None + try: + if p.stat().st_size > _MEDIA_DATA_URL_MAX_BYTES: + return None + b64 = base64.b64encode(p.read_bytes()).decode() + except OSError: + return None + return f"![image](data:{_MEDIA_MIME[suffix]};base64,{b64})" + + def _repl(m: "re.Match[str]") -> str: + return _to_data_url(m.group("path")) or m.group(0) + + try: + return MEDIA_TAG_CLEANUP_RE.sub(_repl, text) + except Exception: + return text + + +def _redact_api_error_text(value: Any, *, limit: int | None = None) -> str: + """Redact API-bound error text before it crosses the HTTP boundary.""" + redacted = redact_sensitive_text(str(value), force=True) + if limit is not None: + return redacted[:limit] + return redacted + + +def _openai_error(message: str, err_type: str = "invalid_request_error", param: str = None, code: str = None) -> Dict[str, Any]: + """OpenAI-style error envelope.""" + return { + "error": { + "message": _redact_api_error_text(message), + "type": err_type, + "param": param, + "code": code, + } + } + + +_api_agent_request_reservation: ContextVar[Optional[dict[str, bool]]] = ContextVar( + "api_agent_request_reservation", default=None +) + + +def _admit_api_agent_request(handler): + """Reserve an authenticated API turn before its handler first awaits. + + Gateway shutdown and aiohttp requests share an event loop. Keeping the + drain check and reservation in one non-awaiting block prevents a request + admitted immediately before shutdown from becoming invisible while it is + still parsing its body or resolving session state. The mutable reservation + is intentionally shared with child tasks so agent/task bookkeeping releases + this one slot exactly once. + """ + @wraps(handler) + async def _wrapped(self, request, *args, **kwargs): + auth_err = self._check_auth(request) + if auth_err: + return auth_err + draining = self._draining_response() + if draining is not None: + return draining + reservation = {"active": True} + token = _api_agent_request_reservation.set(reservation) + self._pending_agent_requests += 1 + try: + return await handler(self, request, *args, **kwargs) + finally: + if reservation["active"]: + reservation["active"] = False + self._pending_agent_requests = max(0, self._pending_agent_requests - 1) + _api_agent_request_reservation.reset(token) + + return _wrapped + + +def _release_pending_api_work(adapter, reservation: dict[str, bool]) -> None: + """Release a pending-work reservation exactly once.""" + if reservation["active"]: + reservation["active"] = False + adapter._pending_agent_requests = max(0, adapter._pending_agent_requests - 1) + + +@contextmanager +def _reserve_pending_api_work(adapter): + """Keep externally-triggered background work visible across awaits. + + A handler can detach the reservation to an asyncio task; its done callback + then owns release so shutdown cannot miss the handoff to background work. + """ + reservation = {"active": True, "detached": False} + adapter._pending_agent_requests += 1 + try: + yield reservation + finally: + if not reservation["detached"]: + _release_pending_api_work(adapter, reservation) + + +if AIOHTTP_AVAILABLE: + @web.middleware + async def body_limit_middleware(request, handler): + """Reject overly large request bodies early based on Content-Length.""" + if request.method in {"POST", "PUT", "PATCH"}: + cl = request.headers.get("Content-Length") + if cl is not None: + try: + if int(cl) > MAX_REQUEST_BYTES: + return web.json_response(_openai_error("Request body too large.", code="body_too_large"), status=413) + except ValueError: + return web.json_response(_openai_error("Invalid Content-Length header.", code="invalid_content_length"), status=400) + try: + return await handler(request) + except web.HTTPRequestEntityTooLarge: + # aiohttp's client_max_size tripped mid-read (chunked bodies carry + # no Content-Length) — return a proper 413 instead of letting the + # handler's broad JSON except turn it into 400 "Invalid JSON". + return web.json_response( + _openai_error("Request body too large.", code="body_too_large"), + status=413, + ) +else: + body_limit_middleware = None # type: ignore[assignment] + +_SECURITY_HEADERS = { + "Content-Security-Policy": "default-src 'none'; frame-ancestors 'none'", + "Permissions-Policy": "camera=(), microphone=(), geolocation=()", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains", + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "DENY", + "X-XSS-Protection": "0", + "Referrer-Policy": "no-referrer", +} + + +if AIOHTTP_AVAILABLE: + @web.middleware + async def security_headers_middleware(request, handler): + """Add security headers to all responses (including errors).""" + response = await handler(request) + for k, v in _SECURITY_HEADERS.items(): + response.headers.setdefault(k, v) + return response +else: + security_headers_middleware = None # type: ignore[assignment] + + +class _IdempotencyCache: + """In-memory idempotency cache with TTL and basic LRU semantics.""" + def __init__(self, max_items: int = 1000, ttl_seconds: int = 300): + from collections import OrderedDict + self._store = OrderedDict() + self._inflight: Dict[tuple[str, str], "asyncio.Task[Any]"] = {} + self._ttl = ttl_seconds + self._max = max_items + + def _purge(self): + now = time.time() + expired = [k for k, v in self._store.items() if now - v["ts"] > self._ttl] + for k in expired: + self._store.pop(k, None) + while len(self._store) > self._max: + self._store.popitem(last=False) + + async def get_or_set(self, key: str, fingerprint: str, compute_coro): + self._purge() + item = self._store.get(key) + if item and item["fp"] == fingerprint: + return item["resp"] + + inflight_key = (key, fingerprint) + task = self._inflight.get(inflight_key) + if task is None: + async def _compute_and_store(): + resp = await compute_coro() + import time as _t + self._store[key] = {"resp": resp, "fp": fingerprint, "ts": _t.time()} + self._purge() + return resp + + task = asyncio.create_task(_compute_and_store()) + self._inflight[inflight_key] = task + + def _clear_inflight(done_task: "asyncio.Task[Any]") -> None: + if self._inflight.get(inflight_key) is done_task: + self._inflight.pop(inflight_key, None) + + task.add_done_callback(_clear_inflight) + + return await asyncio.shield(task) + + +_idem_cache = _IdempotencyCache() + + +def _make_request_fingerprint(body: Dict[str, Any], keys: List[str]) -> str: + from hashlib import sha256 + subset = {k: body.get(k) for k in keys} + return sha256(repr(subset).encode("utf-8")).hexdigest() + + +def _derive_chat_session_id( + system_prompt: Optional[str], + first_user_message: str, +) -> str: + """Derive a stable session ID from the conversation's first user message. + + OpenAI-compatible frontends (Open WebUI, LibreChat, etc.) send the full + conversation history with every request. The system prompt and first user + message are constant across all turns of the same conversation, so hashing + them produces a deterministic session ID that lets the API server reuse + the same Hermes session (and therefore the same Docker container sandbox + directory) across turns. + """ + seed = f"{system_prompt or ''}\n{first_user_message}" + digest = hashlib.sha256(seed.encode("utf-8")).hexdigest()[:16] + return f"api-{digest}" + + +_CRON_AVAILABLE = False +try: + from cron.jobs import ( + list_jobs as _cron_list, + get_job as _cron_get, + create_job as _cron_create, + update_job as _cron_update, + remove_job as _cron_remove, + pause_job as _cron_pause, + resume_job as _cron_resume, + trigger_job as _cron_trigger, + ) + _CRON_AVAILABLE = True +except ImportError: + _cron_list = None + _cron_get = None + _cron_create = None + _cron_update = None + _cron_remove = None + _cron_pause = None + _cron_resume = None + _cron_trigger = None + + +def _notify_cron_provider_jobs_changed() -> None: + """Tell the active cron scheduler provider the job set changed after a REST + mutation (no-op for the built-in). Best-effort — never breaks the handler.""" + try: + from cron.scheduler import _notify_provider_jobs_changed + _notify_provider_jobs_changed() + except Exception: + pass + +# Defense-in-depth: mirror the agent-facing cronjob tool, which scans the +# user-supplied prompt for exfiltration/injection payloads at create/update +# time (tools/cronjob_tools.py). The REST cron endpoints are authenticated +# (every handler runs _check_auth, and connect() refuses to start without +# API_SERVER_KEY), so this is not the trust boundary — it's parity with the +# tool path so a malicious prompt is rejected the same way regardless of +# which surface created the job. Imported defensively: a missing scanner +# must not disable the cron REST API. +try: + from tools.cronjob_tools import _scan_cron_prompt as _scan_cron_prompt +except Exception: # pragma: no cover - scanner is optional hardening + _scan_cron_prompt = None + + +class APIServerAdapter(BasePlatformAdapter): + """ + OpenAI-compatible HTTP API server adapter. + + Runs an aiohttp web server that accepts OpenAI-format requests + and routes them through hermes-agent's AIAgent. + """ + + # Stateless request/response: every route (the OpenAI-spec + # /v1/chat/completions and /v1/responses, and the proprietary /v1/runs SSE + # stream) tears down its channel when the turn ends. There is no persistent + # outbound channel to push a background completion to a client that already + # received its response, and ``send()`` is a no-op stub. So async-delivery + # tools (terminal notify_on_complete / watch_patterns, delegate_task + # background=True) must NOT promise delivery on this path — see + # ``async_delivery_supported()``. + supports_async_delivery: bool = False + + def __init__(self, config: PlatformConfig): + super().__init__(config, Platform.API_SERVER) + extra = config.extra or {} + self._host: str = extra.get("host", os.getenv("API_SERVER_HOST", DEFAULT_HOST)) + raw_port = extra.get("port") + if raw_port is None: + raw_port = os.getenv("API_SERVER_PORT", str(DEFAULT_PORT)) + self._port: int = _coerce_port(raw_port, DEFAULT_PORT) + self._api_key: str = extra.get("key", os.getenv("API_SERVER_KEY", "")) + self._cors_origins: tuple[str, ...] = self._parse_cors_origins( + extra.get("cors_origins", os.getenv("API_SERVER_CORS_ORIGINS", "")), + ) + self._model_name: str = self._resolve_model_name( + extra.get("model_name", os.getenv("API_SERVER_MODEL_NAME", "")), + ) + # model_routes: maps incoming ``model`` field values to specific + # provider/model configs so one API server instance can serve + # multiple clients on different backends. + # + # Config format (platforms.api_server.extra in the gateway config): + # model_routes: + # minimax-m2: # alias the client sends as the "model" field + # model: "minimax/minimax-m1" + # provider: "openrouter" # optional — resolved via the provider + # # credential chain when set + # api_key: "sk-…" # optional — per-route UPSTREAM provider + # # key override (NOT caller auth; never logged) + # base_url: "https://…" # optional — per-route base URL override + self._model_routes: Dict[str, Dict[str, Any]] = self._parse_model_routes( + extra.get("model_routes"), + ) + self._app: Optional["web.Application"] = None + self._runner: Optional["web.AppRunner"] = None + self._site: Optional["web.TCPSite"] = None + self._response_store = ResponseStore() + # Active run streams: run_id -> asyncio.Queue of SSE event dicts + self._run_streams: Dict[str, "asyncio.Queue[Optional[Dict]]"] = {} + # Creation timestamps for orphaned-run TTL sweep + self._run_streams_created: Dict[str, float] = {} + # Runs with a connected SSE consumer; their queue is actively draining. + self._run_stream_subscribers: set[str] = set() + # Active run agent/task references for stop support + self._active_run_agents: Dict[str, Any] = {} + self._active_run_tasks: Dict[str, "asyncio.Task"] = {} + # Stop is cooperative: the executor thread may outlive the HTTP request. + self._stopping_run_ids: set[str] = set() + # Pollable run status for dashboards and external control-plane UIs. + self._run_statuses: Dict[str, Dict[str, Any]] = {} + # Active approval session key for each run_id. The approval core + # resolves requests by session key, while API clients address the + # in-flight run by run_id. + self._run_approval_sessions: Dict[str, str] = {} + self._session_db: Optional[Any] = None # Lazy-init SessionDB for session continuity + # Concurrency cap shared across all agent-serving endpoints + # (/v1/chat/completions, /v1/responses, /v1/runs). Read from + # config.yaml gateway.api_server.max_concurrent_runs; 0 disables + # the cap. Bounds CPU / memory / upstream-LLM-quota exhaustion + # from a request flood (#7483). + self._max_concurrent_runs: int = self._resolve_max_concurrent_runs() + # Number of in-flight runs on the non-streaming chat/responses paths + # (the /v1/runs path tracks its own in-flight set via + # _active_run_tasks). + self._inflight_agent_runs: int = 0 + # Requests admitted before their handler reaches agent bookkeeping. + # Shutdown counts this reservation so the request cannot slip through + # the drain between its first await and _run_agent()/task registration. + self._pending_agent_requests: int = 0 + + def active_agent_work_count(self) -> int: + """Return all live agent work owned by this API adapter. + + ``/v1/runs`` registers an asyncio task before it constructs and stores + its agent, so ``_active_run_agents`` has a real queued-before-agent gap. + Reuse the task-based accounting used by the concurrent-run limit: it + covers that gap and excludes completed tasks retained until cleanup. + """ + try: + return ( + int(getattr(self, "_pending_agent_requests", 0)) + + int(self._inflight_agent_runs) + + sum(not task.done() for task in self._active_run_tasks.values()) + ) + except Exception: + return 0 + + @staticmethod + def _gateway_is_draining() -> bool: + """Whether the owning gateway currently refuses new agent turns.""" + try: + from gateway.run import _gateway_runner_ref + + runner = _gateway_runner_ref() + return bool( + runner + and ( + getattr(runner, "_draining", False) + or getattr(runner, "_external_drain_active", False) + ) + ) + except Exception: + return False + + def _draining_response(self) -> Optional["web.Response"]: + """Return a retryable response while the gateway drains existing work.""" + if not self._gateway_is_draining(): + return None + return web.json_response( + _openai_error( + "Gateway is draining existing work; retry shortly.", + code="gateway_draining", + ), + status=503, + headers={"Retry-After": "1"}, + ) + + def _activate_admitted_request(self) -> None: + """Transfer this request's drain reservation to agent bookkeeping.""" + reservation = _api_agent_request_reservation.get() + if reservation and reservation["active"]: + reservation["active"] = False + self._pending_agent_requests = max(0, self._pending_agent_requests - 1) + + def _readiness_work_counts(self) -> tuple[int, int, int]: + """Return bounded work counts from each subsystem's public state.""" + active_api_runs = sum( + 1 + for status in self._run_statuses.values() + if status.get("status") in {"queued", "running", "waiting_for_approval"} + ) + process_depth = 0 + active_delegations = 0 + try: + from tools.process_registry import process_registry + + process_depth = process_registry.completion_queue.qsize() + except Exception: + pass + try: + from tools.async_delegation import active_count + + active_delegations = active_count() + except Exception: + pass + return active_api_runs, process_depth, active_delegations + + @staticmethod + def _parse_cors_origins(value: Any) -> tuple[str, ...]: + """Normalize configured CORS origins into a stable tuple.""" + if not value: + return () + + if isinstance(value, str): + items = value.split(",") + elif isinstance(value, (list, tuple, set)): + items = value + else: + items = [str(value)] + + return tuple(str(item).strip() for item in items if str(item).strip()) + + @staticmethod + def _resolve_max_concurrent_runs() -> int: + """Read the concurrent-run cap from config.yaml (0 disables). + + gateway.api_server.max_concurrent_runs. Falls back to the historical + default of 10 when unset or malformed. Negative values are clamped + to 0 (disabled). + """ + default = 10 + try: + from hermes_cli.config import cfg_get, load_config + + raw = cfg_get( + load_config(), + "gateway", + "api_server", + "max_concurrent_runs", + default=default, + ) + value = int(raw) + except Exception: + return default + return max(0, value) + + @staticmethod + def _resolve_model_name(explicit: str) -> str: + """Derive the advertised model name for /v1/models. + + Priority: + 1. Explicit override (config extra or API_SERVER_MODEL_NAME env var) + 2. Active profile name (so each profile advertises a distinct model) + 3. Fallback: "hermes-agent" + """ + if explicit and explicit.strip(): + return explicit.strip() + try: + from hermes_cli.profiles import get_active_profile_name + profile = get_active_profile_name() + if profile and profile not in {"default", "custom"}: + return profile + except Exception: + pass + return "hermes-agent" + + def _cors_headers_for_origin(self, origin: str) -> Optional[Dict[str, str]]: + """Return CORS headers for an allowed browser origin.""" + if not origin or not self._cors_origins: + return None + + if "*" in self._cors_origins: + headers = dict(_CORS_HEADERS) + headers["Access-Control-Allow-Origin"] = "*" + headers["Access-Control-Max-Age"] = "600" + return headers + + if origin not in self._cors_origins: + return None + + headers = dict(_CORS_HEADERS) + headers["Access-Control-Allow-Origin"] = origin + headers["Vary"] = "Origin" + headers["Access-Control-Max-Age"] = "600" + return headers + + def _origin_allowed(self, origin: str) -> bool: + """Allow non-browser clients and explicitly configured browser origins.""" + if not origin: + return True + + if not self._cors_origins: + return False + + return "*" in self._cors_origins or origin in self._cors_origins + + @staticmethod + def _clean_log_value(value: Any, *, max_len: int = 200) -> str: + """Sanitize request metadata before it reaches security logs.""" + if value is None: + return "" + text = str(value).replace("\r", " ").replace("\n", " ").strip() + return text[:max_len] + + def _request_audit_context(self, request: "web.Request") -> Dict[str, str]: + """Return non-secret source metadata for security/audit warnings.""" + peer_ip = "" + try: + peer = request.transport.get_extra_info("peername") if request.transport else None + if isinstance(peer, (tuple, list)) and peer: + peer_ip = str(peer[0]) + except Exception: + peer_ip = "" + + return { + "remote": self._clean_log_value(getattr(request, "remote", "") or peer_ip), + "peer_ip": self._clean_log_value(peer_ip), + "forwarded_for": self._clean_log_value(request.headers.get("X-Forwarded-For", "")), + "real_ip": self._clean_log_value(request.headers.get("X-Real-IP", "")), + "method": self._clean_log_value(request.method, max_len=16), + "path": self._clean_log_value(request.path_qs, max_len=500), + "user_agent": self._clean_log_value(request.headers.get("User-Agent", ""), max_len=300), + } + + def _request_audit_log_suffix(self, request: "web.Request") -> str: + ctx = self._request_audit_context(request) + fields = [f"{key}={value!r}" for key, value in ctx.items() if value] + return " ".join(fields) if fields else "source='unknown'" + + def _cron_origin_from_request(self, request: "web.Request") -> Dict[str, str]: + """Persist safe API source metadata on cron jobs created over HTTP.""" + ctx = self._request_audit_context(request) + origin = { + "platform": "api_server", + "chat_id": "api", + } + if ctx.get("remote"): + origin["source_ip"] = ctx["remote"] + if ctx.get("peer_ip"): + origin["peer_ip"] = ctx["peer_ip"] + if ctx.get("forwarded_for"): + origin["forwarded_for"] = ctx["forwarded_for"] + if ctx.get("real_ip"): + origin["real_ip"] = ctx["real_ip"] + if ctx.get("user_agent"): + origin["user_agent"] = ctx["user_agent"] + return origin + + # ------------------------------------------------------------------ + # Auth helper + # ------------------------------------------------------------------ + + def _check_auth(self, request: "web.Request") -> Optional["web.Response"]: + """ + Validate Bearer token from Authorization header. + + Returns None if auth is OK, or a 401 web.Response on failure. + connect() refuses to start the API server without API_SERVER_KEY, so + the no-key branch only exists for tests or unsupported manual wiring. + """ + if not self._api_key: + return None + + auth_header = request.headers.get("Authorization", "") + if auth_header.startswith("Bearer "): + token = auth_header[7:].strip() + if hmac.compare_digest(token, self._api_key): + return None # Auth OK + + logger.warning( + "API server rejected invalid API key: %s", + self._request_audit_log_suffix(request), + ) + return web.json_response( + {"error": {"message": "Invalid API key", "type": "invalid_request_error", "code": "invalid_api_key"}}, + status=401, + ) + + # ------------------------------------------------------------------ + # Session header helpers + # ------------------------------------------------------------------ + + # Soft length cap for session identifiers. Headers are bounded in + # aggregate by aiohttp (``client_max_size`` / default 8 KiB per + # header), but we impose a tighter limit on the session headers so a + # caller can't burn memory by passing a multi-kilobyte "session key". + # 256 chars is well above any realistic stable channel identifier + # (e.g. ``agent:main:webui:dm:user-42``) while staying small enough + # that the sanitized form is safe to pass into Honcho / state.db. + _MAX_SESSION_HEADER_LEN = 256 + + def _parse_session_key_header( + self, request: "web.Request" + ) -> tuple[Optional[str], Optional["web.Response"]]: + """Extract and validate the ``X-Hermes-Session-Key`` header. + + The session key is a stable per-channel identifier that scopes + long-term memory (e.g. Honcho sessions) across transcripts. It + is independent of ``X-Hermes-Session-Id``: callers may send + either, both, or neither. + + Returns ``(session_key, None)`` on success (with an empty/absent + header yielding ``None`` for the key), or ``(None, error_response)`` + on validation failure. + + Security: like session continuation, accepting a caller-supplied + memory scope requires API-key authentication so that an + unauthenticated client on a local-only server can't inject itself + into another user's long-term memory scope by guessing a key. + """ + raw = request.headers.get("X-Hermes-Session-Key", "").strip() + if not raw: + return None, None + + if not self._api_key: + logger.warning( + "X-Hermes-Session-Key rejected: no API key configured. " + "Set API_SERVER_KEY to enable long-term memory scoping." + ) + return None, web.json_response( + _openai_error( + "X-Hermes-Session-Key requires API key authentication. " + "Configure API_SERVER_KEY to enable this feature." + ), + status=403, + ) + + # Reject control characters that could enable header injection on + # the echo path. + if re.search(r'[\r\n\x00]', raw): + return None, web.json_response( + {"error": {"message": "Invalid session key", "type": "invalid_request_error"}}, + status=400, + ) + + if len(raw) > self._MAX_SESSION_HEADER_LEN: + return None, web.json_response( + {"error": {"message": "Session key too long", "type": "invalid_request_error"}}, + status=400, + ) + + return raw, None + + # ------------------------------------------------------------------ + # Session DB helper + # ------------------------------------------------------------------ + + def _ensure_session_db(self): + """Lazily initialise and return the shared SessionDB instance. + + Sessions are persisted to ``state.db`` so that ``hermes sessions list`` + shows API-server conversations alongside CLI and gateway ones. + """ + if self._session_db is None: + try: + from hermes_state import SessionDB + self._session_db = SessionDB() + except Exception as e: + logger.debug("SessionDB unavailable for API server: %s", e) + return self._session_db + + # ------------------------------------------------------------------ + # Agent creation helper + # ------------------------------------------------------------------ + + @staticmethod + def _parse_model_routes(raw: Any) -> Dict[str, Dict[str, Any]]: + """Validate and normalize the ``model_routes`` config block. + + Accepts a mapping of ``alias -> {model, provider?, api_key?, base_url?}``. + Invalid shapes are dropped (never raised) so a config typo can't take + the whole API server down. Route values are coerced to strings. + + Security: per-route ``api_key`` values are UPSTREAM provider + credentials (used to call the routed model's backend), not caller + authentication — callers still authenticate with the global + API_SERVER_KEY bearer token via ``_check_auth``. Route api_keys must + never be logged; only alias names and non-secret fields may appear in + logs. + """ + if not isinstance(raw, dict): + if raw: + logger.warning( + "api_server model_routes ignored: expected a mapping, got %s", + type(raw).__name__, + ) + return {} + + allowed_keys = ("model", "provider", "api_key", "base_url") + routes: Dict[str, Dict[str, Any]] = {} + for alias, cfg in raw.items(): + alias_str = str(alias).strip() + if not alias_str or not isinstance(cfg, dict): + logger.warning( + "api_server model_routes: dropping invalid route entry %r", alias_str or alias + ) + continue + route = { + key: str(cfg[key]).strip() + for key in allowed_keys + if cfg.get(key) is not None and str(cfg[key]).strip() + } + if not route.get("model"): + logger.warning( + "api_server model_routes: route %r has no 'model'; dropping", alias_str + ) + continue + routes[alias_str] = route + return routes + + def _resolve_route(self, model_alias: Any) -> Optional[Dict[str, Any]]: + """Return the model_routes entry for *model_alias*, or None.""" + if not self._model_routes or not isinstance(model_alias, str): + return None + return self._model_routes.get(model_alias) + + def _session_model_override_for(self, session_key: Optional[str]) -> Optional[Dict[str, Any]]: + """Return the gateway's session ``/model`` override for *session_key*, if any. + + The gateway tracks per-session ``/model`` switches in + ``GatewayRunner._session_model_overrides``. API-server requests that + share such a session key must keep honouring the explicit session + override even when the request's ``model`` field matches a configured + route — a user-issued ``/model`` always wins over static config. + """ + if not session_key: + return None + try: + from gateway.run import _gateway_runner_ref + runner = _gateway_runner_ref() + if runner is None: + return None + override = runner._session_model_overrides.get(session_key) + return dict(override) if isinstance(override, dict) else None + except Exception: + return None + + def _create_agent( + self, + ephemeral_system_prompt: Optional[str] = None, + session_id: Optional[str] = None, + stream_delta_callback=None, + tool_progress_callback=None, + tool_start_callback=None, + tool_complete_callback=None, + gateway_session_key: Optional[str] = None, + route: Optional[Dict[str, Any]] = None, + ) -> Any: + """ + Create an AIAgent instance using the gateway's runtime config. + + Uses _resolve_runtime_agent_kwargs() to pick up model, api_key, + base_url, etc. from config.yaml / env vars. Toolsets are resolved + from config.yaml platform_toolsets.api_server (same as all other + gateway platforms), falling back to the hermes-api-server default. + + ``gateway_session_key`` is a stable per-channel identifier supplied + by the client (via ``X-Hermes-Session-Key``). Unlike ``session_id`` + which scopes the short-term transcript and rotates on /new, this + key is meant to persist across transcripts so long-term memory + providers (e.g. Honcho) can scope their per-chat state correctly + — matching the semantics of the native gateway's ``session_key``. + + ``route`` is an optional ``model_routes`` entry (per-client model + routing). When set — and no session ``/model`` override exists for + this session — its model/provider/api_key/base_url override the + global defaults for this agent instance only. + """ + from run_agent import AIAgent + from gateway.run import ( + _current_max_iterations, + _resolve_runtime_agent_kwargs, + _resolve_gateway_model, + _load_gateway_config, + GatewayRunner, + ) + from hermes_cli.tools_config import _get_platform_tools + + runtime_kwargs = _resolve_runtime_agent_kwargs() + reasoning_config = GatewayRunner._load_reasoning_config() + model = _resolve_gateway_model() + + # When the primary provider's auth fails (expired token / 429 quota + # cap), _resolve_runtime_agent_kwargs() falls through to the fallback + # provider chain, whose runtime dict carries its own ``model`` key. + # Pop it and let it override the config model, mirroring the native + # gateway path (_resolve_session_agent_runtime in run.py). Otherwise + # the explicit ``model=model`` below collides with the ``**runtime_kwargs`` + # spread → "got multiple values for keyword argument 'model'", 500ing + # every /v1/chat/completions request while a fallback is active. + runtime_model = runtime_kwargs.pop("model", None) + if runtime_model: + model = runtime_model + + # Per-client model routing (model_routes config). The route was + # resolved from the request's ``model`` field by the HTTP handler. + # Precedence (highest first): session ``/model`` override → model_routes + # route → global config — an explicit user-issued ``/model`` on the + # session always beats static per-client route config. + session_override = self._session_model_override_for( + gateway_session_key or session_id + ) + if route and not session_override: + if route.get("provider"): + # Resolve real credentials for the routed provider (mirrors + # the channel_overrides path in gateway/run.py) so a route + # without an explicit api_key/base_url still gets the right + # provider auth instead of the default provider's key. + try: + from gateway.run import _resolve_runtime_agent_kwargs_for_provider + provider_kwargs = _resolve_runtime_agent_kwargs_for_provider( + route["provider"] + ) + provider_kwargs.pop("model", None) + runtime_kwargs.update(provider_kwargs) + except Exception: + # Fall back to just switching the provider name; explicit + # per-route api_key/base_url below can still complete auth. + runtime_kwargs["provider"] = route["provider"] + if route.get("model"): + model = route["model"] + # Per-route secrets are upstream provider credentials. Never log + # them (compare _check_auth: caller auth stays the global bearer + # key checked with hmac.compare_digest). + if route.get("api_key"): + runtime_kwargs["api_key"] = route["api_key"] + if route.get("base_url"): + runtime_kwargs["base_url"] = route["base_url"] + logger.debug( + "api_server model route applied: model=%s provider=%s", + model, + runtime_kwargs.get("provider"), + ) + elif route and session_override: + logger.debug( + "api_server model route skipped: session /model override wins for %s", + gateway_session_key or session_id, + ) + + user_config = _load_gateway_config() + enabled_toolsets = sorted(_get_platform_tools(user_config, "api_server")) + + max_iterations = _current_max_iterations() + + # Load fallback provider chain so the API server platform has the + # same fallback behaviour as Telegram/Discord/Slack (fixes #4954). + fallback_model = GatewayRunner._load_fallback_model() + + agent = AIAgent( + model=model, + **runtime_kwargs, + max_iterations=max_iterations, + quiet_mode=True, + verbose_logging=False, + ephemeral_system_prompt=ephemeral_system_prompt or None, + enabled_toolsets=enabled_toolsets, + session_id=session_id, + platform="api_server", + stream_delta_callback=stream_delta_callback, + tool_progress_callback=tool_progress_callback, + tool_start_callback=tool_start_callback, + tool_complete_callback=tool_complete_callback, + session_db=self._ensure_session_db(), + fallback_model=fallback_model, + reasoning_config=reasoning_config, + gateway_session_key=gateway_session_key, + ) + return agent + + # ------------------------------------------------------------------ + # HTTP Handlers + # ------------------------------------------------------------------ + + async def _handle_health(self, request: "web.Request") -> "web.Response": + """GET /health — simple health check.""" + return web.json_response( + {"status": "ok", "platform": "hermes-agent", "version": _hermes_version()} + ) + + async def _handle_health_detailed(self, request: "web.Request") -> "web.Response": + """GET /health/detailed — rich status for cross-container dashboard probing. + + Returns gateway state, connected platforms, PID, and uptime so the + dashboard can display full status without needing a shared PID file or + /proc access. Requires the same Bearer auth as other API routes. + """ + auth_err = self._check_auth(request) + if auth_err: + return auth_err + + from gateway.status import ( + derive_gateway_busy, + derive_gateway_drainable, + parse_active_agents, + read_runtime_status, + ) + + runtime = read_runtime_status() or {} + gw_state = runtime.get("gateway_state") + gw_active = parse_active_agents(runtime.get("active_agents", 0)) + # This endpoint is served BY the gateway process, so it is by definition + # alive — gateway_running is True. Derive busy/drainable from the same + # shared contract /api/status uses so the two surfaces never disagree. + active_api_runs, process_depth, active_delegations = self._readiness_work_counts() + from gateway.run import _resolve_gateway_model + + readiness = collect_runtime_readiness( + configured_model=_resolve_gateway_model(), + runtime_status=runtime, + active_api_runs=active_api_runs, + process_completion_queue_depth=process_depth, + active_delegations=active_delegations, + ) + return web.json_response({ + "status": readiness["status"], + "readiness": readiness, + "platform": "hermes-agent", + "version": _hermes_version(), + "gateway_state": gw_state, + "platforms": runtime.get("platforms", {}), + "active_agents": gw_active, + "gateway_busy": derive_gateway_busy( + gateway_running=True, + gateway_state=gw_state, + active_agents=gw_active, + ), + "gateway_drainable": derive_gateway_drainable( + gateway_running=True, + gateway_state=gw_state, + ), + "exit_reason": runtime.get("exit_reason"), + "updated_at": runtime.get("updated_at"), + "pid": os.getpid(), + }) + + @staticmethod + def _audio_extension_for_content_type(content_type: str) -> str: + """Map an inbound audio Content-Type to a safe cache-file suffix.""" + media_type = (content_type or "").split(";", 1)[0].strip().lower() + return _ESP32_AUDIO_EXTENSIONS.get(media_type, ".wav") + + @staticmethod + def _audio_response_mime(path: str) -> str: + """Return an HTTP Content-Type for a generated audio file path.""" + suffix = Path(path).suffix.lower() + return _AUDIO_RESPONSE_MIME_BY_SUFFIX.get(suffix, "application/octet-stream") + + @staticmethod + + def _convert_audio_to_esp32_wav(input_path: str, *, output_dir: Path, device_id: str) -> Path: + """Normalize generated TTS audio to ESP32-friendly WAV. + + Hermes TTS providers commonly emit MP3 (Edge) or OGG. The ESP32 test + client expects a small, predictable RIFF/WAVE file, so decode with + ffmpeg to raw PCM and rewrite the WAV container with Python's ``wave`` + module. That produces a minimal ``fmt`` + ``data`` WAV: mono, 16 kHz, + signed 16-bit little-endian PCM. + """ + import subprocess + import wave + + safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" + output_dir.mkdir(parents=True, exist_ok=True) + raw_path = output_dir / f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}.s16le" + wav_path = raw_path.with_suffix(".wav") + cmd = [ + "ffmpeg", + "-y", + "-hide_banner", + "-loglevel", + "error", + "-i", + input_path, + "-ac", + "1", + "-ar", + "16000", + "-f", + "s16le", + str(raw_path), + ] + try: + subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + raw = raw_path.read_bytes() + with wave.open(str(wav_path), "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16000) + wav.writeframes(raw) + return wav_path + finally: + try: + raw_path.unlink(missing_ok=True) + except Exception: + pass + + @staticmethod + def _safe_audio_header(value: Any, max_length: int) -> str: + """Make diagnostic text safe for HTTP response headers.""" + text = str(value or "")[:max_length] + return text.replace("\r", " ").replace("\n", " ").replace("\x00", " ") + + @staticmethod + def _esp32_reply_mode(request: "web.Request") -> str: + """Return 'sync' or 'ack' for the ESP32 voice response contract.""" + value = ( + request.headers.get("X-Hermes-Reply-Mode") + or request.query.get("reply_mode") + or request.query.get("mode") + or "sync" + ) + value = str(value).strip().lower() + if value in {"ack", "async", "background", "quick_ack", "quick-ack"}: + return "ack" + return "sync" + + @staticmethod + def _looks_like_http_url(value: str) -> bool: + try: + parsed = urlparse(value) + return parsed.scheme in {"http", "https"} and bool(parsed.netloc) + except Exception: + return False + + @staticmethod + def _load_local_device_url(device_name: str) -> Optional[str]: + """Best-effort local MCP screen lookup from ~/.hermes/local_devices.yaml.""" + if not device_name: + return None + try: + import yaml + registry = Path.home() / ".hermes" / "local_devices.yaml" + data = yaml.safe_load(registry.read_text()) if registry.exists() else {} + aliases = data.get("aliases") if isinstance(data, dict) else None + devices = data.get("devices") if isinstance(data, dict) else None + names = [str(device_name).strip(), str(device_name).strip().replace("-", "_")] + seen: set[str] = set() + while names: + name = names.pop(0) + if not name or name in seen: + continue + seen.add(name) + if isinstance(aliases, dict): + alias = aliases.get(name) + if isinstance(alias, str) and alias: + if alias.startswith("http"): + return alias + names.append(alias) + info = devices.get(name) if isinstance(devices, dict) else None + if isinstance(info, dict): + return str(info.get("fallback_url") or info.get("url") or "") or None + except Exception: + return None + return None + + @staticmethod + def _load_local_device_url_by_ip(source_ip: str) -> Optional[str]: + """Best-effort local MCP screen lookup by observed device IP. + + Some MCP-capable devices, notably the iPhone companion app, do not run + their MCP endpoint on port 80. When the voice gateway sees traffic from + one of those devices, prefer the registry URL over synthesizing + http:///api/mcp. + """ + value = str(source_ip or "").strip() + if not value: + return None + if value.startswith("::ffff:"): + value = value.removeprefix("::ffff:") + try: + import yaml + registry = Path.home() / ".hermes" / "local_devices.yaml" + data = yaml.safe_load(registry.read_text()) if registry.exists() else {} + devices = data.get("devices") if isinstance(data, dict) else None + if not isinstance(devices, dict): + return None + for info in devices.values(): + if not isinstance(info, dict): + continue + candidates = { + str(info.get("reserved_ip") or "").strip(), + str(info.get("host") or "").strip(), + } + for url_key in ("fallback_url", "url"): + raw_url = str(info.get(url_key) or "").strip() + if raw_url: + parsed = urlparse(raw_url) + if parsed.hostname: + candidates.add(parsed.hostname) + if value in candidates: + return str(info.get("fallback_url") or info.get("url") or "") or None + except Exception: + return None + return None + + @staticmethod + def _esp32_screen_url_from_source_ip(source_ip: str) -> Optional[str]: + """Build the MCP endpoint for the ESP32 that made this request. + + ESP32 boards can move between DHCP leases. For voice ACK follow-ups the + initiating board is the safest target, so prefer its observed peer IP + instead of a possibly stale local-device registry entry. + """ + value = str(source_ip or "").strip() + if not value: + return None + if value.startswith("::ffff:"): + value = value.removeprefix("::ffff:") + # Do not route to loopback/proxy/invalid addresses; registry fallback is + # better in those cases. + try: + ip = ipaddress.ip_address(value) + except ValueError: + return None + if ip.is_loopback or ip.is_unspecified or ip.is_multicast: + return None + host = f"[{value}]" if ip.version == 6 else value + return f"http://{host}/api/mcp" + + def _resolve_esp32_screen_url(self, request: "web.Request", device_id: str) -> Optional[str]: + """Resolve the screen MCP endpoint associated with this voice turn.""" + explicit = ( + request.headers.get("X-Hermes-Screen-Url") + or request.headers.get("X-Screen-MCP-Url") + or request.query.get("screen_url") + ) + if explicit and self._looks_like_http_url(explicit.strip()): + return explicit.strip() + + if str(device_id or "").strip().lower().startswith("iphone"): + iphone_url = self._load_local_device_url(str(device_id).strip()) + if iphone_url and self._looks_like_http_url(iphone_url): + return iphone_url + + audit_ctx = self._request_audit_context(request) + for source_ip in (audit_ctx.get("peer_ip"), audit_ctx.get("remote")): + registry_url = self._load_local_device_url_by_ip(source_ip or "") + if registry_url and self._looks_like_http_url(registry_url): + return registry_url + + audit_ctx = self._request_audit_context(request) + for source_ip in (audit_ctx.get("peer_ip"), audit_ctx.get("remote")): + source_url = self._esp32_screen_url_from_source_ip(source_ip or "") + if source_url: + return source_url + + screen_device = ( + request.headers.get("X-Hermes-Screen-Device") + or request.query.get("screen_device") + or device_id + ) + candidates = [screen_device, "little32", "esp32_screen"] + for candidate in candidates: + url = self._load_local_device_url(str(candidate).strip()) + if url and self._looks_like_http_url(url): + return url + return None + + async def _call_screen_mcp( + self, + screen_url: Optional[str], + tool_name: str, + arguments: Dict[str, Any], + *, + timeout: float = 8.0, + ) -> bool: + """Call a simple HTTP JSON-RPC MCP screen tool directly.""" + if not screen_url: + return False + try: + from aiohttp import ClientSession, ClientTimeout + payload = { + "jsonrpc": "2.0", + "id": uuid.uuid4().hex[:12], + "method": "tools/call", + "params": {"name": tool_name, "arguments": arguments}, + } + async with ClientSession(timeout=ClientTimeout(total=timeout)) as session: + async with session.post(screen_url, json=payload) as resp: + body = await resp.text() + if resp.status >= 400: + logger.warning( + "[api_server] screen MCP %s failed: HTTP %s %s", + tool_name, + resp.status, + body[:300], + ) + return False + try: + data = json.loads(body) if body else {} + if isinstance(data, dict) and data.get("error"): + logger.warning("[api_server] screen MCP %s returned error: %s", tool_name, data.get("error")) + return False + except Exception: + pass + return True + except Exception as exc: + logger.warning("[api_server] screen MCP %s failed: %s", tool_name, exc) + return False + + @staticmethod + def _screen_stream_target(screen_url: Optional[str]) -> Optional[tuple[str, int]]: + """Return the raw UDP video-stream target for MCP screens that expose one.""" + if not screen_url: + return None + try: + parsed = urlparse(screen_url) + host = parsed.hostname + if not host: + return None + return (host, 8082) + except Exception: + return None + + @staticmethod + def _voice_animation_udp_frames(state: str, text: str = "", frames: int = 18) -> List[bytes]: + """Render small Hermes voice-state animations as RLCD-compatible frames.""" + from PIL import Image, ImageDraw + import math + + width, height = 400, 300 + + def to_rlcd(img: "Image.Image") -> bytes: + img_1bit = img.convert("1", dither=Image.Dither.FLOYDSTEINBERG) + px = img_1bit.load() + buf = bytearray(15000) + for y in range(height): + for x in range(width): + if px[x, y]: + inv_y = 299 - y + bx = x // 2 + by = inv_y // 4 + idx = bx * 75 + by + lx = x % 2 + ly = inv_y % 4 + bit = 7 - (ly * 2 + lx) + buf[idx] |= (1 << bit) + return bytes(buf) + + label_by_state = { + "ack": "Got it", + "captioning": "Listening", + "waiting": "Thinking", + "speaking": "Replying", + "success": "Done", + "error": "Try again", + } + label = label_by_state.get(str(state), str(state or "Hermes").title()) + snippet = " ".join(str(text or "").split())[:42] + rendered: List[bytes] = [] + for frame in range(max(1, int(frames))): + img = Image.new("1", (width, height), 0) + draw = ImageDraw.Draw(img) + cx, cy = width // 2, height // 2 + phase = frame * 0.35 + if state == "captioning": + draw.rectangle((188, 38, 212, 75), outline=1) + draw.arc((175, 55, 225, 105), 20, 160, fill=1) + for x in range(42, width - 42, 5): + y = cy + int(32 * math.sin(x * 0.045 + phase * 2)) + y2 = cy + int(14 * math.sin(x * 0.085 - phase * 3)) + draw.line((x, y, min(width - 1, x + 4), y2), fill=1) + elif state in {"waiting", "speaking"}: + for radius in (34, 64, 94): + pulse = radius + int(8 * math.sin(phase + radius)) + draw.ellipse((cx - pulse, cy - pulse, cx + pulse, cy + pulse), outline=1) + for i in range(7): + x = 93 + i * 34 + bar = int(18 + 42 * (0.5 + 0.5 * math.sin(phase * 2 + i))) + draw.rectangle((x, 228 - bar, x + 17, 228), fill=1) + elif state == "error": + draw.line((150, 88, 250, 188), fill=1) + draw.line((250, 88, 150, 188), fill=1) + draw.rectangle((145, 82, 255, 194), outline=1) + else: + r = 42 + int(16 * math.sin(phase)) + draw.ellipse((cx - r, cy - r, cx + r, cy + r), outline=1) + draw.ellipse((cx - 8, cy - 8, cx + 8, cy + 8), fill=1) + draw.text((max(8, (width - len(label) * 6) // 2), 24), label, fill=1) + if snippet: + draw.text((max(8, (width - len(snippet) * 6) // 2), 260), snippet, fill=1) + rendered.append(to_rlcd(img)) + return rendered + + async def _send_stream_animation( + self, + screen_url: Optional[str], + state: str, + *, + text: str = "", + frames: Optional[int] = None, + ) -> bool: + """Send a server-rendered voice animation over the screen streaming socket.""" + target = self._screen_stream_target(screen_url) + if not target: + return False + frame_count = frames if frames is not None else (12 if state == "ack" else 18) + try: + rendered = await asyncio.to_thread(self._voice_animation_udp_frames, state, text, frame_count) + sock = _socket.socket(_socket.AF_INET, _socket.SOCK_DGRAM) + try: + for frame_id, frame in enumerate(rendered): + fid = frame_id % 256 + for chunk_idx in range(15): + start = chunk_idx * 1000 + sock.sendto(bytes([fid, chunk_idx]) + frame[start : start + 1000], target) + await asyncio.sleep(0.045) + finally: + sock.close() + return True + except Exception as exc: + logger.warning("[api_server] screen stream animation failed: %s", exc) + return False + + async def _send_screen_animation( + self, + screen_url: Optional[str], + state: str, + *, + text: str = "", + frames: Optional[int] = None, + ) -> bool: + """Upload and run the ESP32 voice-state animation script on the initiating screen.""" + if not screen_url: + return False + if await self._send_stream_animation(screen_url, state, text=text, frames=frames): + return True + if screen_url not in self._esp32_voice_animation_uploaded: + uploaded = await self._call_screen_mcp( + screen_url, + "write_file", + {"path": ESP32_VOICE_ANIMATION_PATH, "content": ESP32_VOICE_ANIMATION_SCRIPT}, + ) + if not uploaded: + return False + self._esp32_voice_animation_uploaded.add(screen_url) + code = ( + f"exec(open({ESP32_VOICE_ANIMATION_PATH!r}).read()); " + f"run_state({str(state)!r}, text={str(text or '')[:240]!r}, frames={frames!r})" + ) + return await self._call_screen_mcp(screen_url, "execute_python", {"code": code}) + + async def _send_screen_text(self, screen_url: Optional[str], text: str, *, clear: bool = False) -> None: + """Best-effort text update to the initiating MCP screen.""" + if not screen_url: + return + if clear: + await self._call_screen_mcp(screen_url, "clear_screen", {"color": 0}) + # Most Reyna screens accept x/y/text. Extra wrapping is avoided here so + # firmware-specific clients can keep their own font/layout behavior. + await self._call_screen_mcp(screen_url, "draw_text", {"x": 8, "y": 8, "text": text[:900]}) + + @staticmethod + def _extract_first_image_reference(text: str) -> Optional[str]: + """Return the first local path, MEDIA path, data URL, or HTTP image URL in text.""" + if not text: + return None + patterns = [ + r"MEDIA:([^\s)]+)", + r"!\[[^\]]*\]\(([^)]+)\)", + r"(?Phttps?://[^\s)]+\.(?:png|jpe?g|webp|gif)(?:\?[^\s)]*)?)", + r"(?P/[^\s)]+\.(?:png|jpe?g|webp|gif|bmp))", + r"(?Pdata:image/[^\s)]+)", + ] + for pattern in patterns: + for match in re.finditer(pattern, text, flags=re.IGNORECASE): + ref = match.groupdict().get("url") or match.groupdict().get("path") or match.groupdict().get("data") or match.group(1) + if ref: + return ref.strip().strip('"\'') + return None + + async def _send_screen_image_response(self, screen_url: Optional[str], text: str) -> bool: + """Best-effort render of the first generated/linked image onto the ESP32 screen.""" + if not screen_url: + return False + image_ref = self._extract_first_image_reference(text) + if not image_ref: + return False + try: + import base64 + import io + from PIL import Image, ImageOps + + raw: bytes + lowered = image_ref.lower() + if lowered.startswith("data:image/"): + _, _, encoded = image_ref.partition(",") + raw = base64.b64decode(encoded) + elif self._looks_like_http_url(image_ref): + from aiohttp import ClientSession, ClientTimeout + async with ClientSession(timeout=ClientTimeout(total=20)) as session: + async with session.get(image_ref) as resp: + if resp.status >= 400: + logger.warning("[api_server] screen image fetch failed: HTTP %s", resp.status) + return False + raw = await resp.read() + else: + path = Path(image_ref).expanduser() + if not path.exists() or not path.is_file(): + return False + raw = await asyncio.to_thread(path.read_bytes) + if not raw: + return False + + def render_pbm() -> str: + with Image.open(io.BytesIO(raw)) as img: + img = ImageOps.exif_transpose(img).convert("RGB") + canvas = Image.new("RGB", (320, 240), "white") + img.thumbnail((320, 240), Image.Resampling.LANCZOS) + x = (320 - img.width) // 2 + y = (240 - img.height) // 2 + canvas.paste(img, (x, y)) + mono = canvas.convert("1", dither=Image.Dither.FLOYDSTEINBERG) + out = io.BytesIO() + mono.save(out, format="PPM") # mode=1 writes raw PBM (P4) + return base64.b64encode(out.getvalue()).decode("ascii") + + pbm_base64 = await asyncio.to_thread(render_pbm) + return await self._call_screen_mcp( + screen_url, + "draw_image", + {"pbm_base64": pbm_base64, "x": 0, "y": 0, "dither": False}, + timeout=15, + ) + except Exception: + logger.exception("[api_server] failed to render ESP32 screen image response") + return False + + async def _send_screen_audio_response( + self, + screen_url: Optional[str], + text: str, + *, + audio_dir: Path, + device_id: str, + volume: int = 80, + ) -> bool: + """Best-effort spoken follow-up for ACK-mode ESP32 voice turns.""" + if not screen_url or not text.strip(): + return False + try: + import base64 + from tools.tts_tool import text_to_speech_tool + + # Keep board playback short and payloads small. The ESP32 HTTP MCP + # endpoint resets on large base64 WAV bodies; MP3 is much smaller + # and the firmware exposes play_mp3_base64 for exactly this path. + spoken_text = self._short_voice_summary(text, 420) + tts_raw = await asyncio.to_thread(text_to_speech_tool, spoken_text) + tts_result = json.loads(tts_raw) + src = str(tts_result.get("file_path") or "") + if not tts_result.get("success") or not src or not Path(src).exists(): + logger.warning( + "[api_server] screen audio TTS failed: %s", + tts_result.get("error") or "missing TTS output", + ) + return False + + src_path = Path(src) + + # Detect if this is iPhone (has no play_mp3_base64, only play_audio_base64) + is_iphone = "iphone" in str(device_id).lower() or ":8080" in str(screen_url or "") + + if not is_iphone and src_path.suffix.lower() == ".mp3": + mp3_base64 = base64.b64encode(src_path.read_bytes()).decode("ascii") + if await self._call_screen_mcp( + screen_url, + "play_mp3_base64", + {"mp3_base64": mp3_base64, "volume": volume}, + timeout=75, + ): + return True + + wav_path = await asyncio.to_thread( + self._convert_audio_to_esp32_wav, + src, + output_dir=audio_dir, + device_id=f"{device_id}_response", + ) + wav_bytes = wav_path.read_bytes() + # iPhone can handle larger WAVs than ESP32 (tested 180k+), increase limit for iPhone + max_wav = 600_000 if is_iphone else 220_000 + if len(wav_bytes) > max_wav: + # For iPhone, truncate to first 6 seconds to fit + if is_iphone: + import wave, io + try: + with wave.open(str(wav_path), "rb") as wf: + fr = wf.getframerate() + nframes = int(fr * 6.0) + wf.rewind() + frames = wf.readframes(nframes) + bio = io.BytesIO() + with wave.open(bio, "wb") as out: + out.setnchannels(wf.getnchannels()) + out.setsampwidth(wf.getsampwidth()) + out.setframerate(fr) + out.writeframes(frames) + wav_bytes = bio.getvalue() + except Exception: + pass + else: + logger.warning( + "[api_server] screen audio WAV too large for MCP payload: %s bytes", + len(wav_bytes), + ) + return False + wav_base64 = base64.b64encode(wav_bytes).decode("ascii") + return await self._call_screen_mcp( + screen_url, + "play_audio_base64", + {"wav_base64": wav_base64, "volume": volume}, + timeout=45, + ) + except Exception: + logger.exception("[api_server] failed to play ESP32 screen audio response") + return False + + @staticmethod + def _short_voice_summary(text: str, limit: int = 220) -> str: + """Local fallback summary for voice/screen snippets without another LLM call.""" + clean = " ".join(str(text or "").split()) + if len(clean) <= limit: + return clean + return clean[: max(0, limit - 1)].rstrip() + "…" + + async def _esp32_ack_wav(self, *, audio_dir: Path, device_id: str) -> Path: + """Return a reusable quiet ACK WAV; the visual screen animation is the acknowledgement.""" + ack_path = audio_dir / "quick_ack.wav" + if ack_path.exists(): + return ack_path + + # ACK-mode clients expect a valid audio/wav response immediately, but + # Reyna ESP32 screens now show the acknowledgement visually. Keep the + # HTTP contract while avoiding the old spoken "Got it" message. + import wave + sample_rate = 16000 + duration_s = 0.08 + frames = b"\x00\x00" * int(sample_rate * duration_s) + with wave.open(str(ack_path), "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(sample_rate) + wav.writeframes(frames) + return ack_path + + def _append_esp32_voice_audit(self, audio_dir: Path, event: Dict[str, Any]) -> None: + """Append a best-effort ESP32 voice request audit event.""" + try: + log_path = audio_dir / "voice_requests.jsonl" + with log_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(event, default=str) + "\n") + except Exception: + logger.debug("[api_server] failed to append ESP32 voice audit log", exc_info=True) + + async def _run_esp32_voice_audio_background( + self, + *, + audio_path: Path, + audit_event: Dict[str, Any], + device_id: str, + instructions: Optional[str], + audio_dir: Path, + screen_url: Optional[str], + ) -> None: + """Transcribe uploaded ESP32 audio and continue the voice turn after an immediate ACK.""" + try: + await self._send_screen_animation(screen_url, "captioning") + from tools.transcription_tools import transcribe_audio + stt_result = await asyncio.to_thread(transcribe_audio, str(audio_path)) + if not stt_result.get("success"): + failed_event = dict(audit_event) + failed_event.update({ + "status": "transcription_failed", + "error": stt_result.get("error") or "Transcription failed", + "timestamp": time.time(), + }) + self._append_esp32_voice_audit(audio_dir, failed_event) + await self._send_screen_text( + screen_url, + "I heard audio, but transcription failed. Try again.", + clear=True, + ) + return + transcript = str(stt_result.get("transcript") or "").strip() + if not transcript: + empty_event = dict(audit_event) + empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) + self._append_esp32_voice_audit(audio_dir, empty_event) + await self._send_screen_text( + screen_url, + "I heard audio, but got no words. Try again.", + clear=True, + ) + return + transcript_event = dict(audit_event) + transcript_event.update({"status": "transcribed", "transcript": transcript, "timestamp": time.time()}) + self._append_esp32_voice_audit(audio_dir, transcript_event) + await self._run_esp32_voice_background( + transcript=transcript, + device_id=device_id, + instructions=instructions, + audio_dir=audio_dir, + screen_url=screen_url, + ) + except Exception as exc: + logger.exception("[api_server] ESP32 ACK background voice processing failed") + failed_event = dict(audit_event) + failed_event.update({"status": "background_exception", "error": str(exc), "timestamp": time.time()}) + self._append_esp32_voice_audit(audio_dir, failed_event) + await self._send_screen_text( + screen_url, + "I got the audio, but background processing failed.", + clear=True, + ) + + async def _run_esp32_voice_background( + self, + *, + transcript: str, + device_id: str, + instructions: Optional[str], + audio_dir: Path, + screen_url: Optional[str], + ) -> None: + """Run the slow ESP32 voice turn after the device already heard an ACK.""" + try: + animation_ok = await self._send_screen_animation(screen_url, "waiting", text=transcript) + if not animation_ok: + await self._send_screen_text( + screen_url, + f"Heard: {self._short_voice_summary(transcript, 180)}\n\nWorking…", + clear=True, + ) + screen_hint = ( + "\n\nThis request came from an ESP32 voice/screen device. " + "Keep the spoken follow-up short. The gateway will display the full answer on the initiating screen when possible." + ) + merged_instructions = (instructions or "") + screen_hint + response_data, final_text = await self._run_esp32_voice_turn( + transcript=transcript, + device_id=device_id, + instructions=merged_instructions, + ) + if not final_text.strip(): + final_text = self._extract_response_text(response_data) or "I finished, but I do not have a text answer." + display_text = final_text + if len(display_text) > 1200: + display_text = self._short_voice_summary(display_text, 900) + "\n\nFull answer is long; WhatsApp delivery is the next wiring step." + await self._send_screen_text(screen_url, display_text, clear=True) + await self._send_screen_image_response(screen_url, final_text) + await self._send_screen_audio_response( + screen_url, + final_text, + audio_dir=audio_dir, + device_id=device_id, + ) + except Exception: + logger.exception("[api_server] ESP32 background voice turn failed") + await self._send_screen_text(screen_url, "Hermes hit an error while preparing the full answer.", clear=True) + + @staticmethod + def _extract_response_text(response_data: Dict[str, Any]) -> str: + """Extract assistant text from an OpenAI Responses-shaped payload.""" + chunks: List[str] = [] + for item in response_data.get("output") or []: + if not isinstance(item, dict) or item.get("type") != "message": + continue + for part in item.get("content") or []: + if isinstance(part, dict) and part.get("type") in {"output_text", "text"}: + text = part.get("text") + if isinstance(text, str) and text: + chunks.append(text) + return "\n".join(chunks).strip() + + async def _run_esp32_voice_turn( + self, + *, + transcript: str, + device_id: str, + instructions: Optional[str] = None, + ) -> tuple[Dict[str, Any], str]: + """Run one persistent Responses-style turn for an ESP32 voice device.""" + conversation = f"esp32:{device_id}" + previous_response_id = self._response_store.get_conversation(conversation) + conversation_history: List[Dict[str, Any]] = [] + stored_session_id = None + stored_instructions = None + + if previous_response_id: + stored = self._response_store.get(previous_response_id) + if stored is not None: + conversation_history = list(stored.get("conversation_history", [])) + stored_session_id = stored.get("session_id") + stored_instructions = stored.get("instructions") + + if instructions is None: + instructions = stored_instructions + + session_id = stored_session_id or str(uuid.uuid4()) + user_message = ( + f'[Voice input from ESP32 device "{device_id}"]\n' + f'{transcript}' + ) + + result, usage = await self._run_agent( + user_message=user_message, + conversation_history=conversation_history, + ephemeral_system_prompt=instructions, + session_id=session_id, + gateway_session_key=conversation, + ) + + final_response = result.get("final_response", "") or result.get("error", "") or "(No response generated)" + response_id = f"resp_{uuid.uuid4().hex[:28]}" + created_at = int(time.time()) + full_history = self._build_response_conversation_history( + conversation_history, + user_message, + result, + final_response, + ) + output_start_index = self._response_messages_turn_start_index( + conversation_history, + user_message, + result, + ) + output_items = self._extract_output_items(result, start_index=output_start_index) + response_data = { + "id": response_id, + "object": "response", + "status": "completed", + "created_at": created_at, + "model": self._model_name, + "output": output_items, + "usage": { + "input_tokens": usage.get("input_tokens", 0), + "output_tokens": usage.get("output_tokens", 0), + "total_tokens": usage.get("total_tokens", 0), + }, + } + self._response_store.put(response_id, { + "response": response_data, + "conversation_history": full_history, + "instructions": instructions, + "session_id": session_id, + }) + self._response_store.set_conversation(conversation, response_id) + return response_data, final_response + + @staticmethod + def _esp32_audio_cache_dir() -> Path: + """Return the cache directory for ESP32 voice artifacts.""" + try: + from hermes_constants import get_hermes_dir + return get_hermes_dir("cache/audio", "audio_cache") / "esp32" + except Exception: + return Path.home() / ".hermes" / "cache" / "audio" / "esp32" + + @staticmethod + def _write_pcm_s16le_wav(raw_path: Path, wav_path: Path, *, sample_rate: int, channels: int = 1) -> None: + """Wrap a raw PCM s16le file in a minimal RIFF/WAVE container.""" + import wave + + with wave.open(str(wav_path), "wb") as wav: + wav.setnchannels(channels) + wav.setsampwidth(2) + wav.setframerate(sample_rate) + with raw_path.open("rb") as raw_handle: + while True: + chunk = raw_handle.read(64 * 1024) + if not chunk: + break + wav.writeframes(chunk) + + # --- Live draft helpers (from your whisper-translation/apple_speech pipe protocol) --- + def _load_macmini_mcp_config(self) -> dict: + try: + if ESP32_MACMINI_MCP_URL and ESP32_MACMINI_MCP_TOKEN: + return {"url": ESP32_MACMINI_MCP_URL, "token": ESP32_MACMINI_MCP_TOKEN} + cfg_path = Path.home() / ".hermes" / "config.yaml" + if cfg_path.exists(): + import yaml + cfg = yaml.safe_load(cfg_path.read_text()) or {} + m = (cfg.get("mcp_servers") or {}).get("macmini") or {} + url = m.get("url") or "" + token = "" + headers = m.get("headers") or {} + auth = headers.get("Authorization") or "" + if auth.startswith("Bearer "): + token = auth[7:] + if url: + return {"url": url, "token": token} + except Exception: + pass + return {} + + @staticmethod + def _s16le_to_wav_bytes(pcm_bytes: bytes, sample_rate: int = 16000, channels: int = 1) -> bytes: + import io, wave + bio = io.BytesIO() + with wave.open(bio, "wb") as wf: + wf.setnchannels(channels) + wf.setsampwidth(2) + wf.setframerate(sample_rate) + wf.writeframes(pcm_bytes) + return bio.getvalue() + + async def _transcribe_draft_via_macmini(self, wav_bytes: bytes, locale: str = "en-US") -> str | None: + if not ESP32_LIVE_DRAFT_USE_MACMINI: + return None + cfg = self._load_macmini_mcp_config() + url = cfg.get("url") + token = cfg.get("token") + if not url: + return None + try: + import base64 + b64 = base64.b64encode(wav_bytes).decode("ascii") + # First try new live pipe tool + for tool_name in ["speech_live_transcribe", "speech_transcribe_file"]: + if tool_name == "speech_transcribe_file": + # file tool expects filePath on Mac, not base64 - can't use unless we upload, skip for draft + continue + payload = { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": tool_name, "arguments": {"audioBase64": b64, "locale": locale} if tool_name=="speech_live_transcribe" else {"filePath": "/tmp/dummy.wav", "locale": locale}}, + } + headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"} + if token: + headers["Authorization"] = f"Bearer {token}" + import aiohttp + async with aiohttp.ClientSession() as session: + async with session.post(url, json=payload, headers=headers, timeout=aiohttp.ClientTimeout(total=8)) as resp: + txt = await resp.text() + if resp.status != 200: + logger.debug(f"[api_server] macmini {tool_name} HTTP {resp.status}: {txt[:500]}") + continue + for line in txt.splitlines(): + if not line.startswith("data: "): + continue + try: + data = json.loads(line[6:]) + if data.get("result", {}).get("isError"): + # log error to help debugging + err_txt = (data.get("result", {}).get("content") or [{}])[0].get("text","")[:500] + if "not found" in err_txt.lower(): + logger.warning(f"[api_server] macmini tool {tool_name} not found — need git pull on macmini (have {err_txt[:100]})") + else: + logger.debug(f"[api_server] macmini {tool_name} error: {err_txt}") + continue + content = (data.get("result", {}).get("content") or [{}])[0].get("text", "") or "" + if not content: + continue + inner = json.loads(content) + t = inner.get("text") or "" + if t: + return t.strip() + except Exception as ex: + logger.debug(f"[api_server] macmini parse err: {ex}") + continue + except Exception as e: + logger.debug(f"[api_server] macmini live draft exception: {e}") + return None + + async def _transcribe_draft_local(self, wav_bytes: bytes) -> str | None: + try: + import tempfile + fd, tmp_path = tempfile.mkstemp(suffix=".wav") + os.close(fd) + Path(tmp_path).write_bytes(wav_bytes) + def _do(): + try: + from faster_whisper import WhisperModel + model = None + try: + from tools.transcription_tools import _local_model + if _local_model is not None: + model = _local_model + except Exception: + pass + if model is None: + model = WhisperModel("tiny", device="cpu", compute_type="int8", num_workers=1) + segs, _ = model.transcribe(tmp_path, beam_size=1, best_of=1, language="en", condition_on_previous_text=False, vad_filter=True) + return " ".join(s.text.strip() for s in segs).strip() + except Exception as ex: + logger.debug(f"[api_server] local draft err: {ex}") + return None + finally: + try: + Path(tmp_path).unlink(missing_ok=True) + except Exception: + pass + return await asyncio.to_thread(_do) + except Exception: + return None + + async def _apple_llm_quick_reply_via_macmini(self, draft: str, context: str = "", instructions: str = "") -> str | None: + """Call Mac mini apple_llm_quick_reply for instant smart ACK from draft.""" + if not ESP32_LIVE_DRAFT_USE_MACMINI: + return None + cfg = self._load_macmini_mcp_config() + url = cfg.get("url") + token = cfg.get("token") + if not url: + return None + try: + payload = { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "apple_llm_quick_reply", "arguments": {"draft": draft, "context": context or "", "instructions": instructions or ""}}, + } + headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"} + if token: + headers["Authorization"] = f"Bearer {token}" + import aiohttp + async with aiohttp.ClientSession() as session: + async with session.post(url, json=payload, headers=headers, timeout=aiohttp.ClientTimeout(total=4)) as resp: + txt = await resp.text() + if resp.status != 200: + logger.debug(f"[api_server] macmini quick_reply HTTP {resp.status}: {txt[:500]}") + return None + for line in txt.splitlines(): + if not line.startswith("data: "): + continue + try: + data = json.loads(line[6:]) + if data.get("result", {}).get("isError"): + err_txt = (data.get("result", {}).get("content") or [{}])[0].get("text","")[:500] + if "not found" in err_txt.lower(): + logger.warning(f"[api_server] macmini apple_llm_quick_reply not found — need git pull + npm restart on mac_mini (192.168.68.102). Have: {err_txt[:120]}") + else: + logger.debug(f"[api_server] quick_reply error: {err_txt}") + continue + content = (data.get("result", {}).get("content") or [{}])[0].get("text", "") or "" + if not content: + continue + inner = json.loads(content) + t = inner.get("text") or "" + if t: + return t.strip() + except Exception as ex: + logger.debug(f"[api_server] quick_reply parse err: {ex}") + continue + except Exception as e: + logger.debug(f"[api_server] macmini quick_reply exception: {e}") + return None + + async def _run_quick_reply_then_full(self, draft_text: str, final_transcript: str, device_id: str, screen_url, instructions: str | None, audio_dir): + """New flow: when voice stops, we already have draft. Emit instant smart reply via Apple LLM, then full Hermes turn.""" + # This is kept for future speculative pre-start; for now we just log — actual quick reply emitted in WS handler before background start + pass + + async def _speak_instant_via_macmini(self, text: str, voice: str = "Alex", use_voicebox: bool = False) -> str | None: + """Synthesize instant reply text via Mac mini voice for immediate audio while Hermes thinks. + - Default: use Mac say Alex boy voice via speech_synthesize_base64 (fast <500ms, reliable) + - If use_voicebox=True, use Voicebox Qwen Aiden boy voice via voicebox_quick_reply (higher quality, slower 2-4s when queue not stuck) + Returns base64 wav if success, else None. + """ + cfg = self._load_macmini_mcp_config() + url = cfg.get("url") + token = cfg.get("token") + if not url: + return None + import aiohttp, json + headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"} + if token: + headers["Authorization"] = f"Bearer {token}" + + tool_name = "voicebox_quick_reply" if use_voicebox else "speech_synthesize_base64" + args = {"text": text[:300]} + if use_voicebox: + args["profile"] = "Aiden" + else: + args["voice"] = voice + + try: + payload = { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": tool_name, "arguments": args}, + } + async with aiohttp.ClientSession() as session: + async with session.post(url, json=payload, headers=headers, timeout=aiohttp.ClientTimeout(total=10 if not use_voicebox else 20)) as resp: + txt = await resp.text() + if resp.status != 200: + logger.debug(f"[api_server] macmini {tool_name} HTTP {resp.status}: {txt[:500]}") + return None + for line in txt.splitlines(): + if not line.startswith("data: "): + continue + try: + data = json.loads(line[6:]) + if data.get("result", {}).get("isError"): + err_txt = (data.get("result", {}).get("content") or [{}])[0].get("text","")[:500] + logger.debug(f"[api_server] macmini {tool_name} error: {err_txt}") + continue + content = (data.get("result", {}).get("content") or [{}])[0].get("text", "") or "" + if not content: + continue + inner = json.loads(content) + b64 = inner.get("wavBase64") or inner.get("wav_base64") + if b64: + return b64 + except Exception as ex: + logger.debug(f"[api_server] {tool_name} parse err: {ex}") + continue + except Exception as e: + logger.debug(f"[api_server] {tool_name} exception: {e}") + return None + + + + async def _handle_esp32_voice_ws(self, request: "web.Request") -> "web.WebSocketResponse": + """GET /api/esp32/voice/ws — ESP32 PCM streaming with LIVE draft captioning. + LIVE DRAFT MODE (from whisper-translation/engine_apple_transcribe.py): + - While ingesting binary s16le, every ~800ms snapshots current PCM -> in-mem WAV + -> tries Mac mini speech_live_transcribe (ApplePipeTranscriber --pipe volatile, + <200ms drafts) fallback to local tiny for fast feedback. + - Emits WS events: draft + interim_transcript for UI. Final via macmini or local base. + """ + auth_err = self._check_auth(request) + if auth_err: + return auth_err # type: ignore[return-value] + + ws = web.WebSocketResponse(heartbeat=None, max_msg_size=512 * 1024) + await ws.prepare(request) + + async def _send_event(event: str, **payload): + if not ws.closed: + await ws.send_str(json.dumps({"event": event, **payload})) + + device_id = ( + request.headers.get("X-Device-ID") + or request.query.get("device_id") + or "default" + ).strip()[:80] or "default" + safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" + audio_dir = self._esp32_audio_cache_dir() + audio_dir.mkdir(parents=True, exist_ok=True) + stamp = f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}" + raw_path = audio_dir / f"{stamp}.s16le" + wav_path = audio_dir / f"{stamp}.wav" + sample_rate = 16000 + channels = 1 + sample_width = 2 + total_audio_bytes = 0 + started = False + stopped = False + first_audio_at: float | None = None + started_at = time.time() + # live draft state (directly from your whisper-translation ApplePipeTranscriber pattern) + live_enabled = ESP32_LIVE_DRAFT_ENABLED + pcm_buffer = bytearray() + last_draft_at = 0.0 + last_draft_bytes = 0 + last_draft_text = "" + draft_in_progress = False + draft_count = 0 + locale_req = request.headers.get("X-Hermes-Locale") or request.query.get("locale") or "en-US" + + audit_ctx = self._request_audit_context(request) + audit_event = { + "timestamp": started_at, + "device_id": device_id, + "source_ip": audit_ctx.get("remote") or audit_ctx.get("peer_ip") or "", + "peer_ip": audit_ctx.get("peer_ip") or "", + "forwarded_for": audit_ctx.get("forwarded_for") or "", + "real_ip": audit_ctx.get("real_ip") or "", + "user_agent": audit_ctx.get("user_agent") or "", + "transport": "websocket", + "raw_path": str(raw_path), + "audio_path": str(wav_path), + "status": "accepted", + "live_draft_enabled": live_enabled, + } + self._append_esp32_voice_audit(audio_dir, audit_event) + screen_url = self._resolve_esp32_screen_url(request, device_id) + visual_listening_sent = False + + def _schedule_voice_state(state: str, *, text: str = "", frames: int | None = None) -> None: + if not screen_url: + return + task = asyncio.create_task(self._send_screen_animation(screen_url, state, text=text, frames=frames)) + try: + self._background_tasks.add(task) + except TypeError: + pass + if hasattr(task, "add_done_callback"): + task.add_done_callback(self._background_tasks.discard) + + async def _do_live_draft(snapshot_bytes: bytes): + nonlocal last_draft_text, draft_in_progress, draft_count + if draft_in_progress: + return + if len(snapshot_bytes) < int(sample_rate * 0.5 * 2): # at least 500ms of s16le at 16k mono + return + draft_in_progress = True + try: + wav_b = self._s16le_to_wav_bytes(snapshot_bytes, sample_rate=sample_rate, channels=channels) + txt = None + if ESP32_LIVE_DRAFT_USE_MACMINI: + txt = await self._transcribe_draft_via_macmini(wav_b, locale=locale_req) + if not txt: + txt = await self._transcribe_draft_local(wav_b) + if not txt: + return + txt = txt.strip() + if not txt or txt == last_draft_text or len(txt) < 2: + return + last_draft_text = txt + draft_count += 1 + await _send_event("draft", text=txt, is_final=False, isFinal=False, chunk=draft_count, bytes=len(snapshot_bytes)) + await _send_event("interim_transcript", text=txt, is_final=False, draft_count=draft_count) + if screen_url and len(txt) > 1: + try: + t = asyncio.create_task(self._send_screen_text(screen_url, txt, clear=False)) + self._background_tasks.add(t) + t.add_done_callback(self._background_tasks.discard) + except Exception: + pass + finally: + draft_in_progress = False + + await _send_event("ready", device_id=device_id, screen_url=screen_url or "", live_draft=live_enabled, live_model="apple-pipe-volatile") + + try: + raw_handle = raw_path.open("wb") + try: + async for msg in ws: + if msg.type == web.WSMsgType.TEXT: + try: + payload = json.loads(msg.data) + except json.JSONDecodeError: + await _send_event("error", code="bad_json", message="Text frames must be JSON") + continue + event = str(payload.get("event") or payload.get("type") or "").strip().lower() + if event == "start": + if started: + await _send_event("error", code="already_started", message="Audio stream already started") + continue + fmt = str(payload.get("format") or "pcm_s16le").strip().lower() + channels = int(payload.get("channels") or 1) + sample_width = int(payload.get("sample_width") or 2) + sample_rate = int(payload.get("sample_rate") or 16000) + device_id = str(payload.get("device_id") or device_id).strip()[:80] or device_id + if fmt not in {"pcm_s16le", "s16le", "pcm"} or channels != 1 or sample_width != 2: + await _send_event("error", code="unsupported_audio_format", message="Use 16 kHz mono signed 16-bit little-endian PCM") + await ws.close() + return ws + if sample_rate < 8000 or sample_rate > 48000: + await _send_event("error", code="bad_sample_rate", message="sample_rate must be 8000-48000") + await ws.close() + return ws + started = True + if not visual_listening_sent: + _schedule_voice_state("captioning", frames=8) + visual_listening_sent = True + await _send_event("listening", sample_rate=sample_rate, channels=channels, sample_width=sample_width) + elif event in {"stop", "end"}: + stopped = True + break + elif event in {"ping", "keepalive"}: + await _send_event("pong") + else: + await _send_event("error", code="unknown_event", message=f"Unknown event: {event or ''}") + elif msg.type == web.WSMsgType.BINARY: + if not started: + started = True + if not visual_listening_sent: + _schedule_voice_state("captioning", frames=8) + visual_listening_sent = True + await _send_event("listening", sample_rate=sample_rate, channels=channels, sample_width=sample_width) + chunk = bytes(msg.data or b"") + if not chunk: + continue + if first_audio_at is None: + first_audio_at = time.time() + total_audio_bytes += len(chunk) + pcm_buffer.extend(chunk) + if total_audio_bytes > ESP32_WS_MAX_AUDIO_BYTES: + await _send_event("error", code="audio_too_large", message=f"Audio stream exceeded {ESP32_WS_MAX_AUDIO_BYTES} bytes") + await ws.close() + return ws + raw_handle.write(chunk) + # live draft trigger (every 800ms & 8KB new data) — direct captioning fluff stripped + if live_enabled: + now = time.time() + elapsed_ms = (now - last_draft_at) * 1000 + new_bytes = len(pcm_buffer) - last_draft_bytes + total_ms = len(pcm_buffer) / 2 / sample_rate * 1000 + if total_ms >= ESP32_LIVE_DRAFT_MIN_TOTAL_MS and elapsed_ms >= ESP32_LIVE_DRAFT_INTERVAL_MS and new_bytes >= ESP32_LIVE_DRAFT_MIN_BYTES: + last_draft_at = now + last_draft_bytes = len(pcm_buffer) + # copy snapshot, fire-and-forget draft (but guard concurrent) + snapshot = bytes(pcm_buffer) + asyncio.create_task(_do_live_draft(snapshot)) + elif msg.type == web.WSMsgType.ERROR: + logger.warning("[api_server] ESP32 voice websocket error: %s", ws.exception()) + break + finally: + raw_handle.close() + + if not stopped: + await _send_event("error", code="missing_stop", message="Send a stop event after audio chunks") + await ws.close() + return ws + if total_audio_bytes < ESP32_WS_MIN_AUDIO_BYTES: + await _send_event("error", code="audio_too_short", message="Not enough audio received") + await ws.close() + return ws + + await asyncio.to_thread( + self._write_pcm_s16le_wav, + raw_path, + wav_path, + sample_rate=sample_rate, + channels=channels, + ) + audit_event.update({ + "status": "received", + "audio_bytes": total_audio_bytes, + "sample_rate": sample_rate, + "first_audio_latency_ms": int(((first_audio_at or started_at) - started_at) * 1000), + "stop_latency_ms": int((time.time() - started_at) * 1000), + "draft_count": draft_count, + "last_draft": last_draft_text, + }) + self._append_esp32_voice_audit(audio_dir, dict(audit_event)) + + _schedule_voice_state("captioning", frames=10) + await _send_event("thinking", stage="transcribing", draft_count=draft_count) + + # Parallel instant: start quick_reply from draft ASAP, but prefer full transcript for context + # Previously we used last_draft_text which was often just "Perfect. Perfect." leading to generic reply + t_quick_start = time.time() + instant_future = None + draft_for_llm = last_draft_text if last_draft_text and len(last_draft_text.strip()) > 2 else None + # Only start early if draft seems substantial (>30 chars and not just generic starter) - else wait for final transcript + if draft_for_llm and ESP32_LIVE_DRAFT_USE_MACMINI and len(draft_for_llm.strip()) > 30: + try: + instant_future = asyncio.create_task( + self._apple_llm_quick_reply_via_macmini( + draft=draft_for_llm, + context="", + instructions=request.headers.get("X-Hermes-Instructions") or "" + ) + ) + except Exception: + instant_future = None + + # Final transcription: prefer Mac mini live pipe (ApplePipeTranscriber) + transcript = None + final_source = "none" + t_transcribe_start = time.time() + + try: + if ESP32_LIVE_DRAFT_USE_MACMINI and len(pcm_buffer) > 0: + full_wav = self._s16le_to_wav_bytes(bytes(pcm_buffer), sample_rate=sample_rate, channels=channels) + mac_txt = await self._transcribe_draft_via_macmini(full_wav, locale=locale_req) + if mac_txt and len(mac_txt.strip()) >= 2: + transcript = mac_txt.strip() + final_source = "macmini-live-final" + if instant_future is None and mac_txt: + # Start instant from final if no draft was available earlier + try: + draft_for_llm = mac_txt[:200] + instant_future = asyncio.create_task( + self._apple_llm_quick_reply_via_macmini( + draft=mac_txt[:200], + context="", + instructions=request.headers.get("X-Hermes-Instructions") or "" + ) + ) + except Exception: + pass + except Exception as e: + logger.debug(f"[api_server] macmini final transcribe failed: {e}") + + if not transcript: + from tools.transcription_tools import transcribe_audio + try: + if not wav_path.exists(): + await asyncio.to_thread( + self._write_pcm_s16le_wav, + raw_path, + wav_path, + sample_rate=sample_rate, + channels=channels, + ) + except Exception: + pass + stt_result = await asyncio.to_thread(transcribe_audio, str(wav_path) if wav_path.exists() else str(raw_path)) + if not stt_result.get("success"): + message = stt_result.get("error") or "Transcription failed" + self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "transcription_failed", "error": message, "timestamp": time.time()}) + await _send_event("error", code="transcription_failed", message=message) + await ws.close() + return ws + transcript = str(stt_result.get("transcript") or "").strip() + final_source = "local-faster-whisper" + + if not transcript: + self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "empty_transcript", "timestamp": time.time()}) + await _send_event("error", code="empty_transcript", message="Transcription was empty") + await ws.close() + return ws + + transcribe_ms = int((time.time() - t_transcribe_start) * 1000) + + self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "transcribed", "transcript": transcript, "timestamp": time.time(), "draft_count": draft_count, "final_source": final_source, "transcribe_ms": transcribe_ms}) + await _send_event("transcript", text=transcript, is_final=True, isFinal=True, draft_count=draft_count, source=final_source, transcribe_ms=transcribe_ms) + await _send_event("final", text=transcript, isFinal=True, source=final_source, transcribe_ms=transcribe_ms) + + summary = self._short_voice_summary(transcript, 180) + + # Now await instant that was started in parallel on stop + instant_reply = None + instant_ok = False + quick_ms = 0 + if draft_for_llm is None or len(draft_for_llm.strip()) < 20: + draft_for_llm = transcript[:300] + # For contextual reply, always feed final transcript if draft was short/generic + draft_to_use_for_llm = transcript[:400] if (not draft_for_llm or len(transcript) > len(draft_for_llm) or len(draft_for_llm.strip()) < 30) else draft_for_llm + try: + if instant_future is not None: + try: + instant_reply = await asyncio.wait_for(instant_future, timeout=3.0) + # If instant from short draft is generic, discard and regenerate from full transcript + if instant_reply and len(draft_for_llm.strip()) < 30 and len(transcript) > len(draft_for_llm): + # Prefer transcript-based reply for context + instant_reply = None + except asyncio.TimeoutError: + instant_reply = None + try: + instant_future.cancel() + except Exception: + pass + if not instant_reply and ESP32_LIVE_DRAFT_USE_MACMINI: + # No draft, or future failed — try once more from final transcript (contextual) + instant_reply = await self._apple_llm_quick_reply_via_macmini( + draft=draft_to_use_for_llm[:400], + context="", + instructions=request.headers.get("X-Hermes-Instructions") or "" + ) + if instant_reply: + quick_ms = int((time.time() - t_quick_start) * 1000) + instant_ok = True + await _send_event("instant_response", text=instant_reply, is_quick=True, quick_ms=quick_ms, draft=draft_for_llm[:200], source="apple-llm-3b-ane", final=False, persistent=True) + await _send_event("quick_reply", text=instant_reply, ms=quick_ms, persistent=True) + await _send_event("response_text", text=instant_reply, response_id="", final=False, is_instant=True, quick_ms=quick_ms, source="apple-llm-3b-ane") + # Also speak instant reply via Mac mini voicebox/boy voice while Hermes generates full answer + # This gives user audio feedback immediately instead of silence + try: + # First try fast Mac say Alex (boy voice) for speed <500ms, voicebox Aiden if available but slower + t_speak_start = time.time() + # Try Aiden voicebox first if you prefer quality, else Alex for speed. Config: try Alex first for speed. + audio_b64 = await self._speak_instant_via_macmini(instant_reply, voice="Alex", use_voicebox=False) + if not audio_b64: + # Fallback try voicebox Aiden boy voice (Qwen 1.7B) if say failed or you prefer quality + try: + audio_b64 = await self._speak_instant_via_macmini(instant_reply, voice="Aiden", use_voicebox=True) + except Exception: + pass + if audio_b64 and screen_url: + speak_ms = int((time.time() - t_speak_start) * 1000) + # Send audio to ESP32 screen via play_audio_base64 - this was your request: hear fast reply while Hermes thinks + try: + # Use play_audio_base64 (WAV 16k mono) - Mac say already 16k + audio_task = asyncio.create_task( + self._call_screen_mcp( + screen_url, + "play_audio_base64", + {"wav_base64": audio_b64, "volume": 80}, + timeout=20, + ) + ) + self._background_tasks.add(audio_task) + audio_task.add_done_callback(self._background_tasks.discard) + except Exception as e: + logger.debug(f"[api_server] instant audio play failed: {e}") + await _send_event("instant_audio", text=instant_reply, quick_ms=quick_ms, speak_ms=speak_ms, voice="Alex/Aiden boy via Mac mini voicebox", source="macmini-say/voicebox") + self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "instant_audio", "speak_ms": speak_ms, "timestamp": time.time()}) + except Exception as e: + logger.debug(f"[api_server] instant speak failed: {e}") + + if screen_url: + try: + task = asyncio.create_task(self._send_screen_text(screen_url, instant_reply, clear=True)) + self._background_tasks.add(task) + task.add_done_callback(self._background_tasks.discard) + except Exception: + pass + self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "instant_reply", "instant_reply": instant_reply, "quick_ms": quick_ms, "timestamp": time.time()}) + except Exception as e: + logger.debug(f"[api_server] instant reply failed: {e}") + + if not instant_ok: + ack_text = f"Heard: {summary}\n\nWorking on the full answer now." + await _send_event("response_text", text=ack_text, response_id="", final=False, is_fallback=True) + + self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "queued_background", "transcript": transcript, "timestamp": time.time(), "instant_ok": instant_ok}) + + try: + await _send_event("done", mode="background", screen_url=screen_url or "", draft_count=draft_count, source=final_source, instant_reply=instant_reply or "", instant_ok=instant_ok, quick_ms=quick_ms, transcribe_ms=transcribe_ms) + if instant_ok: + await _send_event("thinking", stage="full_answer", keep_instant=True, instant_text=instant_reply, persistent_instant=True) + await ws.close() + except Exception: + pass + background_task = asyncio.create_task( + self._run_esp32_voice_background( + transcript=transcript, + device_id=device_id, + instructions=request.headers.get("X-Hermes-Instructions") or None, + audio_dir=audio_dir, + screen_url=screen_url, + ) + ) + try: + self._background_tasks.add(background_task) + except TypeError: + pass + if hasattr(background_task, "add_done_callback"): + background_task.add_done_callback(self._background_tasks.discard) + return ws + except Exception as exc: + logger.exception("[api_server] ESP32 voice websocket processing failed") + self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "websocket_exception", "error": str(exc), "timestamp": time.time()}) + try: + await _send_event("error", code="internal_error", message=str(exc)) + await ws.close() + except Exception: + pass + return ws + finally: + try: + raw_path.unlink(missing_ok=True) + except Exception: + pass + + async def _handle_esp32_voice(self, request: "web.Request") -> "web.Response": + + """POST /api/esp32/voice — receive audio, run Hermes, return TTS audio.""" + auth_err = self._check_auth(request) + if auth_err: + return auth_err + + device_id = ( + request.headers.get("X-Device-ID") + or request.query.get("device_id") + or "default" + ).strip()[:80] or "default" + + try: + audio_bytes = await request.read() + except Exception as exc: + return web.json_response(_openai_error(f"Failed to read audio body: {exc}"), status=400) + + if not audio_bytes: + return web.json_response(_openai_error("Empty audio body", code="empty_audio"), status=400) + if len(audio_bytes) > ESP32_AUDIO_MAX_BYTES: + return web.json_response( + _openai_error( + f"Audio body is too large ({len(audio_bytes)} bytes); max is {ESP32_AUDIO_MAX_BYTES}", + code="audio_too_large", + ), + status=413, + ) + + suffix = self._audio_extension_for_content_type(request.headers.get("Content-Type", "")) + try: + from hermes_constants import get_hermes_dir + audio_dir = get_hermes_dir("cache/audio", "audio_cache") / "esp32" + except Exception: + audio_dir = Path.home() / ".hermes" / "cache" / "audio" / "esp32" + audio_dir.mkdir(parents=True, exist_ok=True) + safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" + audio_path = audio_dir / f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}{suffix}" + audio_path.write_bytes(audio_bytes) + + audit_ctx = self._request_audit_context(request) + audit_event: Dict[str, Any] = { + "timestamp": time.time(), + "device_id": device_id, + "source_ip": audit_ctx.get("remote") or audit_ctx.get("peer_ip") or "", + "peer_ip": audit_ctx.get("peer_ip") or "", + "forwarded_for": audit_ctx.get("forwarded_for") or "", + "real_ip": audit_ctx.get("real_ip") or "", + "user_agent": audit_ctx.get("user_agent") or "", + "reply_mode": self._esp32_reply_mode(request), + "screen_device": request.headers.get("X-Hermes-Screen-Device") or request.query.get("screen_device") or "", + "screen_url": request.headers.get("X-Hermes-Screen-Url") or request.headers.get("X-Screen-MCP-Url") or request.query.get("screen_url") or "", + "content_type": request.headers.get("Content-Type", ""), + "audio_bytes": len(audio_bytes), + "audio_path": str(audio_path), + "status": "received", + } + + def _append_audit(event: Dict[str, Any]) -> None: + try: + log_path = audio_dir / "voice_requests.jsonl" + with log_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(event, default=str) + "\n") + except Exception: + logger.debug("[api_server] failed to append ESP32 voice audit log", exc_info=True) + + _append_audit(audit_event) + + if self._esp32_reply_mode(request) == "ack": + screen_url = self._resolve_esp32_screen_url(request, device_id) + + async def _ack_background() -> None: + try: + await self._send_screen_animation(screen_url, "ack") + await self._send_screen_animation(screen_url, "captioning") + from tools.transcription_tools import transcribe_audio + stt_result_bg = await asyncio.wait_for( + asyncio.to_thread(transcribe_audio, str(audio_path)), + timeout=45, + ) + if not stt_result_bg.get("success"): + failed_event = dict(audit_event) + failed_event.update({ + "status": "transcription_failed", + "error": stt_result_bg.get("error") or "Transcription failed", + "timestamp": time.time(), + }) + _append_audit(failed_event) + await self._send_screen_text( + screen_url, + "I heard audio, but transcription failed. Try again.", + clear=True, + ) + return + transcript_bg = str(stt_result_bg.get("transcript") or "").strip() + if not transcript_bg: + empty_event = dict(audit_event) + empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) + _append_audit(empty_event) + await self._send_screen_text( + screen_url, + "I heard audio, but got no words. Try again.", + clear=True, + ) + return + transcript_event = dict(audit_event) + transcript_event.update({"status": "transcribed", "transcript": transcript_bg, "timestamp": time.time()}) + _append_audit(transcript_event) + await self._run_esp32_voice_background( + transcript=transcript_bg, + device_id=device_id, + instructions=request.headers.get("X-Hermes-Instructions") or None, + audio_dir=audio_dir, + screen_url=screen_url, + ) + except asyncio.TimeoutError: + timeout_event = dict(audit_event) + timeout_event.update({"status": "transcription_timeout", "error": "Transcription exceeded 45s", "timestamp": time.time()}) + _append_audit(timeout_event) + await self._send_screen_text( + screen_url, + "I heard audio, but captioning timed out. Try again.", + clear=True, + ) + except Exception as exc: + logger.exception("[api_server] ESP32 ack-mode background processing failed") + failed_event = dict(audit_event) + failed_event.update({"status": "background_exception", "error": str(exc), "timestamp": time.time()}) + _append_audit(failed_event) + + task = asyncio.create_task(_ack_background()) + try: + self._background_tasks.add(task) + except TypeError: + pass + if hasattr(task, "add_done_callback"): + task.add_done_callback(self._background_tasks.discard) + ack_path = await self._esp32_ack_wav(audio_dir=audio_dir, device_id=device_id) + headers = { + "X-Hermes-Conversation": f"esp32:{device_id}", + "X-Hermes-Reply-Mode": "ack", + "X-Hermes-Screen-Url": screen_url or "", + "Content-Type": "audio/wav", + } + return web.FileResponse(ack_path, headers=headers) + + try: + from tools.transcription_tools import transcribe_audio + stt_result = await asyncio.to_thread(transcribe_audio, str(audio_path)) + except Exception as exc: + logger.exception("[api_server] ESP32 transcription failed") + failed_event = dict(audit_event) + failed_event.update({"status": "transcription_exception", "error": str(exc), "timestamp": time.time()}) + _append_audit(failed_event) + return web.json_response(_openai_error(f"Transcription failed: {exc}"), status=500) + + if not stt_result.get("success"): + failed_event = dict(audit_event) + failed_event.update({ + "status": "transcription_failed", + "error": stt_result.get("error") or "Transcription failed", + "timestamp": time.time(), + }) + _append_audit(failed_event) + return web.json_response( + _openai_error(stt_result.get("error") or "Transcription failed", code="transcription_failed"), + status=502, + ) + + transcript = str(stt_result.get("transcript") or "").strip() + if not transcript: + empty_event = dict(audit_event) + empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) + _append_audit(empty_event) + return web.json_response(_openai_error("Transcription was empty", code="empty_transcript"), status=422) + + transcript_event = dict(audit_event) + transcript_event.update({"status": "transcribed", "transcript": transcript, "timestamp": time.time()}) + _append_audit(transcript_event) + + instructions = request.headers.get("X-Hermes-Instructions") or None + try: + response_data, final_text = await self._run_esp32_voice_turn( + transcript=transcript, + device_id=device_id, + instructions=instructions, + ) + except Exception as exc: + logger.exception("[api_server] ESP32 agent turn failed") + return web.json_response(_openai_error(f"Agent turn failed: {exc}"), status=500) + + if not final_text.strip(): + final_text = self._extract_response_text(response_data) or "I heard you, but I do not have a spoken answer." + + try: + from tools.tts_tool import text_to_speech_tool + tts_raw = await asyncio.to_thread(text_to_speech_tool, final_text) + tts_result = json.loads(tts_raw) + except Exception as exc: + logger.exception("[api_server] ESP32 TTS failed") + return web.json_response( + _openai_error(f"TTS failed after response was generated: {exc}", code="tts_failed"), + status=500, + headers={ + "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), + "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), + }, + ) + + if not tts_result.get("success"): + return web.json_response( + _openai_error(tts_result.get("error") or "TTS failed", code="tts_failed"), + status=502, + headers={ + "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), + "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), + }, + ) + + tts_path = str(tts_result.get("file_path") or "") + if not tts_path or not Path(tts_path).exists(): + return web.json_response( + _openai_error("TTS succeeded but no audio file was produced", code="tts_file_missing"), + status=500, + ) + + try: + response_audio_path = self._convert_audio_to_esp32_wav( + tts_path, + output_dir=audio_dir, + device_id=device_id, + ) + except Exception as exc: + logger.exception("[api_server] ESP32 WAV normalization failed") + return web.json_response( + _openai_error(f"Could not convert TTS audio to ESP32 WAV: {exc}", code="wav_conversion_failed"), + status=500, + headers={ + "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), + "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), + }, + ) + + headers = { + "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), + "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), + "X-Hermes-Response-Id": str(response_data.get("id") or ""), + "X-Hermes-Conversation": f"esp32:{device_id}", + "X-Hermes-TTS-Source-Type": self._audio_response_mime(tts_path), + "Content-Type": "audio/wav", + } + return web.FileResponse( + response_audio_path, + headers=headers, + ) + diff --git a/hermes_plugin/__init__.py b/hermes_plugin/__init__.py new file mode 100644 index 0000000..0f14a19 --- /dev/null +++ b/hermes_plugin/__init__.py @@ -0,0 +1,304 @@ +""" +Reyna family ESP32 Voice Gateway - durable Hermes plugin +Survives `hermes update` which wipes in-tree patches to gateway/platforms/api_server.py. + +What it does: +- On Hermes import (plugin load), checks if api_server.py has ESP32 routes. +- If missing (after update), auto-restores from bundled patch + logs actionable warning. +- Exposes tools for manual verify/restore from inside Hermes chat (via slash or tool). + +The actual voice logic stays in api_server.py (patched file) because it's 1600+ lines +deeply integrated with APIServerAdapter internals (ResponseStore, _run_agent, TTS, transcription, +screen MCP). Extracting as runtime monkeypatch would be brittle. The durable strategy is: + 1. Patch file lives in this repo + bundled in plugin as `patch/*.patch` + 2. Plugin's register() does self-heal check (fast ~10ms) + 3. Gateway auto-restarts handled by operator (can't restart self from inside gateway - see skill pitfalls) + +Future: true runtime injection plugin that patches APIServerAdapter.connect to add routes +without touching file — feasible but needs tracking _handle_esp32_* methods. Kept separate as v2. +""" + +import logging +import subprocess +import sys +from pathlib import Path +from typing import Optional + +logger = logging.getLogger(__name__) + +PLUGIN_DIR = Path(__file__).parent +PATCH_DIR = PLUGIN_DIR / "patch" +# repo root for dev checkout (symlinked plugin) +REPO_PATCH = PLUGIN_DIR.parent / "patch" / "api_server_esp32_live_draft_with_llm.patch" + + +def _find_hermes_agent_dir() -> Optional[Path]: + # Try standard locations + candidates = [ + Path.home() / ".hermes" / "hermes-agent", + Path(__file__).resolve().parents[3], # if running from within hermes-agent/plugins/esp32-... + ] + # Also check HERMES_AGENT_DIR env + import os + env_dir = os.getenv("HERMES_AGENT_DIR") + if env_dir: + candidates.insert(0, Path(env_dir)) + # pip install location + try: + import hermes_cli + candidates.append(Path(hermes_cli.__file__).parents[1]) + except Exception: + pass + for c in candidates: + if (c / "gateway" / "platforms" / "api_server.py").exists(): + return c + return None + + +def _api_server_has_esp32() -> bool: + agent_dir = _find_hermes_agent_dir() + if not agent_dir: + return False + api_file = agent_dir / "gateway" / "platforms" / "api_server.py" + try: + txt = api_file.read_text()[:5000] # top matter + first routes + # quick check + if "ESP32_AUDIO_MAX_BYTES" not in api_file.read_text()[:20000]: + return False + # full route registration present? + content = api_file.read_text() + return "/api/esp32/voice" in content and "_handle_esp32_voice_ws" in content + except Exception: + return False + + +def _find_patch_file() -> Optional[Path]: + # Search order: bundled patch in plugin, repo patch, hermes-home fallback + for p in [ + PATCH_DIR / "api_server_esp32_live_draft_with_llm.patch", + REPO_PATCH, + Path.home() / ".hermes" / "esp32_live_draft_with_llm_patch.patch", + Path.home() / ".hermes" / "esp32_voice_patch.patch", + PATCH_DIR / "esp32_live_draft_with_llm_patch.patch", + ]: + if p.exists(): + return p + # Also search inside plugin dir itself (dev checkout) + for pattern in ["*.patch"]: + for f in PATCH_DIR.glob(pattern): + if f.stat().st_size > 1000: + return f + return None + + +def _try_apply_patch(patch_path: Path, agent_dir: Path) -> bool: + """Apply patch via git apply. Returns True if applied or already applied.""" + api_file = agent_dir / "gateway" / "platforms" / "api_server.py" + if not api_file.exists(): + logger.warning("[esp32-voice-gateway] api_server.py not found at %s", api_file) + return False + # Check if git + git_dir = agent_dir / ".git" + if not git_dir.exists(): + logger.warning("[esp32-voice-gateway] not a git repo: %s", agent_dir) + return False + # Try git apply --check first + try: + # if already patched, check will fail with "already applied" type – we detect via has_esp32 already + res = subprocess.run( + ["git", "apply", "--check", str(patch_path)], + cwd=str(agent_dir), + capture_output=True, + text=True, + timeout=15, + ) + if res.returncode != 0: + # Could be already applied or conflict. Log but try to see if content already has marker + if "ESP32_AUDIO_MAX_BYTES" not in api_file.read_text()[:20000]: + logger.warning("[esp32-voice-gateway] patch check failed: %s %s", res.stdout[:500], res.stderr[:800]) + # Try --3way if possible + res2 = subprocess.run( + ["git", "apply", "--3way", str(patch_path)], + cwd=str(agent_dir), + capture_output=True, + text=True, + timeout=30, + ) + if res2.returncode != 0: + logger.error("[esp32-voice-gateway] git apply --3way failed: %s %s", res2.stdout[-500:], res2.stderr[-1000:]) + return False + else: + # already has marker, consider applied + logger.info("[esp32-voice-gateway] patch already present (has ESP32 marker)") + return True + else: + res_apply = subprocess.run( + ["git", "apply", str(patch_path)], + cwd=str(agent_dir), + capture_output=True, + text=True, + timeout=30, + ) + if res_apply.returncode != 0: + logger.error("[esp32-voice-gateway] git apply failed: %s %s", res_apply.stdout[-500:], res_apply.stderr[-1000:]) + return False + # Verify + if _api_server_has_esp32(): + logger.info("[esp32-voice-gateway] Successfully restored ESP32 routes from %s -> %s", patch_path, api_file) + return True + else: + logger.warning("[esp32-voice-gateway] Patch applied but route marker still missing – need manual check") + return False + except Exception as e: + logger.exception("[esp32-voice-gateway] Exception applying patch %s: %s", patch_path, e) + return False + + +def restore_if_needed() -> dict: + """Check and auto-restore if needed. Safe to call multiple times.""" + result = {"had_routes": False, "restored": False, "patch": None, "agent_dir": None, "error": None} + try: + agent_dir = _find_hermes_agent_dir() + result["agent_dir"] = str(agent_dir) if agent_dir else None + if not agent_dir: + result["error"] = "hermes-agent dir not found" + return result + if _api_server_has_esp32(): + result["had_routes"] = True + return result + # missing -> try restore + patch_file = _find_patch_file() + result["patch"] = str(patch_file) if patch_file else None + if not patch_file: + result["error"] = "no patch file found (checked patch/*.patch + ~/.hermes/esp32*.patch)" + logger.error("[esp32-voice-gateway] Routes missing after update and no patch file found!") + return result + logger.warning("[esp32-voice-gateway] ESP32 routes missing (likely after `hermes update`) – auto-restoring from %s", patch_file) + ok = _try_apply_patch(patch_file, agent_dir) + result["restored"] = ok + if ok: + # also copy patch to ~/.hermes for next fallback + try: + dest = Path.home() / ".hermes" / "esp32_live_draft_with_llm_patch.patch" + if patch_file.resolve() != dest.resolve(): + import shutil + shutil.copy2(patch_file, dest) + except Exception: + pass + return result + except Exception as e: + result["error"] = str(e) + logger.exception("[esp32-voice-gateway] restore_if_needed failed") + return result + + +# --- Plugin registration --- + +def register(ctx): + """Hermes plugin entry point.""" + # On import, do fast self-heal check (10ms if routes present) + info = restore_if_needed() + + # Register a tool so user can ask inside Hermes: "check ESP32 voice gateway" + import json + + def _tool_check_esp32_gateway(args, **kwargs) -> str: + res = restore_if_needed() + # also run live curl checks if gateway running + checks = {} + try: + import socket + # try to find api_server port from config + port = 8642 + try: + from hermes_cli.config import load_config + cfg = load_config() + g_port = ((cfg.get("gateway") or {}).get("api_server") or {}).get("port") + p_port = ((cfg.get("platforms") or {}).get("api_server") or {}).get("port") + raw_port = g_port or p_port or port + port = int(raw_port) + except Exception: + pass + # quick TCP check + s = socket.socket() + s.settimeout(1.0) + try: + s.connect(("127.0.0.1", port)) + checks["tcp"] = f"127.0.0.1:{port} open" + except Exception as e: + checks["tcp"] = f"closed: {e}" + finally: + s.close() + except Exception as e: + checks["tcp_error"] = str(e) + + return json.dumps({ + "had_routes_before": info.get("had_routes"), + "restored": info.get("restored"), + "patch_used": info.get("patch"), + "agent_dir": info.get("agent_dir"), + "error": info.get("error"), + "has_routes_now": _api_server_has_esp32(), + "checks": checks, + "verify_curl": f'curl -H "Authorization: Bearer $API_SERVER_KEY" http://127.0.0.1:8642/api/esp32/voice/ws -v 2>&1 | grep "< HTTP" -> want 400 not 404' + }, indent=2) + + def _tool_restore_esp32_gateway(args, **kwargs) -> str: + res = restore_if_needed() + # force reapply even if has routes if requested + force = bool(args.get("force")) + if force: + patch = _find_patch_file() + agent_dir = _find_hermes_agent_dir() + if patch and agent_dir: + # reset file first then reapply? just apply + ok = _try_apply_patch(patch, agent_dir) + res["force_restored"] = ok + return json.dumps(res, indent=2) + + ctx.register_tool( + name="esp32_voice_gateway_status", + toolset="esp32_voice", + schema={ + "name": "esp32_voice_gateway_status", + "description": "Check if ESP32 voice gateway routes (POST /api/esp32/voice + WS /api/esp32/voice/ws) are present after hermes update. Auto-restores from bundled patch if missing.", + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + handler=_tool_check_esp32_gateway, + description="Check ESP32 voice gateway route status and auto-restore after update", + ) + + ctx.register_tool( + name="esp32_voice_gateway_restore", + toolset="esp32_voice", + schema={ + "name": "esp32_voice_gateway_restore", + "description": "Force restore ESP32 voice gateway routes from bundled patch after hermes update wipes api_server.py", + "parameters": { + "type": "object", + "properties": { + "force": {"type": "boolean", "description": "Force re-apply even if routes seem present"}, + }, + "required": [], + }, + }, + handler=_tool_restore_esp32_gateway, + description="Force restore ESP32 routes from patch", + ) + + def _on_startup(session_id=None, **kw): + # Gateway hook – log status on startup for visibility + try: + if _api_server_has_esp32(): + logger.info("[esp32-voice-gateway] Routes OK: /api/esp32/voice + /api/esp32/voice/ws present") + else: + logger.warning("[esp32-voice-gateway] Routes MISSING at startup – will try restore on next tool call; if you just ran `hermes update`, run: ~/Projects/hermes-esp32-voice-gateway/scripts/restore_after_update.py && hermes gateway restart") + except Exception: + pass + + try: + ctx.register_hook("on_session_start", _on_startup) + except Exception: + pass + + logger.info("[esp32-voice-gateway] Plugin loaded (had_routes=%s restored=%s has_now=%s)", info.get("had_routes"), info.get("restored"), _api_server_has_esp32()) diff --git a/hermes_plugin/plugin.yaml b/hermes_plugin/plugin.yaml new file mode 100644 index 0000000..80f70d7 --- /dev/null +++ b/hermes_plugin/plugin.yaml @@ -0,0 +1,8 @@ +name: esp32-voice-gateway +version: "1.0.0" +description: "Reyna family ESP32 + iPhone Watch voice gateway - auto-restores custom api_server.py routes after hermes update (POST /api/esp32/voice + WS /api/esp32/voice/ws + live draft + Apple 3B instant reply)" +author: "Adolfo Reyna" +kind: standalone +provides_tools: [] +provides_hooks: + - on_session_start diff --git a/patch/api_server_esp32_live_draft_with_llm.patch b/patch/api_server_esp32_live_draft_with_llm.patch new file mode 100644 index 0000000..0655c9e --- /dev/null +++ b/patch/api_server_esp32_live_draft_with_llm.patch @@ -0,0 +1,1675 @@ +diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py +index 628713224..32bd47a6c 100644 +--- a/gateway/platforms/api_server.py ++++ b/gateway/platforms/api_server.py +@@ -34,6 +34,7 @@ Requires: + import asyncio + import hashlib + import hmac ++import ipaddress + import json + from contextlib import contextmanager + from contextvars import ContextVar +@@ -47,6 +48,7 @@ import time + import uuid + from pathlib import Path + from typing import Any, Dict, List, Optional ++from urllib.parse import urlparse + + def _approval_event_choices(*, smart_denied: bool, allow_permanent: bool) -> list[str]: + if smart_denied: +@@ -106,6 +108,47 @@ CHAT_COMPLETIONS_SSE_KEEPALIVE_SECONDS = 30.0 + MAX_NORMALIZED_TEXT_LENGTH = 65_536 # 64 KB cap for normalized content parts + MAX_CONTENT_LIST_SIZE = 1_000 # Max items when content is an array + ++ESP32_AUDIO_MAX_BYTES = 8_000_000 # ~4 minutes of 16 kHz mono PCM WAV; enough for push-to-talk tests ++ESP32_WS_MAX_AUDIO_BYTES = 8_000_000 ++ESP32_WS_MIN_AUDIO_BYTES = 1_600 # 50 ms at 16 kHz mono s16le; rejects empty accidental taps ++ESP32_WS_TTS_CHUNK_BYTES = 16_384 ++ ++# Live draft ingestion config (from whisper-translation/apple_speech engine) ++ESP32_LIVE_DRAFT_ENABLED = os.getenv("ESP32_LIVE_DRAFT_ENABLED", "1").lower() not in {"0", "false", "no", "off"} ++ESP32_LIVE_DRAFT_INTERVAL_MS = int(os.getenv("ESP32_LIVE_DRAFT_INTERVAL_MS", "800")) # how often to snapshot for draft ++ESP32_LIVE_DRAFT_MIN_BYTES = int(os.getenv("ESP32_LIVE_DRAFT_MIN_BYTES", "8000")) # min new bytes since last draft ++ESP32_LIVE_DRAFT_MIN_TOTAL_MS = int(os.getenv("ESP32_LIVE_DRAFT_MIN_TOTAL_MS", "500")) # don't draft until we have 500ms ++ESP32_MACMINI_MCP_URL = os.getenv("MACMINI_MCP_URL") or None # overridable, else from config.yaml mcp_servers.macmini ++ESP32_MACMINI_MCP_TOKEN = os.getenv("MACMINI_MCP_TOKEN") or None ++ESP32_LIVE_DRAFT_USE_MACMINI = os.getenv("ESP32_LIVE_DRAFT_USE_MACMINI", "1").lower() not in {"0", "false", "no", "off"} ++ ++_ESP32_AUDIO_EXTENSIONS = { ++ "audio/wav": ".wav", ++ "audio/wave": ".wav", ++ "audio/x-wav": ".wav", ++ "audio/mpeg": ".mp3", ++ "audio/mp3": ".mp3", ++ "audio/ogg": ".ogg", ++ "audio/opus": ".ogg", ++ "audio/webm": ".webm", ++ "audio/mp4": ".m4a", ++ "audio/x-m4a": ".m4a", ++ "application/octet-stream": ".wav", ++} ++ ++_AUDIO_RESPONSE_MIME_BY_SUFFIX = { ++ ".mp3": "audio/mpeg", ++ ".wav": "audio/wav", ++ ".ogg": "audio/ogg", ++ ".opus": "audio/ogg", ++ ".m4a": "audio/mp4", ++ ".mp4": "audio/mp4", ++ ".flac": "audio/flac", ++} ++ ++ESP32_VOICE_ANIMATION_PATH = "hermes_voice_animations.py" ++ESP32_VOICE_ANIMATION_SCRIPT = "# Hermes ESP32 voice-state animations for MicroPython screen boards.\n# Uploaded by the API server; can also be run manually with:\n# exec(open('hermes_voice_animations.py').read()); run_state('ack')\nimport time\ntry:\n import board_config\nexcept Exception:\n board_config = None\n\n\ndef _display():\n return getattr(board_config, 'display_instance', None) if board_config else None\n\n\ndef _sleep(ms):\n try:\n time.sleep_ms(ms)\n except AttributeError:\n time.sleep(ms / 1000)\n\n\ndef _text(d, msg, x, y, scale=1, color=1):\n if hasattr(d, 'text_large') and scale > 1:\n try:\n d.text_large(msg, x, y, scale=scale, color=color)\n return\n except TypeError:\n pass\n d.text(msg, x, y, color)\n\n\ndef _center_text(d, msg, y, scale=1, color=1):\n width = getattr(d, 'width', 320)\n char_w = 8 * scale\n x = max(0, (width - len(msg) * char_w) // 2)\n _text(d, msg, x, y, scale=scale, color=color)\n\n\ndef _wrap(text, width_chars=34, max_lines=4):\n words = str(text or '').replace('\\n', ' ').split()\n lines, line = [], ''\n for w in words:\n candidate = (line + ' ' + w).strip()\n if len(candidate) > width_chars and line:\n lines.append(line)\n line = w\n if len(lines) >= max_lines:\n break\n else:\n line = candidate\n if line and len(lines) < max_lines:\n lines.append(line)\n return lines\n\n\ndef _spinner(d, title, subtitle='', frames=24, delay=70):\n width, height = getattr(d, 'width', 320), getattr(d, 'height', 240)\n cx, cy = width // 2, height // 2 - 8\n spokes = [(-22,0),(-16,-16),(0,-22),(16,-16),(22,0),(16,16),(0,22),(-16,16)]\n for frame in range(frames):\n d.clear(0)\n _center_text(d, title, 18, scale=2)\n for i, (dx, dy) in enumerate(spokes):\n color = 1 if (i + frame) % len(spokes) in (0, 1, 2, 3) else 0\n size = 4 if i == frame % len(spokes) else 2\n d.fill_rect(cx + dx - size // 2, cy + dy - size // 2, size, size, color)\n d.rect(8, height - 34, width - 16, 18, 1)\n d.fill_rect(10, height - 32, ((width - 20) * ((frame % 16) + 1)) // 16, 14, 1)\n if subtitle:\n _center_text(d, subtitle[:32], height - 56, scale=1)\n d.show()\n _sleep(delay)\n\n\ndef ack(frames=18):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'GOT IT', 18, scale=2)\n # expanding check pulse\n box = 48 + (i % 6) * 8\n x, y = width // 2 - box // 2, height // 2 - box // 2\n d.rect(x, y, box, box, 1)\n d.line(width//2 - 24, height//2, width//2 - 6, height//2 + 18, 1)\n d.line(width//2 - 6, height//2 + 18, width//2 + 30, height//2 - 22, 1)\n _center_text(d, 'Listening captured', height - 36, scale=1)\n d.show()\n _sleep(55)\n\n\ndef captioning(frames=36):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'CAPTIONING', 16, scale=2)\n for bar in range(9):\n h = 8 + ((i * 5 + bar * 13) % 42)\n x = width // 2 - 72 + bar * 18\n d.fill_rect(x, height // 2 - h // 2, 10, h, 1)\n dots = '.' * ((i // 5) % 4)\n _center_text(d, 'Transcribing' + dots, height - 42, scale=1)\n d.show()\n _sleep(65)\n\n\ndef waiting(text='', frames=44):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'THINKING', 12, scale=2)\n r = 18 + (i % 10) * 2\n cx, cy = width // 2, height // 2 - 10\n d.rect(cx - r, cy - r, r * 2, r * 2, 1)\n d.rect(cx - r//2, cy - r//2, r, r, 1)\n if text:\n d.text('Heard:', 10, height - 68, 1)\n for idx, line in enumerate(_wrap(text, 36, 3)):\n d.text(line[:38], 10, height - 54 + idx * 12, 1)\n else:\n _center_text(d, 'Waiting for reply...', height - 42, scale=1)\n d.show()\n _sleep(75)\n\n\ndef run_state(state='ack', text='', frames=None):\n state = str(state or 'ack').lower()\n if state in ('ack', 'listening', 'got_it'):\n ack(frames or 18)\n elif state in ('caption', 'captioning', 'transcribing'):\n captioning(frames or 36)\n elif state in ('wait', 'waiting', 'thinking', 'reply'):\n waiting(text, frames or 44)\n else:\n _spinner(_display(), 'HERMES', state[:28], frames or 20)\n" ++ + + def _coerce_port(value: Any, default: int = DEFAULT_PORT) -> int: + """Parse a listen port without letting malformed env/config values crash startup.""" +@@ -1585,6 +1628,1596 @@ class APIServerAdapter(BasePlatformAdapter): + "pid": os.getpid(), + }) + ++ @staticmethod ++ def _audio_extension_for_content_type(content_type: str) -> str: ++ """Map an inbound audio Content-Type to a safe cache-file suffix.""" ++ media_type = (content_type or "").split(";", 1)[0].strip().lower() ++ return _ESP32_AUDIO_EXTENSIONS.get(media_type, ".wav") ++ ++ @staticmethod ++ def _audio_response_mime(path: str) -> str: ++ """Return an HTTP Content-Type for a generated audio file path.""" ++ suffix = Path(path).suffix.lower() ++ return _AUDIO_RESPONSE_MIME_BY_SUFFIX.get(suffix, "application/octet-stream") ++ ++ @staticmethod ++ ++ def _convert_audio_to_esp32_wav(input_path: str, *, output_dir: Path, device_id: str) -> Path: ++ """Normalize generated TTS audio to ESP32-friendly WAV. ++ ++ Hermes TTS providers commonly emit MP3 (Edge) or OGG. The ESP32 test ++ client expects a small, predictable RIFF/WAVE file, so decode with ++ ffmpeg to raw PCM and rewrite the WAV container with Python's ``wave`` ++ module. That produces a minimal ``fmt`` + ``data`` WAV: mono, 16 kHz, ++ signed 16-bit little-endian PCM. ++ """ ++ import subprocess ++ import wave ++ ++ safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" ++ output_dir.mkdir(parents=True, exist_ok=True) ++ raw_path = output_dir / f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}.s16le" ++ wav_path = raw_path.with_suffix(".wav") ++ cmd = [ ++ "ffmpeg", ++ "-y", ++ "-hide_banner", ++ "-loglevel", ++ "error", ++ "-i", ++ input_path, ++ "-ac", ++ "1", ++ "-ar", ++ "16000", ++ "-f", ++ "s16le", ++ str(raw_path), ++ ] ++ try: ++ subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ++ raw = raw_path.read_bytes() ++ with wave.open(str(wav_path), "wb") as wav: ++ wav.setnchannels(1) ++ wav.setsampwidth(2) ++ wav.setframerate(16000) ++ wav.writeframes(raw) ++ return wav_path ++ finally: ++ try: ++ raw_path.unlink(missing_ok=True) ++ except Exception: ++ pass ++ ++ @staticmethod ++ def _safe_audio_header(value: Any, max_length: int) -> str: ++ """Make diagnostic text safe for HTTP response headers.""" ++ text = str(value or "")[:max_length] ++ return text.replace("\r", " ").replace("\n", " ").replace("\x00", " ") ++ ++ @staticmethod ++ def _esp32_reply_mode(request: "web.Request") -> str: ++ """Return 'sync' or 'ack' for the ESP32 voice response contract.""" ++ value = ( ++ request.headers.get("X-Hermes-Reply-Mode") ++ or request.query.get("reply_mode") ++ or request.query.get("mode") ++ or "sync" ++ ) ++ value = str(value).strip().lower() ++ if value in {"ack", "async", "background", "quick_ack", "quick-ack"}: ++ return "ack" ++ return "sync" ++ ++ @staticmethod ++ def _looks_like_http_url(value: str) -> bool: ++ try: ++ parsed = urlparse(value) ++ return parsed.scheme in {"http", "https"} and bool(parsed.netloc) ++ except Exception: ++ return False ++ ++ @staticmethod ++ def _load_local_device_url(device_name: str) -> Optional[str]: ++ """Best-effort local MCP screen lookup from ~/.hermes/local_devices.yaml.""" ++ if not device_name: ++ return None ++ try: ++ import yaml ++ registry = Path.home() / ".hermes" / "local_devices.yaml" ++ data = yaml.safe_load(registry.read_text()) if registry.exists() else {} ++ aliases = data.get("aliases") if isinstance(data, dict) else None ++ devices = data.get("devices") if isinstance(data, dict) else None ++ names = [str(device_name).strip(), str(device_name).strip().replace("-", "_")] ++ seen: set[str] = set() ++ while names: ++ name = names.pop(0) ++ if not name or name in seen: ++ continue ++ seen.add(name) ++ if isinstance(aliases, dict): ++ alias = aliases.get(name) ++ if isinstance(alias, str) and alias: ++ if alias.startswith("http"): ++ return alias ++ names.append(alias) ++ info = devices.get(name) if isinstance(devices, dict) else None ++ if isinstance(info, dict): ++ return str(info.get("fallback_url") or info.get("url") or "") or None ++ except Exception: ++ return None ++ return None ++ ++ @staticmethod ++ def _load_local_device_url_by_ip(source_ip: str) -> Optional[str]: ++ """Best-effort local MCP screen lookup by observed device IP. ++ ++ Some MCP-capable devices, notably the iPhone companion app, do not run ++ their MCP endpoint on port 80. When the voice gateway sees traffic from ++ one of those devices, prefer the registry URL over synthesizing ++ http:///api/mcp. ++ """ ++ value = str(source_ip or "").strip() ++ if not value: ++ return None ++ if value.startswith("::ffff:"): ++ value = value.removeprefix("::ffff:") ++ try: ++ import yaml ++ registry = Path.home() / ".hermes" / "local_devices.yaml" ++ data = yaml.safe_load(registry.read_text()) if registry.exists() else {} ++ devices = data.get("devices") if isinstance(data, dict) else None ++ if not isinstance(devices, dict): ++ return None ++ for info in devices.values(): ++ if not isinstance(info, dict): ++ continue ++ candidates = { ++ str(info.get("reserved_ip") or "").strip(), ++ str(info.get("host") or "").strip(), ++ } ++ for url_key in ("fallback_url", "url"): ++ raw_url = str(info.get(url_key) or "").strip() ++ if raw_url: ++ parsed = urlparse(raw_url) ++ if parsed.hostname: ++ candidates.add(parsed.hostname) ++ if value in candidates: ++ return str(info.get("fallback_url") or info.get("url") or "") or None ++ except Exception: ++ return None ++ return None ++ ++ @staticmethod ++ def _esp32_screen_url_from_source_ip(source_ip: str) -> Optional[str]: ++ """Build the MCP endpoint for the ESP32 that made this request. ++ ++ ESP32 boards can move between DHCP leases. For voice ACK follow-ups the ++ initiating board is the safest target, so prefer its observed peer IP ++ instead of a possibly stale local-device registry entry. ++ """ ++ value = str(source_ip or "").strip() ++ if not value: ++ return None ++ if value.startswith("::ffff:"): ++ value = value.removeprefix("::ffff:") ++ # Do not route to loopback/proxy/invalid addresses; registry fallback is ++ # better in those cases. ++ try: ++ ip = ipaddress.ip_address(value) ++ except ValueError: ++ return None ++ if ip.is_loopback or ip.is_unspecified or ip.is_multicast: ++ return None ++ host = f"[{value}]" if ip.version == 6 else value ++ return f"http://{host}/api/mcp" ++ ++ def _resolve_esp32_screen_url(self, request: "web.Request", device_id: str) -> Optional[str]: ++ """Resolve the screen MCP endpoint associated with this voice turn.""" ++ explicit = ( ++ request.headers.get("X-Hermes-Screen-Url") ++ or request.headers.get("X-Screen-MCP-Url") ++ or request.query.get("screen_url") ++ ) ++ if explicit and self._looks_like_http_url(explicit.strip()): ++ return explicit.strip() ++ ++ if str(device_id or "").strip().lower().startswith("iphone"): ++ iphone_url = self._load_local_device_url(str(device_id).strip()) ++ if iphone_url and self._looks_like_http_url(iphone_url): ++ return iphone_url ++ ++ audit_ctx = self._request_audit_context(request) ++ for source_ip in (audit_ctx.get("peer_ip"), audit_ctx.get("remote")): ++ registry_url = self._load_local_device_url_by_ip(source_ip or "") ++ if registry_url and self._looks_like_http_url(registry_url): ++ return registry_url ++ ++ audit_ctx = self._request_audit_context(request) ++ for source_ip in (audit_ctx.get("peer_ip"), audit_ctx.get("remote")): ++ source_url = self._esp32_screen_url_from_source_ip(source_ip or "") ++ if source_url: ++ return source_url ++ ++ screen_device = ( ++ request.headers.get("X-Hermes-Screen-Device") ++ or request.query.get("screen_device") ++ or device_id ++ ) ++ candidates = [screen_device, "little32", "esp32_screen"] ++ for candidate in candidates: ++ url = self._load_local_device_url(str(candidate).strip()) ++ if url and self._looks_like_http_url(url): ++ return url ++ return None ++ ++ async def _call_screen_mcp( ++ self, ++ screen_url: Optional[str], ++ tool_name: str, ++ arguments: Dict[str, Any], ++ *, ++ timeout: float = 8.0, ++ ) -> bool: ++ """Call a simple HTTP JSON-RPC MCP screen tool directly.""" ++ if not screen_url: ++ return False ++ try: ++ from aiohttp import ClientSession, ClientTimeout ++ payload = { ++ "jsonrpc": "2.0", ++ "id": uuid.uuid4().hex[:12], ++ "method": "tools/call", ++ "params": {"name": tool_name, "arguments": arguments}, ++ } ++ async with ClientSession(timeout=ClientTimeout(total=timeout)) as session: ++ async with session.post(screen_url, json=payload) as resp: ++ body = await resp.text() ++ if resp.status >= 400: ++ logger.warning( ++ "[api_server] screen MCP %s failed: HTTP %s %s", ++ tool_name, ++ resp.status, ++ body[:300], ++ ) ++ return False ++ try: ++ data = json.loads(body) if body else {} ++ if isinstance(data, dict) and data.get("error"): ++ logger.warning("[api_server] screen MCP %s returned error: %s", tool_name, data.get("error")) ++ return False ++ except Exception: ++ pass ++ return True ++ except Exception as exc: ++ logger.warning("[api_server] screen MCP %s failed: %s", tool_name, exc) ++ return False ++ ++ @staticmethod ++ def _screen_stream_target(screen_url: Optional[str]) -> Optional[tuple[str, int]]: ++ """Return the raw UDP video-stream target for MCP screens that expose one.""" ++ if not screen_url: ++ return None ++ try: ++ parsed = urlparse(screen_url) ++ host = parsed.hostname ++ if not host: ++ return None ++ return (host, 8082) ++ except Exception: ++ return None ++ ++ @staticmethod ++ def _voice_animation_udp_frames(state: str, text: str = "", frames: int = 18) -> List[bytes]: ++ """Render small Hermes voice-state animations as RLCD-compatible frames.""" ++ from PIL import Image, ImageDraw ++ import math ++ ++ width, height = 400, 300 ++ ++ def to_rlcd(img: "Image.Image") -> bytes: ++ img_1bit = img.convert("1", dither=Image.Dither.FLOYDSTEINBERG) ++ px = img_1bit.load() ++ buf = bytearray(15000) ++ for y in range(height): ++ for x in range(width): ++ if px[x, y]: ++ inv_y = 299 - y ++ bx = x // 2 ++ by = inv_y // 4 ++ idx = bx * 75 + by ++ lx = x % 2 ++ ly = inv_y % 4 ++ bit = 7 - (ly * 2 + lx) ++ buf[idx] |= (1 << bit) ++ return bytes(buf) ++ ++ label_by_state = { ++ "ack": "Got it", ++ "captioning": "Listening", ++ "waiting": "Thinking", ++ "speaking": "Replying", ++ "success": "Done", ++ "error": "Try again", ++ } ++ label = label_by_state.get(str(state), str(state or "Hermes").title()) ++ snippet = " ".join(str(text or "").split())[:42] ++ rendered: List[bytes] = [] ++ for frame in range(max(1, int(frames))): ++ img = Image.new("1", (width, height), 0) ++ draw = ImageDraw.Draw(img) ++ cx, cy = width // 2, height // 2 ++ phase = frame * 0.35 ++ if state == "captioning": ++ draw.rectangle((188, 38, 212, 75), outline=1) ++ draw.arc((175, 55, 225, 105), 20, 160, fill=1) ++ for x in range(42, width - 42, 5): ++ y = cy + int(32 * math.sin(x * 0.045 + phase * 2)) ++ y2 = cy + int(14 * math.sin(x * 0.085 - phase * 3)) ++ draw.line((x, y, min(width - 1, x + 4), y2), fill=1) ++ elif state in {"waiting", "speaking"}: ++ for radius in (34, 64, 94): ++ pulse = radius + int(8 * math.sin(phase + radius)) ++ draw.ellipse((cx - pulse, cy - pulse, cx + pulse, cy + pulse), outline=1) ++ for i in range(7): ++ x = 93 + i * 34 ++ bar = int(18 + 42 * (0.5 + 0.5 * math.sin(phase * 2 + i))) ++ draw.rectangle((x, 228 - bar, x + 17, 228), fill=1) ++ elif state == "error": ++ draw.line((150, 88, 250, 188), fill=1) ++ draw.line((250, 88, 150, 188), fill=1) ++ draw.rectangle((145, 82, 255, 194), outline=1) ++ else: ++ r = 42 + int(16 * math.sin(phase)) ++ draw.ellipse((cx - r, cy - r, cx + r, cy + r), outline=1) ++ draw.ellipse((cx - 8, cy - 8, cx + 8, cy + 8), fill=1) ++ draw.text((max(8, (width - len(label) * 6) // 2), 24), label, fill=1) ++ if snippet: ++ draw.text((max(8, (width - len(snippet) * 6) // 2), 260), snippet, fill=1) ++ rendered.append(to_rlcd(img)) ++ return rendered ++ ++ async def _send_stream_animation( ++ self, ++ screen_url: Optional[str], ++ state: str, ++ *, ++ text: str = "", ++ frames: Optional[int] = None, ++ ) -> bool: ++ """Send a server-rendered voice animation over the screen streaming socket.""" ++ target = self._screen_stream_target(screen_url) ++ if not target: ++ return False ++ frame_count = frames if frames is not None else (12 if state == "ack" else 18) ++ try: ++ rendered = await asyncio.to_thread(self._voice_animation_udp_frames, state, text, frame_count) ++ sock = _socket.socket(_socket.AF_INET, _socket.SOCK_DGRAM) ++ try: ++ for frame_id, frame in enumerate(rendered): ++ fid = frame_id % 256 ++ for chunk_idx in range(15): ++ start = chunk_idx * 1000 ++ sock.sendto(bytes([fid, chunk_idx]) + frame[start : start + 1000], target) ++ await asyncio.sleep(0.045) ++ finally: ++ sock.close() ++ return True ++ except Exception as exc: ++ logger.warning("[api_server] screen stream animation failed: %s", exc) ++ return False ++ ++ async def _send_screen_animation( ++ self, ++ screen_url: Optional[str], ++ state: str, ++ *, ++ text: str = "", ++ frames: Optional[int] = None, ++ ) -> bool: ++ """Upload and run the ESP32 voice-state animation script on the initiating screen.""" ++ if not screen_url: ++ return False ++ if await self._send_stream_animation(screen_url, state, text=text, frames=frames): ++ return True ++ if screen_url not in self._esp32_voice_animation_uploaded: ++ uploaded = await self._call_screen_mcp( ++ screen_url, ++ "write_file", ++ {"path": ESP32_VOICE_ANIMATION_PATH, "content": ESP32_VOICE_ANIMATION_SCRIPT}, ++ ) ++ if not uploaded: ++ return False ++ self._esp32_voice_animation_uploaded.add(screen_url) ++ code = ( ++ f"exec(open({ESP32_VOICE_ANIMATION_PATH!r}).read()); " ++ f"run_state({str(state)!r}, text={str(text or '')[:240]!r}, frames={frames!r})" ++ ) ++ return await self._call_screen_mcp(screen_url, "execute_python", {"code": code}) ++ ++ async def _send_screen_text(self, screen_url: Optional[str], text: str, *, clear: bool = False) -> None: ++ """Best-effort text update to the initiating MCP screen.""" ++ if not screen_url: ++ return ++ if clear: ++ await self._call_screen_mcp(screen_url, "clear_screen", {"color": 0}) ++ # Most Reyna screens accept x/y/text. Extra wrapping is avoided here so ++ # firmware-specific clients can keep their own font/layout behavior. ++ await self._call_screen_mcp(screen_url, "draw_text", {"x": 8, "y": 8, "text": text[:900]}) ++ ++ @staticmethod ++ def _extract_first_image_reference(text: str) -> Optional[str]: ++ """Return the first local path, MEDIA path, data URL, or HTTP image URL in text.""" ++ if not text: ++ return None ++ patterns = [ ++ r"MEDIA:([^\s)]+)", ++ r"!\[[^\]]*\]\(([^)]+)\)", ++ r"(?Phttps?://[^\s)]+\.(?:png|jpe?g|webp|gif)(?:\?[^\s)]*)?)", ++ r"(?P/[^\s)]+\.(?:png|jpe?g|webp|gif|bmp))", ++ r"(?Pdata:image/[^\s)]+)", ++ ] ++ for pattern in patterns: ++ for match in re.finditer(pattern, text, flags=re.IGNORECASE): ++ ref = match.groupdict().get("url") or match.groupdict().get("path") or match.groupdict().get("data") or match.group(1) ++ if ref: ++ return ref.strip().strip('"\'') ++ return None ++ ++ async def _send_screen_image_response(self, screen_url: Optional[str], text: str) -> bool: ++ """Best-effort render of the first generated/linked image onto the ESP32 screen.""" ++ if not screen_url: ++ return False ++ image_ref = self._extract_first_image_reference(text) ++ if not image_ref: ++ return False ++ try: ++ import base64 ++ import io ++ from PIL import Image, ImageOps ++ ++ raw: bytes ++ lowered = image_ref.lower() ++ if lowered.startswith("data:image/"): ++ _, _, encoded = image_ref.partition(",") ++ raw = base64.b64decode(encoded) ++ elif self._looks_like_http_url(image_ref): ++ from aiohttp import ClientSession, ClientTimeout ++ async with ClientSession(timeout=ClientTimeout(total=20)) as session: ++ async with session.get(image_ref) as resp: ++ if resp.status >= 400: ++ logger.warning("[api_server] screen image fetch failed: HTTP %s", resp.status) ++ return False ++ raw = await resp.read() ++ else: ++ path = Path(image_ref).expanduser() ++ if not path.exists() or not path.is_file(): ++ return False ++ raw = await asyncio.to_thread(path.read_bytes) ++ if not raw: ++ return False ++ ++ def render_pbm() -> str: ++ with Image.open(io.BytesIO(raw)) as img: ++ img = ImageOps.exif_transpose(img).convert("RGB") ++ canvas = Image.new("RGB", (320, 240), "white") ++ img.thumbnail((320, 240), Image.Resampling.LANCZOS) ++ x = (320 - img.width) // 2 ++ y = (240 - img.height) // 2 ++ canvas.paste(img, (x, y)) ++ mono = canvas.convert("1", dither=Image.Dither.FLOYDSTEINBERG) ++ out = io.BytesIO() ++ mono.save(out, format="PPM") # mode=1 writes raw PBM (P4) ++ return base64.b64encode(out.getvalue()).decode("ascii") ++ ++ pbm_base64 = await asyncio.to_thread(render_pbm) ++ return await self._call_screen_mcp( ++ screen_url, ++ "draw_image", ++ {"pbm_base64": pbm_base64, "x": 0, "y": 0, "dither": False}, ++ timeout=15, ++ ) ++ except Exception: ++ logger.exception("[api_server] failed to render ESP32 screen image response") ++ return False ++ ++ async def _send_screen_audio_response( ++ self, ++ screen_url: Optional[str], ++ text: str, ++ *, ++ audio_dir: Path, ++ device_id: str, ++ volume: int = 80, ++ ) -> bool: ++ """Best-effort spoken follow-up for ACK-mode ESP32 voice turns.""" ++ if not screen_url or not text.strip(): ++ return False ++ try: ++ import base64 ++ from tools.tts_tool import text_to_speech_tool ++ ++ # Keep board playback short and payloads small. The ESP32 HTTP MCP ++ # endpoint resets on large base64 WAV bodies; MP3 is much smaller ++ # and the firmware exposes play_mp3_base64 for exactly this path. ++ spoken_text = self._short_voice_summary(text, 420) ++ tts_raw = await asyncio.to_thread(text_to_speech_tool, spoken_text) ++ tts_result = json.loads(tts_raw) ++ src = str(tts_result.get("file_path") or "") ++ if not tts_result.get("success") or not src or not Path(src).exists(): ++ logger.warning( ++ "[api_server] screen audio TTS failed: %s", ++ tts_result.get("error") or "missing TTS output", ++ ) ++ return False ++ ++ src_path = Path(src) ++ if src_path.suffix.lower() == ".mp3": ++ mp3_base64 = base64.b64encode(src_path.read_bytes()).decode("ascii") ++ if await self._call_screen_mcp( ++ screen_url, ++ "play_mp3_base64", ++ {"mp3_base64": mp3_base64, "volume": volume}, ++ timeout=75, ++ ): ++ return True ++ ++ wav_path = await asyncio.to_thread( ++ self._convert_audio_to_esp32_wav, ++ src, ++ output_dir=audio_dir, ++ device_id=f"{device_id}_response", ++ ) ++ wav_bytes = wav_path.read_bytes() ++ if len(wav_bytes) > 220_000: ++ logger.warning( ++ "[api_server] screen audio WAV too large for MCP payload: %s bytes", ++ len(wav_bytes), ++ ) ++ return False ++ wav_base64 = base64.b64encode(wav_bytes).decode("ascii") ++ return await self._call_screen_mcp( ++ screen_url, ++ "play_audio_base64", ++ {"wav_base64": wav_base64, "volume": volume}, ++ timeout=45, ++ ) ++ except Exception: ++ logger.exception("[api_server] failed to play ESP32 screen audio response") ++ return False ++ ++ @staticmethod ++ def _short_voice_summary(text: str, limit: int = 220) -> str: ++ """Local fallback summary for voice/screen snippets without another LLM call.""" ++ clean = " ".join(str(text or "").split()) ++ if len(clean) <= limit: ++ return clean ++ return clean[: max(0, limit - 1)].rstrip() + "…" ++ ++ async def _esp32_ack_wav(self, *, audio_dir: Path, device_id: str) -> Path: ++ """Return a reusable quiet ACK WAV; the visual screen animation is the acknowledgement.""" ++ ack_path = audio_dir / "quick_ack.wav" ++ if ack_path.exists(): ++ return ack_path ++ ++ # ACK-mode clients expect a valid audio/wav response immediately, but ++ # Reyna ESP32 screens now show the acknowledgement visually. Keep the ++ # HTTP contract while avoiding the old spoken "Got it" message. ++ import wave ++ sample_rate = 16000 ++ duration_s = 0.08 ++ frames = b"\x00\x00" * int(sample_rate * duration_s) ++ with wave.open(str(ack_path), "wb") as wav: ++ wav.setnchannels(1) ++ wav.setsampwidth(2) ++ wav.setframerate(sample_rate) ++ wav.writeframes(frames) ++ return ack_path ++ ++ def _append_esp32_voice_audit(self, audio_dir: Path, event: Dict[str, Any]) -> None: ++ """Append a best-effort ESP32 voice request audit event.""" ++ try: ++ log_path = audio_dir / "voice_requests.jsonl" ++ with log_path.open("a", encoding="utf-8") as handle: ++ handle.write(json.dumps(event, default=str) + "\n") ++ except Exception: ++ logger.debug("[api_server] failed to append ESP32 voice audit log", exc_info=True) ++ ++ async def _run_esp32_voice_audio_background( ++ self, ++ *, ++ audio_path: Path, ++ audit_event: Dict[str, Any], ++ device_id: str, ++ instructions: Optional[str], ++ audio_dir: Path, ++ screen_url: Optional[str], ++ ) -> None: ++ """Transcribe uploaded ESP32 audio and continue the voice turn after an immediate ACK.""" ++ try: ++ await self._send_screen_animation(screen_url, "captioning") ++ from tools.transcription_tools import transcribe_audio ++ stt_result = await asyncio.to_thread(transcribe_audio, str(audio_path)) ++ if not stt_result.get("success"): ++ failed_event = dict(audit_event) ++ failed_event.update({ ++ "status": "transcription_failed", ++ "error": stt_result.get("error") or "Transcription failed", ++ "timestamp": time.time(), ++ }) ++ self._append_esp32_voice_audit(audio_dir, failed_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but transcription failed. Try again.", ++ clear=True, ++ ) ++ return ++ transcript = str(stt_result.get("transcript") or "").strip() ++ if not transcript: ++ empty_event = dict(audit_event) ++ empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) ++ self._append_esp32_voice_audit(audio_dir, empty_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but got no words. Try again.", ++ clear=True, ++ ) ++ return ++ transcript_event = dict(audit_event) ++ transcript_event.update({"status": "transcribed", "transcript": transcript, "timestamp": time.time()}) ++ self._append_esp32_voice_audit(audio_dir, transcript_event) ++ await self._run_esp32_voice_background( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=instructions, ++ audio_dir=audio_dir, ++ screen_url=screen_url, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 ACK background voice processing failed") ++ failed_event = dict(audit_event) ++ failed_event.update({"status": "background_exception", "error": str(exc), "timestamp": time.time()}) ++ self._append_esp32_voice_audit(audio_dir, failed_event) ++ await self._send_screen_text( ++ screen_url, ++ "I got the audio, but background processing failed.", ++ clear=True, ++ ) ++ ++ async def _run_esp32_voice_background( ++ self, ++ *, ++ transcript: str, ++ device_id: str, ++ instructions: Optional[str], ++ audio_dir: Path, ++ screen_url: Optional[str], ++ ) -> None: ++ """Run the slow ESP32 voice turn after the device already heard an ACK.""" ++ try: ++ animation_ok = await self._send_screen_animation(screen_url, "waiting", text=transcript) ++ if not animation_ok: ++ await self._send_screen_text( ++ screen_url, ++ f"Heard: {self._short_voice_summary(transcript, 180)}\n\nWorking…", ++ clear=True, ++ ) ++ screen_hint = ( ++ "\n\nThis request came from an ESP32 voice/screen device. " ++ "Keep the spoken follow-up short. The gateway will display the full answer on the initiating screen when possible." ++ ) ++ merged_instructions = (instructions or "") + screen_hint ++ response_data, final_text = await self._run_esp32_voice_turn( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=merged_instructions, ++ ) ++ if not final_text.strip(): ++ final_text = self._extract_response_text(response_data) or "I finished, but I do not have a text answer." ++ display_text = final_text ++ if len(display_text) > 1200: ++ display_text = self._short_voice_summary(display_text, 900) + "\n\nFull answer is long; WhatsApp delivery is the next wiring step." ++ await self._send_screen_text(screen_url, display_text, clear=True) ++ await self._send_screen_image_response(screen_url, final_text) ++ await self._send_screen_audio_response( ++ screen_url, ++ final_text, ++ audio_dir=audio_dir, ++ device_id=device_id, ++ ) ++ except Exception: ++ logger.exception("[api_server] ESP32 background voice turn failed") ++ await self._send_screen_text(screen_url, "Hermes hit an error while preparing the full answer.", clear=True) ++ ++ @staticmethod ++ def _extract_response_text(response_data: Dict[str, Any]) -> str: ++ """Extract assistant text from an OpenAI Responses-shaped payload.""" ++ chunks: List[str] = [] ++ for item in response_data.get("output") or []: ++ if not isinstance(item, dict) or item.get("type") != "message": ++ continue ++ for part in item.get("content") or []: ++ if isinstance(part, dict) and part.get("type") in {"output_text", "text"}: ++ text = part.get("text") ++ if isinstance(text, str) and text: ++ chunks.append(text) ++ return "\n".join(chunks).strip() ++ ++ async def _run_esp32_voice_turn( ++ self, ++ *, ++ transcript: str, ++ device_id: str, ++ instructions: Optional[str] = None, ++ ) -> tuple[Dict[str, Any], str]: ++ """Run one persistent Responses-style turn for an ESP32 voice device.""" ++ conversation = f"esp32:{device_id}" ++ previous_response_id = self._response_store.get_conversation(conversation) ++ conversation_history: List[Dict[str, Any]] = [] ++ stored_session_id = None ++ stored_instructions = None ++ ++ if previous_response_id: ++ stored = self._response_store.get(previous_response_id) ++ if stored is not None: ++ conversation_history = list(stored.get("conversation_history", [])) ++ stored_session_id = stored.get("session_id") ++ stored_instructions = stored.get("instructions") ++ ++ if instructions is None: ++ instructions = stored_instructions ++ ++ session_id = stored_session_id or str(uuid.uuid4()) ++ user_message = ( ++ f'[Voice input from ESP32 device "{device_id}"]\n' ++ f'{transcript}' ++ ) ++ ++ result, usage = await self._run_agent( ++ user_message=user_message, ++ conversation_history=conversation_history, ++ ephemeral_system_prompt=instructions, ++ session_id=session_id, ++ gateway_session_key=conversation, ++ ) ++ ++ final_response = result.get("final_response", "") or result.get("error", "") or "(No response generated)" ++ response_id = f"resp_{uuid.uuid4().hex[:28]}" ++ created_at = int(time.time()) ++ full_history = self._build_response_conversation_history( ++ conversation_history, ++ user_message, ++ result, ++ final_response, ++ ) ++ output_start_index = self._response_messages_turn_start_index( ++ conversation_history, ++ user_message, ++ result, ++ ) ++ output_items = self._extract_output_items(result, start_index=output_start_index) ++ response_data = { ++ "id": response_id, ++ "object": "response", ++ "status": "completed", ++ "created_at": created_at, ++ "model": self._model_name, ++ "output": output_items, ++ "usage": { ++ "input_tokens": usage.get("input_tokens", 0), ++ "output_tokens": usage.get("output_tokens", 0), ++ "total_tokens": usage.get("total_tokens", 0), ++ }, ++ } ++ self._response_store.put(response_id, { ++ "response": response_data, ++ "conversation_history": full_history, ++ "instructions": instructions, ++ "session_id": session_id, ++ }) ++ self._response_store.set_conversation(conversation, response_id) ++ return response_data, final_response ++ ++ @staticmethod ++ def _esp32_audio_cache_dir() -> Path: ++ """Return the cache directory for ESP32 voice artifacts.""" ++ try: ++ from hermes_constants import get_hermes_dir ++ return get_hermes_dir("cache/audio", "audio_cache") / "esp32" ++ except Exception: ++ return Path.home() / ".hermes" / "cache" / "audio" / "esp32" ++ ++ @staticmethod ++ def _write_pcm_s16le_wav(raw_path: Path, wav_path: Path, *, sample_rate: int, channels: int = 1) -> None: ++ """Wrap a raw PCM s16le file in a minimal RIFF/WAVE container.""" ++ import wave ++ ++ with wave.open(str(wav_path), "wb") as wav: ++ wav.setnchannels(channels) ++ wav.setsampwidth(2) ++ wav.setframerate(sample_rate) ++ with raw_path.open("rb") as raw_handle: ++ while True: ++ chunk = raw_handle.read(64 * 1024) ++ if not chunk: ++ break ++ wav.writeframes(chunk) ++ ++ # --- Live draft helpers (from your whisper-translation/apple_speech pipe protocol) --- ++ def _load_macmini_mcp_config(self) -> dict: ++ try: ++ if ESP32_MACMINI_MCP_URL and ESP32_MACMINI_MCP_TOKEN: ++ return {"url": ESP32_MACMINI_MCP_URL, "token": ESP32_MACMINI_MCP_TOKEN} ++ cfg_path = Path.home() / ".hermes" / "config.yaml" ++ if cfg_path.exists(): ++ import yaml ++ cfg = yaml.safe_load(cfg_path.read_text()) or {} ++ m = (cfg.get("mcp_servers") or {}).get("macmini") or {} ++ url = m.get("url") or "" ++ token = "" ++ headers = m.get("headers") or {} ++ auth = headers.get("Authorization") or "" ++ if auth.startswith("Bearer "): ++ token = auth[7:] ++ if url: ++ return {"url": url, "token": token} ++ except Exception: ++ pass ++ return {} ++ ++ @staticmethod ++ def _s16le_to_wav_bytes(pcm_bytes: bytes, sample_rate: int = 16000, channels: int = 1) -> bytes: ++ import io, wave ++ bio = io.BytesIO() ++ with wave.open(bio, "wb") as wf: ++ wf.setnchannels(channels) ++ wf.setsampwidth(2) ++ wf.setframerate(sample_rate) ++ wf.writeframes(pcm_bytes) ++ return bio.getvalue() ++ ++ async def _transcribe_draft_via_macmini(self, wav_bytes: bytes, locale: str = "en-US") -> str | None: ++ if not ESP32_LIVE_DRAFT_USE_MACMINI: ++ return None ++ cfg = self._load_macmini_mcp_config() ++ url = cfg.get("url") ++ token = cfg.get("token") ++ if not url: ++ return None ++ try: ++ import base64 ++ b64 = base64.b64encode(wav_bytes).decode("ascii") ++ payload = { ++ "jsonrpc": "2.0", ++ "id": 1, ++ "method": "tools/call", ++ "params": {"name": "speech_live_transcribe", "arguments": {"audioBase64": b64, "locale": locale}}, ++ } ++ headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"} ++ if token: ++ headers["Authorization"] = f"Bearer {token}" ++ import aiohttp ++ async with aiohttp.ClientSession() as session: ++ async with session.post(url, json=payload, headers=headers, timeout=aiohttp.ClientTimeout(total=8)) as resp: ++ if resp.status != 200: ++ return None ++ txt = await resp.text() ++ for line in txt.splitlines(): ++ if not line.startswith("data: "): ++ continue ++ try: ++ data = json.loads(line[6:]) ++ content = (data.get("result", {}).get("content") or [{}])[0].get("text", "") or "" ++ if not content: ++ continue ++ inner = json.loads(content) ++ t = inner.get("text") or "" ++ if t: ++ return t.strip() ++ except Exception: ++ continue ++ except Exception as e: ++ logger.debug(f"[api_server] macmini live draft failed: {e}") ++ return None ++ ++ async def _transcribe_draft_local(self, wav_bytes: bytes) -> str | None: ++ try: ++ import tempfile ++ fd, tmp_path = tempfile.mkstemp(suffix=".wav") ++ os.close(fd) ++ Path(tmp_path).write_bytes(wav_bytes) ++ def _do(): ++ try: ++ from faster_whisper import WhisperModel ++ model = None ++ try: ++ from tools.transcription_tools import _local_model ++ if _local_model is not None: ++ model = _local_model ++ except Exception: ++ pass ++ if model is None: ++ model = WhisperModel("tiny", device="cpu", compute_type="int8", num_workers=1) ++ segs, _ = model.transcribe(tmp_path, beam_size=1, best_of=1, language="en", condition_on_previous_text=False, vad_filter=True) ++ return " ".join(s.text.strip() for s in segs).strip() ++ except Exception as ex: ++ logger.debug(f"[api_server] local draft err: {ex}") ++ return None ++ finally: ++ try: ++ Path(tmp_path).unlink(missing_ok=True) ++ except Exception: ++ pass ++ return await asyncio.to_thread(_do) ++ except Exception: ++ return None ++ ++ async def _apple_llm_quick_reply_via_macmini(self, draft: str, context: str = "", instructions: str = "") -> str | None: ++ """Call Mac mini apple_llm_quick_reply for instant smart ACK from draft.""" ++ if not ESP32_LIVE_DRAFT_USE_MACMINI: ++ return None ++ cfg = self._load_macmini_mcp_config() ++ url = cfg.get("url") ++ token = cfg.get("token") ++ if not url: ++ return None ++ try: ++ payload = { ++ "jsonrpc": "2.0", ++ "id": 1, ++ "method": "tools/call", ++ "params": {"name": "apple_llm_quick_reply", "arguments": {"draft": draft, "context": context or "", "instructions": instructions or ""}}, ++ } ++ headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"} ++ if token: ++ headers["Authorization"] = f"Bearer {token}" ++ import aiohttp ++ async with aiohttp.ClientSession() as session: ++ async with session.post(url, json=payload, headers=headers, timeout=aiohttp.ClientTimeout(total=4)) as resp: ++ if resp.status != 200: ++ return None ++ txt = await resp.text() ++ for line in txt.splitlines(): ++ if not line.startswith("data: "): ++ continue ++ try: ++ data = json.loads(line[6:]) ++ content = (data.get("result", {}).get("content") or [{}])[0].get("text", "") or "" ++ if not content: ++ continue ++ inner = json.loads(content) ++ t = inner.get("text") or "" ++ if t: ++ return t.strip() ++ except Exception: ++ continue ++ except Exception as e: ++ logger.debug(f"[api_server] macmini quick_reply failed: {e}") ++ return None ++ ++ async def _run_quick_reply_then_full(self, draft_text: str, final_transcript: str, device_id: str, screen_url, instructions: str | None, audio_dir): ++ """New flow: when voice stops, we already have draft. Emit instant smart reply via Apple LLM, then full Hermes turn.""" ++ # This is kept for future speculative pre-start; for now we just log — actual quick reply emitted in WS handler before background start ++ pass ++ ++ ++ async def _handle_esp32_voice_ws(self, request: "web.Request") -> "web.WebSocketResponse": ++ """GET /api/esp32/voice/ws — ESP32 PCM streaming with LIVE draft captioning. ++ LIVE DRAFT MODE (from whisper-translation/engine_apple_transcribe.py): ++ - While ingesting binary s16le, every ~800ms snapshots current PCM -> in-mem WAV ++ -> tries Mac mini speech_live_transcribe (ApplePipeTranscriber --pipe volatile, ++ <200ms drafts) fallback to local tiny for fast feedback. ++ - Emits WS events: draft + interim_transcript for UI. Final via macmini or local base. ++ """ ++ auth_err = self._check_auth(request) ++ if auth_err: ++ return auth_err # type: ignore[return-value] ++ ++ ws = web.WebSocketResponse(heartbeat=None, max_msg_size=512 * 1024) ++ await ws.prepare(request) ++ ++ async def _send_event(event: str, **payload): ++ if not ws.closed: ++ await ws.send_str(json.dumps({"event": event, **payload})) ++ ++ device_id = ( ++ request.headers.get("X-Device-ID") ++ or request.query.get("device_id") ++ or "default" ++ ).strip()[:80] or "default" ++ safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" ++ audio_dir = self._esp32_audio_cache_dir() ++ audio_dir.mkdir(parents=True, exist_ok=True) ++ stamp = f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}" ++ raw_path = audio_dir / f"{stamp}.s16le" ++ wav_path = audio_dir / f"{stamp}.wav" ++ sample_rate = 16000 ++ channels = 1 ++ sample_width = 2 ++ total_audio_bytes = 0 ++ started = False ++ stopped = False ++ first_audio_at: float | None = None ++ started_at = time.time() ++ # live draft state (directly from your whisper-translation ApplePipeTranscriber pattern) ++ live_enabled = ESP32_LIVE_DRAFT_ENABLED ++ pcm_buffer = bytearray() ++ last_draft_at = 0.0 ++ last_draft_bytes = 0 ++ last_draft_text = "" ++ draft_in_progress = False ++ draft_count = 0 ++ locale_req = request.headers.get("X-Hermes-Locale") or request.query.get("locale") or "en-US" ++ ++ audit_ctx = self._request_audit_context(request) ++ audit_event = { ++ "timestamp": started_at, ++ "device_id": device_id, ++ "source_ip": audit_ctx.get("remote") or audit_ctx.get("peer_ip") or "", ++ "peer_ip": audit_ctx.get("peer_ip") or "", ++ "forwarded_for": audit_ctx.get("forwarded_for") or "", ++ "real_ip": audit_ctx.get("real_ip") or "", ++ "user_agent": audit_ctx.get("user_agent") or "", ++ "transport": "websocket", ++ "raw_path": str(raw_path), ++ "audio_path": str(wav_path), ++ "status": "accepted", ++ "live_draft_enabled": live_enabled, ++ } ++ self._append_esp32_voice_audit(audio_dir, audit_event) ++ screen_url = self._resolve_esp32_screen_url(request, device_id) ++ visual_listening_sent = False ++ ++ def _schedule_voice_state(state: str, *, text: str = "", frames: int | None = None) -> None: ++ if not screen_url: ++ return ++ task = asyncio.create_task(self._send_screen_animation(screen_url, state, text=text, frames=frames)) ++ try: ++ self._background_tasks.add(task) ++ except TypeError: ++ pass ++ if hasattr(task, "add_done_callback"): ++ task.add_done_callback(self._background_tasks.discard) ++ ++ async def _do_live_draft(snapshot_bytes: bytes): ++ nonlocal last_draft_text, draft_in_progress, draft_count ++ if draft_in_progress: ++ return ++ if len(snapshot_bytes) < int(sample_rate * 0.5 * 2): # at least 500ms of s16le at 16k mono ++ return ++ draft_in_progress = True ++ try: ++ wav_b = self._s16le_to_wav_bytes(snapshot_bytes, sample_rate=sample_rate, channels=channels) ++ txt = None ++ if ESP32_LIVE_DRAFT_USE_MACMINI: ++ txt = await self._transcribe_draft_via_macmini(wav_b, locale=locale_req) ++ if not txt: ++ txt = await self._transcribe_draft_local(wav_b) ++ if not txt: ++ return ++ txt = txt.strip() ++ if not txt or txt == last_draft_text or len(txt) < 2: ++ return ++ last_draft_text = txt ++ draft_count += 1 ++ await _send_event("draft", text=txt, is_final=False, isFinal=False, chunk=draft_count, bytes=len(snapshot_bytes)) ++ await _send_event("interim_transcript", text=txt, is_final=False, draft_count=draft_count) ++ if screen_url and len(txt) > 1: ++ try: ++ t = asyncio.create_task(self._send_screen_text(screen_url, txt, clear=False)) ++ self._background_tasks.add(t) ++ t.add_done_callback(self._background_tasks.discard) ++ except Exception: ++ pass ++ finally: ++ draft_in_progress = False ++ ++ await _send_event("ready", device_id=device_id, screen_url=screen_url or "", live_draft=live_enabled, live_model="apple-pipe-volatile") ++ ++ try: ++ raw_handle = raw_path.open("wb") ++ try: ++ async for msg in ws: ++ if msg.type == web.WSMsgType.TEXT: ++ try: ++ payload = json.loads(msg.data) ++ except json.JSONDecodeError: ++ await _send_event("error", code="bad_json", message="Text frames must be JSON") ++ continue ++ event = str(payload.get("event") or payload.get("type") or "").strip().lower() ++ if event == "start": ++ if started: ++ await _send_event("error", code="already_started", message="Audio stream already started") ++ continue ++ fmt = str(payload.get("format") or "pcm_s16le").strip().lower() ++ channels = int(payload.get("channels") or 1) ++ sample_width = int(payload.get("sample_width") or 2) ++ sample_rate = int(payload.get("sample_rate") or 16000) ++ device_id = str(payload.get("device_id") or device_id).strip()[:80] or device_id ++ if fmt not in {"pcm_s16le", "s16le", "pcm"} or channels != 1 or sample_width != 2: ++ await _send_event("error", code="unsupported_audio_format", message="Use 16 kHz mono signed 16-bit little-endian PCM") ++ await ws.close() ++ return ws ++ if sample_rate < 8000 or sample_rate > 48000: ++ await _send_event("error", code="bad_sample_rate", message="sample_rate must be 8000-48000") ++ await ws.close() ++ return ws ++ started = True ++ if not visual_listening_sent: ++ _schedule_voice_state("captioning", frames=8) ++ visual_listening_sent = True ++ await _send_event("listening", sample_rate=sample_rate, channels=channels, sample_width=sample_width) ++ elif event in {"stop", "end"}: ++ stopped = True ++ break ++ elif event in {"ping", "keepalive"}: ++ await _send_event("pong") ++ else: ++ await _send_event("error", code="unknown_event", message=f"Unknown event: {event or ''}") ++ elif msg.type == web.WSMsgType.BINARY: ++ if not started: ++ started = True ++ if not visual_listening_sent: ++ _schedule_voice_state("captioning", frames=8) ++ visual_listening_sent = True ++ await _send_event("listening", sample_rate=sample_rate, channels=channels, sample_width=sample_width) ++ chunk = bytes(msg.data or b"") ++ if not chunk: ++ continue ++ if first_audio_at is None: ++ first_audio_at = time.time() ++ total_audio_bytes += len(chunk) ++ pcm_buffer.extend(chunk) ++ if total_audio_bytes > ESP32_WS_MAX_AUDIO_BYTES: ++ await _send_event("error", code="audio_too_large", message=f"Audio stream exceeded {ESP32_WS_MAX_AUDIO_BYTES} bytes") ++ await ws.close() ++ return ws ++ raw_handle.write(chunk) ++ # live draft trigger (every 800ms & 8KB new data) — direct captioning fluff stripped ++ if live_enabled: ++ now = time.time() ++ elapsed_ms = (now - last_draft_at) * 1000 ++ new_bytes = len(pcm_buffer) - last_draft_bytes ++ total_ms = len(pcm_buffer) / 2 / sample_rate * 1000 ++ if total_ms >= ESP32_LIVE_DRAFT_MIN_TOTAL_MS and elapsed_ms >= ESP32_LIVE_DRAFT_INTERVAL_MS and new_bytes >= ESP32_LIVE_DRAFT_MIN_BYTES: ++ last_draft_at = now ++ last_draft_bytes = len(pcm_buffer) ++ # copy snapshot, fire-and-forget draft (but guard concurrent) ++ snapshot = bytes(pcm_buffer) ++ asyncio.create_task(_do_live_draft(snapshot)) ++ elif msg.type == web.WSMsgType.ERROR: ++ logger.warning("[api_server] ESP32 voice websocket error: %s", ws.exception()) ++ break ++ finally: ++ raw_handle.close() ++ ++ if not stopped: ++ await _send_event("error", code="missing_stop", message="Send a stop event after audio chunks") ++ await ws.close() ++ return ws ++ if total_audio_bytes < ESP32_WS_MIN_AUDIO_BYTES: ++ await _send_event("error", code="audio_too_short", message="Not enough audio received") ++ await ws.close() ++ return ws ++ ++ await asyncio.to_thread( ++ self._write_pcm_s16le_wav, ++ raw_path, ++ wav_path, ++ sample_rate=sample_rate, ++ channels=channels, ++ ) ++ audit_event.update({ ++ "status": "received", ++ "audio_bytes": total_audio_bytes, ++ "sample_rate": sample_rate, ++ "first_audio_latency_ms": int(((first_audio_at or started_at) - started_at) * 1000), ++ "stop_latency_ms": int((time.time() - started_at) * 1000), ++ "draft_count": draft_count, ++ "last_draft": last_draft_text, ++ }) ++ self._append_esp32_voice_audit(audio_dir, dict(audit_event)) ++ ++ _schedule_voice_state("captioning", frames=10) ++ await _send_event("thinking", stage="transcribing", draft_count=draft_count) ++ ++ # Final transcription: prefer Mac mini live pipe (ApplePipeTranscriber) — we already have full pcm_buffer ++ # This replaces the old 700ms faster-whisper base path. Fallback to local only if Mac unavailable. ++ transcript = None ++ final_source = "none" ++ t_transcribe_start = time.time() ++ ++ # Strategy: try Mac mini with FULL buffer via speech_live_transcribe (final), then check drafts ++ try: ++ if ESP32_LIVE_DRAFT_USE_MACMINI and len(pcm_buffer) > 0: ++ full_wav = self._s16le_to_wav_bytes(bytes(pcm_buffer), sample_rate=sample_rate, channels=channels) ++ mac_txt = await self._transcribe_draft_via_macmini(full_wav, locale=locale_req) ++ if mac_txt and len(mac_txt.strip()) >= 2: ++ transcript = mac_txt.strip() ++ final_source = "macmini-live-final" ++ except Exception as e: ++ logger.debug(f"[api_server] macmini final transcribe failed: {e}") ++ ++ if not transcript: ++ from tools.transcription_tools import transcribe_audio ++ # If we already wrote wav_path above, reuse else write now ++ try: ++ # ensure wav_path exists (we haven't needed it till now after refactor) ++ if not wav_path.exists(): ++ await asyncio.to_thread( ++ self._write_pcm_s16le_wav, ++ raw_path, ++ wav_path, ++ sample_rate=sample_rate, ++ channels=channels, ++ ) ++ except Exception: ++ pass ++ stt_result = await asyncio.to_thread(transcribe_audio, str(wav_path) if wav_path.exists() else str(raw_path)) ++ if not stt_result.get("success"): ++ message = stt_result.get("error") or "Transcription failed" ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "transcription_failed", "error": message, "timestamp": time.time()}) ++ await _send_event("error", code="transcription_failed", message=message) ++ await ws.close() ++ return ws ++ transcript = str(stt_result.get("transcript") or "").strip() ++ final_source = "local-faster-whisper" ++ ++ if not transcript: ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "empty_transcript", "timestamp": time.time()}) ++ await _send_event("error", code="empty_transcript", message="Transcription was empty") ++ await ws.close() ++ return ws ++ ++ transcribe_ms = int((time.time() - t_transcribe_start) * 1000) ++ ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "transcribed", "transcript": transcript, "timestamp": time.time(), "draft_count": draft_count, "final_source": final_source, "transcribe_ms": transcribe_ms}) ++ await _send_event("transcript", text=transcript, is_final=True, isFinal=True, draft_count=draft_count, source=final_source, transcribe_ms=transcribe_ms) ++ await _send_event("final", text=transcript, isFinal=True, source=final_source) ++ ++ summary = self._short_voice_summary(transcript, 180) ++ ack_text = f"Heard: {summary}\n\nWorking on the full answer now." ++ await _send_event("response_text", text=ack_text, response_id="", final=False) ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "queued_background", "transcript": transcript, "timestamp": time.time()}) ++ ++ # --- NEW: instant smart reply from draft via Mac mini Apple 3B LLM (ANE, ~0.6s) --- ++ # This gives user a smart ACK almost immediately when voice stops, before full Hermes turn. ++ # From whisper-translation engine_apple_llm.py: AppleLLM polish/quick_reply pipe. ++ t_quick_start = time.time() ++ instant_reply = None ++ try: ++ # Prefer last_draft_text if exists (live), else final transcript ++ draft_for_llm = last_draft_text if last_draft_text and len(last_draft_text) > 3 else transcript ++ # Only call Mac mini if we have draft and Mac enabled ++ if draft_for_llm and ESP32_LIVE_DRAFT_USE_MACMINI: ++ await _send_event("thinking", stage="instant_reply", draft=draft_for_llm[:120]) ++ instant_reply = await self._apple_llm_quick_reply_via_macmini( ++ draft=draft_for_llm, ++ context=transcript[:300] if transcript != draft_for_llm else "", ++ instructions=request.headers.get("X-Hermes-Instructions") or "" ++ ) ++ if instant_reply: ++ quick_ms = int((time.time() - t_quick_start) * 1000) ++ await _send_event("instant_response", text=instant_reply, is_quick=True, quick_ms=quick_ms, draft=draft_for_llm[:200], source="apple-llm-3b-ane") ++ await _send_event("quick_reply", text=instant_reply, ms=quick_ms) ++ # Also push to screen instantly (direct captioning -> instant smart reply) ++ if screen_url: ++ try: ++ task = asyncio.create_task(self._send_screen_text(screen_url, instant_reply, clear=True)) ++ self._background_tasks.add(task) ++ task.add_done_callback(self._background_tasks.discard) ++ except Exception: ++ pass ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "instant_reply", "instant_reply": instant_reply, "quick_ms": quick_ms, "timestamp": time.time()}) ++ except Exception as e: ++ logger.debug(f"[api_server] instant reply failed: {e}") ++ ++ # Immediate background LLM start — no extra wait (user flagged LLM awaiting whole thing) ++ try: ++ await _send_event("done", mode="background", screen_url=screen_url or "", draft_count=draft_count, source=final_source, instant_reply=instant_reply or "") ++ await ws.close() ++ except Exception: ++ pass ++ background_task = asyncio.create_task( ++ self._run_esp32_voice_background( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=request.headers.get("X-Hermes-Instructions") or None, ++ audio_dir=audio_dir, ++ screen_url=screen_url, ++ ) ++ ) ++ try: ++ self._background_tasks.add(background_task) ++ except TypeError: ++ pass ++ if hasattr(background_task, "add_done_callback"): ++ background_task.add_done_callback(self._background_tasks.discard) ++ return ws ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 voice websocket processing failed") ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "websocket_exception", "error": str(exc), "timestamp": time.time()}) ++ try: ++ await _send_event("error", code="internal_error", message=str(exc)) ++ await ws.close() ++ except Exception: ++ pass ++ return ws ++ finally: ++ try: ++ raw_path.unlink(missing_ok=True) ++ except Exception: ++ pass ++ ++ async def _handle_esp32_voice(self, request: "web.Request") -> "web.Response": ++ ++ """POST /api/esp32/voice — receive audio, run Hermes, return TTS audio.""" ++ auth_err = self._check_auth(request) ++ if auth_err: ++ return auth_err ++ ++ device_id = ( ++ request.headers.get("X-Device-ID") ++ or request.query.get("device_id") ++ or "default" ++ ).strip()[:80] or "default" ++ ++ try: ++ audio_bytes = await request.read() ++ except Exception as exc: ++ return web.json_response(_openai_error(f"Failed to read audio body: {exc}"), status=400) ++ ++ if not audio_bytes: ++ return web.json_response(_openai_error("Empty audio body", code="empty_audio"), status=400) ++ if len(audio_bytes) > ESP32_AUDIO_MAX_BYTES: ++ return web.json_response( ++ _openai_error( ++ f"Audio body is too large ({len(audio_bytes)} bytes); max is {ESP32_AUDIO_MAX_BYTES}", ++ code="audio_too_large", ++ ), ++ status=413, ++ ) ++ ++ suffix = self._audio_extension_for_content_type(request.headers.get("Content-Type", "")) ++ try: ++ from hermes_constants import get_hermes_dir ++ audio_dir = get_hermes_dir("cache/audio", "audio_cache") / "esp32" ++ except Exception: ++ audio_dir = Path.home() / ".hermes" / "cache" / "audio" / "esp32" ++ audio_dir.mkdir(parents=True, exist_ok=True) ++ safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" ++ audio_path = audio_dir / f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}{suffix}" ++ audio_path.write_bytes(audio_bytes) ++ ++ audit_ctx = self._request_audit_context(request) ++ audit_event: Dict[str, Any] = { ++ "timestamp": time.time(), ++ "device_id": device_id, ++ "source_ip": audit_ctx.get("remote") or audit_ctx.get("peer_ip") or "", ++ "peer_ip": audit_ctx.get("peer_ip") or "", ++ "forwarded_for": audit_ctx.get("forwarded_for") or "", ++ "real_ip": audit_ctx.get("real_ip") or "", ++ "user_agent": audit_ctx.get("user_agent") or "", ++ "reply_mode": self._esp32_reply_mode(request), ++ "screen_device": request.headers.get("X-Hermes-Screen-Device") or request.query.get("screen_device") or "", ++ "screen_url": request.headers.get("X-Hermes-Screen-Url") or request.headers.get("X-Screen-MCP-Url") or request.query.get("screen_url") or "", ++ "content_type": request.headers.get("Content-Type", ""), ++ "audio_bytes": len(audio_bytes), ++ "audio_path": str(audio_path), ++ "status": "received", ++ } ++ ++ def _append_audit(event: Dict[str, Any]) -> None: ++ try: ++ log_path = audio_dir / "voice_requests.jsonl" ++ with log_path.open("a", encoding="utf-8") as handle: ++ handle.write(json.dumps(event, default=str) + "\n") ++ except Exception: ++ logger.debug("[api_server] failed to append ESP32 voice audit log", exc_info=True) ++ ++ _append_audit(audit_event) ++ ++ if self._esp32_reply_mode(request) == "ack": ++ screen_url = self._resolve_esp32_screen_url(request, device_id) ++ ++ async def _ack_background() -> None: ++ try: ++ await self._send_screen_animation(screen_url, "ack") ++ await self._send_screen_animation(screen_url, "captioning") ++ from tools.transcription_tools import transcribe_audio ++ stt_result_bg = await asyncio.wait_for( ++ asyncio.to_thread(transcribe_audio, str(audio_path)), ++ timeout=45, ++ ) ++ if not stt_result_bg.get("success"): ++ failed_event = dict(audit_event) ++ failed_event.update({ ++ "status": "transcription_failed", ++ "error": stt_result_bg.get("error") or "Transcription failed", ++ "timestamp": time.time(), ++ }) ++ _append_audit(failed_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but transcription failed. Try again.", ++ clear=True, ++ ) ++ return ++ transcript_bg = str(stt_result_bg.get("transcript") or "").strip() ++ if not transcript_bg: ++ empty_event = dict(audit_event) ++ empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) ++ _append_audit(empty_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but got no words. Try again.", ++ clear=True, ++ ) ++ return ++ transcript_event = dict(audit_event) ++ transcript_event.update({"status": "transcribed", "transcript": transcript_bg, "timestamp": time.time()}) ++ _append_audit(transcript_event) ++ await self._run_esp32_voice_background( ++ transcript=transcript_bg, ++ device_id=device_id, ++ instructions=request.headers.get("X-Hermes-Instructions") or None, ++ audio_dir=audio_dir, ++ screen_url=screen_url, ++ ) ++ except asyncio.TimeoutError: ++ timeout_event = dict(audit_event) ++ timeout_event.update({"status": "transcription_timeout", "error": "Transcription exceeded 45s", "timestamp": time.time()}) ++ _append_audit(timeout_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but captioning timed out. Try again.", ++ clear=True, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 ack-mode background processing failed") ++ failed_event = dict(audit_event) ++ failed_event.update({"status": "background_exception", "error": str(exc), "timestamp": time.time()}) ++ _append_audit(failed_event) ++ ++ task = asyncio.create_task(_ack_background()) ++ try: ++ self._background_tasks.add(task) ++ except TypeError: ++ pass ++ if hasattr(task, "add_done_callback"): ++ task.add_done_callback(self._background_tasks.discard) ++ ack_path = await self._esp32_ack_wav(audio_dir=audio_dir, device_id=device_id) ++ headers = { ++ "X-Hermes-Conversation": f"esp32:{device_id}", ++ "X-Hermes-Reply-Mode": "ack", ++ "X-Hermes-Screen-Url": screen_url or "", ++ "Content-Type": "audio/wav", ++ } ++ return web.FileResponse(ack_path, headers=headers) ++ ++ try: ++ from tools.transcription_tools import transcribe_audio ++ stt_result = await asyncio.to_thread(transcribe_audio, str(audio_path)) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 transcription failed") ++ failed_event = dict(audit_event) ++ failed_event.update({"status": "transcription_exception", "error": str(exc), "timestamp": time.time()}) ++ _append_audit(failed_event) ++ return web.json_response(_openai_error(f"Transcription failed: {exc}"), status=500) ++ ++ if not stt_result.get("success"): ++ failed_event = dict(audit_event) ++ failed_event.update({ ++ "status": "transcription_failed", ++ "error": stt_result.get("error") or "Transcription failed", ++ "timestamp": time.time(), ++ }) ++ _append_audit(failed_event) ++ return web.json_response( ++ _openai_error(stt_result.get("error") or "Transcription failed", code="transcription_failed"), ++ status=502, ++ ) ++ ++ transcript = str(stt_result.get("transcript") or "").strip() ++ if not transcript: ++ empty_event = dict(audit_event) ++ empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) ++ _append_audit(empty_event) ++ return web.json_response(_openai_error("Transcription was empty", code="empty_transcript"), status=422) ++ ++ transcript_event = dict(audit_event) ++ transcript_event.update({"status": "transcribed", "transcript": transcript, "timestamp": time.time()}) ++ _append_audit(transcript_event) ++ ++ instructions = request.headers.get("X-Hermes-Instructions") or None ++ try: ++ response_data, final_text = await self._run_esp32_voice_turn( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=instructions, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 agent turn failed") ++ return web.json_response(_openai_error(f"Agent turn failed: {exc}"), status=500) ++ ++ if not final_text.strip(): ++ final_text = self._extract_response_text(response_data) or "I heard you, but I do not have a spoken answer." ++ ++ try: ++ from tools.tts_tool import text_to_speech_tool ++ tts_raw = await asyncio.to_thread(text_to_speech_tool, final_text) ++ tts_result = json.loads(tts_raw) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 TTS failed") ++ return web.json_response( ++ _openai_error(f"TTS failed after response was generated: {exc}", code="tts_failed"), ++ status=500, ++ headers={ ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ }, ++ ) ++ ++ if not tts_result.get("success"): ++ return web.json_response( ++ _openai_error(tts_result.get("error") or "TTS failed", code="tts_failed"), ++ status=502, ++ headers={ ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ }, ++ ) ++ ++ tts_path = str(tts_result.get("file_path") or "") ++ if not tts_path or not Path(tts_path).exists(): ++ return web.json_response( ++ _openai_error("TTS succeeded but no audio file was produced", code="tts_file_missing"), ++ status=500, ++ ) ++ ++ try: ++ response_audio_path = self._convert_audio_to_esp32_wav( ++ tts_path, ++ output_dir=audio_dir, ++ device_id=device_id, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 WAV normalization failed") ++ return web.json_response( ++ _openai_error(f"Could not convert TTS audio to ESP32 WAV: {exc}", code="wav_conversion_failed"), ++ status=500, ++ headers={ ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ }, ++ ) ++ ++ headers = { ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ "X-Hermes-Response-Id": str(response_data.get("id") or ""), ++ "X-Hermes-Conversation": f"esp32:{device_id}", ++ "X-Hermes-TTS-Source-Type": self._audio_response_mime(tts_path), ++ "Content-Type": "audio/wav", ++ } ++ return web.FileResponse( ++ response_audio_path, ++ headers=headers, ++ ) ++ ++ # ------------------------------------------------------------------ ++ # Cron jobs API ++ # ------------------------------------------------------------------ ++ ++ _JOB_ID_RE = __import__("re").compile(r"[a-f0-9]{12}") ++ # Allowed fields for update — prevents clients injecting arbitrary keys ++ _UPDATE_ALLOWED_FIELDS = {"name", "schedule", "prompt", "deliver", "skills", "skill", "repeat", "enabled"} ++ _MAX_NAME_LENGTH = 200 ++ _MAX_PROMPT_LENGTH = 5000 ++ ++ @staticmethod ++ ++ + async def _handle_models(self, request: "web.Request") -> "web.Response": + """GET /v1/models — list hermes-agent and any configured model_routes aliases.""" + auth_err = self._check_auth(request) +@@ -5018,6 +6651,9 @@ class APIServerAdapter(BasePlatformAdapter): + # bootstrap shims use this key as a feature-detection hook; registering + # native routes first lets those shims no-op instead of shadowing the + # upstream session-control handlers. ++ # Local ESP32 push-to-talk endpoints: POST complete audio, or stream PCM over WebSocket. ++ self._app.router.add_post("/api/esp32/voice", self._handle_esp32_voice) ++ self._app.router.add_get("/api/esp32/voice/ws", self._handle_esp32_voice_ws) + self._app["api_server_adapter"] = self + + # Start background sweep to clean up orphaned (unconsumed) run streams diff --git a/patch/esp32_live_draft_patch.patch b/patch/esp32_live_draft_patch.patch new file mode 100644 index 0000000..a938a35 --- /dev/null +++ b/patch/esp32_live_draft_patch.patch @@ -0,0 +1,1572 @@ +diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py +index 628713224..8bcf9e9cc 100644 +--- a/gateway/platforms/api_server.py ++++ b/gateway/platforms/api_server.py +@@ -34,6 +34,7 @@ Requires: + import asyncio + import hashlib + import hmac ++import ipaddress + import json + from contextlib import contextmanager + from contextvars import ContextVar +@@ -47,6 +48,7 @@ import time + import uuid + from pathlib import Path + from typing import Any, Dict, List, Optional ++from urllib.parse import urlparse + + def _approval_event_choices(*, smart_denied: bool, allow_permanent: bool) -> list[str]: + if smart_denied: +@@ -106,6 +108,47 @@ CHAT_COMPLETIONS_SSE_KEEPALIVE_SECONDS = 30.0 + MAX_NORMALIZED_TEXT_LENGTH = 65_536 # 64 KB cap for normalized content parts + MAX_CONTENT_LIST_SIZE = 1_000 # Max items when content is an array + ++ESP32_AUDIO_MAX_BYTES = 8_000_000 # ~4 minutes of 16 kHz mono PCM WAV; enough for push-to-talk tests ++ESP32_WS_MAX_AUDIO_BYTES = 8_000_000 ++ESP32_WS_MIN_AUDIO_BYTES = 1_600 # 50 ms at 16 kHz mono s16le; rejects empty accidental taps ++ESP32_WS_TTS_CHUNK_BYTES = 16_384 ++ ++# Live draft ingestion config (from whisper-translation/apple_speech engine) ++ESP32_LIVE_DRAFT_ENABLED = os.getenv("ESP32_LIVE_DRAFT_ENABLED", "1").lower() not in {"0", "false", "no", "off"} ++ESP32_LIVE_DRAFT_INTERVAL_MS = int(os.getenv("ESP32_LIVE_DRAFT_INTERVAL_MS", "800")) # how often to snapshot for draft ++ESP32_LIVE_DRAFT_MIN_BYTES = int(os.getenv("ESP32_LIVE_DRAFT_MIN_BYTES", "8000")) # min new bytes since last draft ++ESP32_LIVE_DRAFT_MIN_TOTAL_MS = int(os.getenv("ESP32_LIVE_DRAFT_MIN_TOTAL_MS", "500")) # don't draft until we have 500ms ++ESP32_MACMINI_MCP_URL = os.getenv("MACMINI_MCP_URL") or None # overridable, else from config.yaml mcp_servers.macmini ++ESP32_MACMINI_MCP_TOKEN = os.getenv("MACMINI_MCP_TOKEN") or None ++ESP32_LIVE_DRAFT_USE_MACMINI = os.getenv("ESP32_LIVE_DRAFT_USE_MACMINI", "1").lower() not in {"0", "false", "no", "off"} ++ ++_ESP32_AUDIO_EXTENSIONS = { ++ "audio/wav": ".wav", ++ "audio/wave": ".wav", ++ "audio/x-wav": ".wav", ++ "audio/mpeg": ".mp3", ++ "audio/mp3": ".mp3", ++ "audio/ogg": ".ogg", ++ "audio/opus": ".ogg", ++ "audio/webm": ".webm", ++ "audio/mp4": ".m4a", ++ "audio/x-m4a": ".m4a", ++ "application/octet-stream": ".wav", ++} ++ ++_AUDIO_RESPONSE_MIME_BY_SUFFIX = { ++ ".mp3": "audio/mpeg", ++ ".wav": "audio/wav", ++ ".ogg": "audio/ogg", ++ ".opus": "audio/ogg", ++ ".m4a": "audio/mp4", ++ ".mp4": "audio/mp4", ++ ".flac": "audio/flac", ++} ++ ++ESP32_VOICE_ANIMATION_PATH = "hermes_voice_animations.py" ++ESP32_VOICE_ANIMATION_SCRIPT = "# Hermes ESP32 voice-state animations for MicroPython screen boards.\n# Uploaded by the API server; can also be run manually with:\n# exec(open('hermes_voice_animations.py').read()); run_state('ack')\nimport time\ntry:\n import board_config\nexcept Exception:\n board_config = None\n\n\ndef _display():\n return getattr(board_config, 'display_instance', None) if board_config else None\n\n\ndef _sleep(ms):\n try:\n time.sleep_ms(ms)\n except AttributeError:\n time.sleep(ms / 1000)\n\n\ndef _text(d, msg, x, y, scale=1, color=1):\n if hasattr(d, 'text_large') and scale > 1:\n try:\n d.text_large(msg, x, y, scale=scale, color=color)\n return\n except TypeError:\n pass\n d.text(msg, x, y, color)\n\n\ndef _center_text(d, msg, y, scale=1, color=1):\n width = getattr(d, 'width', 320)\n char_w = 8 * scale\n x = max(0, (width - len(msg) * char_w) // 2)\n _text(d, msg, x, y, scale=scale, color=color)\n\n\ndef _wrap(text, width_chars=34, max_lines=4):\n words = str(text or '').replace('\\n', ' ').split()\n lines, line = [], ''\n for w in words:\n candidate = (line + ' ' + w).strip()\n if len(candidate) > width_chars and line:\n lines.append(line)\n line = w\n if len(lines) >= max_lines:\n break\n else:\n line = candidate\n if line and len(lines) < max_lines:\n lines.append(line)\n return lines\n\n\ndef _spinner(d, title, subtitle='', frames=24, delay=70):\n width, height = getattr(d, 'width', 320), getattr(d, 'height', 240)\n cx, cy = width // 2, height // 2 - 8\n spokes = [(-22,0),(-16,-16),(0,-22),(16,-16),(22,0),(16,16),(0,22),(-16,16)]\n for frame in range(frames):\n d.clear(0)\n _center_text(d, title, 18, scale=2)\n for i, (dx, dy) in enumerate(spokes):\n color = 1 if (i + frame) % len(spokes) in (0, 1, 2, 3) else 0\n size = 4 if i == frame % len(spokes) else 2\n d.fill_rect(cx + dx - size // 2, cy + dy - size // 2, size, size, color)\n d.rect(8, height - 34, width - 16, 18, 1)\n d.fill_rect(10, height - 32, ((width - 20) * ((frame % 16) + 1)) // 16, 14, 1)\n if subtitle:\n _center_text(d, subtitle[:32], height - 56, scale=1)\n d.show()\n _sleep(delay)\n\n\ndef ack(frames=18):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'GOT IT', 18, scale=2)\n # expanding check pulse\n box = 48 + (i % 6) * 8\n x, y = width // 2 - box // 2, height // 2 - box // 2\n d.rect(x, y, box, box, 1)\n d.line(width//2 - 24, height//2, width//2 - 6, height//2 + 18, 1)\n d.line(width//2 - 6, height//2 + 18, width//2 + 30, height//2 - 22, 1)\n _center_text(d, 'Listening captured', height - 36, scale=1)\n d.show()\n _sleep(55)\n\n\ndef captioning(frames=36):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'CAPTIONING', 16, scale=2)\n for bar in range(9):\n h = 8 + ((i * 5 + bar * 13) % 42)\n x = width // 2 - 72 + bar * 18\n d.fill_rect(x, height // 2 - h // 2, 10, h, 1)\n dots = '.' * ((i // 5) % 4)\n _center_text(d, 'Transcribing' + dots, height - 42, scale=1)\n d.show()\n _sleep(65)\n\n\ndef waiting(text='', frames=44):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'THINKING', 12, scale=2)\n r = 18 + (i % 10) * 2\n cx, cy = width // 2, height // 2 - 10\n d.rect(cx - r, cy - r, r * 2, r * 2, 1)\n d.rect(cx - r//2, cy - r//2, r, r, 1)\n if text:\n d.text('Heard:', 10, height - 68, 1)\n for idx, line in enumerate(_wrap(text, 36, 3)):\n d.text(line[:38], 10, height - 54 + idx * 12, 1)\n else:\n _center_text(d, 'Waiting for reply...', height - 42, scale=1)\n d.show()\n _sleep(75)\n\n\ndef run_state(state='ack', text='', frames=None):\n state = str(state or 'ack').lower()\n if state in ('ack', 'listening', 'got_it'):\n ack(frames or 18)\n elif state in ('caption', 'captioning', 'transcribing'):\n captioning(frames or 36)\n elif state in ('wait', 'waiting', 'thinking', 'reply'):\n waiting(text, frames or 44)\n else:\n _spinner(_display(), 'HERMES', state[:28], frames or 20)\n" ++ + + def _coerce_port(value: Any, default: int = DEFAULT_PORT) -> int: + """Parse a listen port without letting malformed env/config values crash startup.""" +@@ -1585,6 +1628,1493 @@ class APIServerAdapter(BasePlatformAdapter): + "pid": os.getpid(), + }) + ++ @staticmethod ++ def _audio_extension_for_content_type(content_type: str) -> str: ++ """Map an inbound audio Content-Type to a safe cache-file suffix.""" ++ media_type = (content_type or "").split(";", 1)[0].strip().lower() ++ return _ESP32_AUDIO_EXTENSIONS.get(media_type, ".wav") ++ ++ @staticmethod ++ def _audio_response_mime(path: str) -> str: ++ """Return an HTTP Content-Type for a generated audio file path.""" ++ suffix = Path(path).suffix.lower() ++ return _AUDIO_RESPONSE_MIME_BY_SUFFIX.get(suffix, "application/octet-stream") ++ ++ @staticmethod ++ ++ def _convert_audio_to_esp32_wav(input_path: str, *, output_dir: Path, device_id: str) -> Path: ++ """Normalize generated TTS audio to ESP32-friendly WAV. ++ ++ Hermes TTS providers commonly emit MP3 (Edge) or OGG. The ESP32 test ++ client expects a small, predictable RIFF/WAVE file, so decode with ++ ffmpeg to raw PCM and rewrite the WAV container with Python's ``wave`` ++ module. That produces a minimal ``fmt`` + ``data`` WAV: mono, 16 kHz, ++ signed 16-bit little-endian PCM. ++ """ ++ import subprocess ++ import wave ++ ++ safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" ++ output_dir.mkdir(parents=True, exist_ok=True) ++ raw_path = output_dir / f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}.s16le" ++ wav_path = raw_path.with_suffix(".wav") ++ cmd = [ ++ "ffmpeg", ++ "-y", ++ "-hide_banner", ++ "-loglevel", ++ "error", ++ "-i", ++ input_path, ++ "-ac", ++ "1", ++ "-ar", ++ "16000", ++ "-f", ++ "s16le", ++ str(raw_path), ++ ] ++ try: ++ subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ++ raw = raw_path.read_bytes() ++ with wave.open(str(wav_path), "wb") as wav: ++ wav.setnchannels(1) ++ wav.setsampwidth(2) ++ wav.setframerate(16000) ++ wav.writeframes(raw) ++ return wav_path ++ finally: ++ try: ++ raw_path.unlink(missing_ok=True) ++ except Exception: ++ pass ++ ++ @staticmethod ++ def _safe_audio_header(value: Any, max_length: int) -> str: ++ """Make diagnostic text safe for HTTP response headers.""" ++ text = str(value or "")[:max_length] ++ return text.replace("\r", " ").replace("\n", " ").replace("\x00", " ") ++ ++ @staticmethod ++ def _esp32_reply_mode(request: "web.Request") -> str: ++ """Return 'sync' or 'ack' for the ESP32 voice response contract.""" ++ value = ( ++ request.headers.get("X-Hermes-Reply-Mode") ++ or request.query.get("reply_mode") ++ or request.query.get("mode") ++ or "sync" ++ ) ++ value = str(value).strip().lower() ++ if value in {"ack", "async", "background", "quick_ack", "quick-ack"}: ++ return "ack" ++ return "sync" ++ ++ @staticmethod ++ def _looks_like_http_url(value: str) -> bool: ++ try: ++ parsed = urlparse(value) ++ return parsed.scheme in {"http", "https"} and bool(parsed.netloc) ++ except Exception: ++ return False ++ ++ @staticmethod ++ def _load_local_device_url(device_name: str) -> Optional[str]: ++ """Best-effort local MCP screen lookup from ~/.hermes/local_devices.yaml.""" ++ if not device_name: ++ return None ++ try: ++ import yaml ++ registry = Path.home() / ".hermes" / "local_devices.yaml" ++ data = yaml.safe_load(registry.read_text()) if registry.exists() else {} ++ aliases = data.get("aliases") if isinstance(data, dict) else None ++ devices = data.get("devices") if isinstance(data, dict) else None ++ names = [str(device_name).strip(), str(device_name).strip().replace("-", "_")] ++ seen: set[str] = set() ++ while names: ++ name = names.pop(0) ++ if not name or name in seen: ++ continue ++ seen.add(name) ++ if isinstance(aliases, dict): ++ alias = aliases.get(name) ++ if isinstance(alias, str) and alias: ++ if alias.startswith("http"): ++ return alias ++ names.append(alias) ++ info = devices.get(name) if isinstance(devices, dict) else None ++ if isinstance(info, dict): ++ return str(info.get("fallback_url") or info.get("url") or "") or None ++ except Exception: ++ return None ++ return None ++ ++ @staticmethod ++ def _load_local_device_url_by_ip(source_ip: str) -> Optional[str]: ++ """Best-effort local MCP screen lookup by observed device IP. ++ ++ Some MCP-capable devices, notably the iPhone companion app, do not run ++ their MCP endpoint on port 80. When the voice gateway sees traffic from ++ one of those devices, prefer the registry URL over synthesizing ++ http:///api/mcp. ++ """ ++ value = str(source_ip or "").strip() ++ if not value: ++ return None ++ if value.startswith("::ffff:"): ++ value = value.removeprefix("::ffff:") ++ try: ++ import yaml ++ registry = Path.home() / ".hermes" / "local_devices.yaml" ++ data = yaml.safe_load(registry.read_text()) if registry.exists() else {} ++ devices = data.get("devices") if isinstance(data, dict) else None ++ if not isinstance(devices, dict): ++ return None ++ for info in devices.values(): ++ if not isinstance(info, dict): ++ continue ++ candidates = { ++ str(info.get("reserved_ip") or "").strip(), ++ str(info.get("host") or "").strip(), ++ } ++ for url_key in ("fallback_url", "url"): ++ raw_url = str(info.get(url_key) or "").strip() ++ if raw_url: ++ parsed = urlparse(raw_url) ++ if parsed.hostname: ++ candidates.add(parsed.hostname) ++ if value in candidates: ++ return str(info.get("fallback_url") or info.get("url") or "") or None ++ except Exception: ++ return None ++ return None ++ ++ @staticmethod ++ def _esp32_screen_url_from_source_ip(source_ip: str) -> Optional[str]: ++ """Build the MCP endpoint for the ESP32 that made this request. ++ ++ ESP32 boards can move between DHCP leases. For voice ACK follow-ups the ++ initiating board is the safest target, so prefer its observed peer IP ++ instead of a possibly stale local-device registry entry. ++ """ ++ value = str(source_ip or "").strip() ++ if not value: ++ return None ++ if value.startswith("::ffff:"): ++ value = value.removeprefix("::ffff:") ++ # Do not route to loopback/proxy/invalid addresses; registry fallback is ++ # better in those cases. ++ try: ++ ip = ipaddress.ip_address(value) ++ except ValueError: ++ return None ++ if ip.is_loopback or ip.is_unspecified or ip.is_multicast: ++ return None ++ host = f"[{value}]" if ip.version == 6 else value ++ return f"http://{host}/api/mcp" ++ ++ def _resolve_esp32_screen_url(self, request: "web.Request", device_id: str) -> Optional[str]: ++ """Resolve the screen MCP endpoint associated with this voice turn.""" ++ explicit = ( ++ request.headers.get("X-Hermes-Screen-Url") ++ or request.headers.get("X-Screen-MCP-Url") ++ or request.query.get("screen_url") ++ ) ++ if explicit and self._looks_like_http_url(explicit.strip()): ++ return explicit.strip() ++ ++ if str(device_id or "").strip().lower().startswith("iphone"): ++ iphone_url = self._load_local_device_url(str(device_id).strip()) ++ if iphone_url and self._looks_like_http_url(iphone_url): ++ return iphone_url ++ ++ audit_ctx = self._request_audit_context(request) ++ for source_ip in (audit_ctx.get("peer_ip"), audit_ctx.get("remote")): ++ registry_url = self._load_local_device_url_by_ip(source_ip or "") ++ if registry_url and self._looks_like_http_url(registry_url): ++ return registry_url ++ ++ audit_ctx = self._request_audit_context(request) ++ for source_ip in (audit_ctx.get("peer_ip"), audit_ctx.get("remote")): ++ source_url = self._esp32_screen_url_from_source_ip(source_ip or "") ++ if source_url: ++ return source_url ++ ++ screen_device = ( ++ request.headers.get("X-Hermes-Screen-Device") ++ or request.query.get("screen_device") ++ or device_id ++ ) ++ candidates = [screen_device, "little32", "esp32_screen"] ++ for candidate in candidates: ++ url = self._load_local_device_url(str(candidate).strip()) ++ if url and self._looks_like_http_url(url): ++ return url ++ return None ++ ++ async def _call_screen_mcp( ++ self, ++ screen_url: Optional[str], ++ tool_name: str, ++ arguments: Dict[str, Any], ++ *, ++ timeout: float = 8.0, ++ ) -> bool: ++ """Call a simple HTTP JSON-RPC MCP screen tool directly.""" ++ if not screen_url: ++ return False ++ try: ++ from aiohttp import ClientSession, ClientTimeout ++ payload = { ++ "jsonrpc": "2.0", ++ "id": uuid.uuid4().hex[:12], ++ "method": "tools/call", ++ "params": {"name": tool_name, "arguments": arguments}, ++ } ++ async with ClientSession(timeout=ClientTimeout(total=timeout)) as session: ++ async with session.post(screen_url, json=payload) as resp: ++ body = await resp.text() ++ if resp.status >= 400: ++ logger.warning( ++ "[api_server] screen MCP %s failed: HTTP %s %s", ++ tool_name, ++ resp.status, ++ body[:300], ++ ) ++ return False ++ try: ++ data = json.loads(body) if body else {} ++ if isinstance(data, dict) and data.get("error"): ++ logger.warning("[api_server] screen MCP %s returned error: %s", tool_name, data.get("error")) ++ return False ++ except Exception: ++ pass ++ return True ++ except Exception as exc: ++ logger.warning("[api_server] screen MCP %s failed: %s", tool_name, exc) ++ return False ++ ++ @staticmethod ++ def _screen_stream_target(screen_url: Optional[str]) -> Optional[tuple[str, int]]: ++ """Return the raw UDP video-stream target for MCP screens that expose one.""" ++ if not screen_url: ++ return None ++ try: ++ parsed = urlparse(screen_url) ++ host = parsed.hostname ++ if not host: ++ return None ++ return (host, 8082) ++ except Exception: ++ return None ++ ++ @staticmethod ++ def _voice_animation_udp_frames(state: str, text: str = "", frames: int = 18) -> List[bytes]: ++ """Render small Hermes voice-state animations as RLCD-compatible frames.""" ++ from PIL import Image, ImageDraw ++ import math ++ ++ width, height = 400, 300 ++ ++ def to_rlcd(img: "Image.Image") -> bytes: ++ img_1bit = img.convert("1", dither=Image.Dither.FLOYDSTEINBERG) ++ px = img_1bit.load() ++ buf = bytearray(15000) ++ for y in range(height): ++ for x in range(width): ++ if px[x, y]: ++ inv_y = 299 - y ++ bx = x // 2 ++ by = inv_y // 4 ++ idx = bx * 75 + by ++ lx = x % 2 ++ ly = inv_y % 4 ++ bit = 7 - (ly * 2 + lx) ++ buf[idx] |= (1 << bit) ++ return bytes(buf) ++ ++ label_by_state = { ++ "ack": "Got it", ++ "captioning": "Listening", ++ "waiting": "Thinking", ++ "speaking": "Replying", ++ "success": "Done", ++ "error": "Try again", ++ } ++ label = label_by_state.get(str(state), str(state or "Hermes").title()) ++ snippet = " ".join(str(text or "").split())[:42] ++ rendered: List[bytes] = [] ++ for frame in range(max(1, int(frames))): ++ img = Image.new("1", (width, height), 0) ++ draw = ImageDraw.Draw(img) ++ cx, cy = width // 2, height // 2 ++ phase = frame * 0.35 ++ if state == "captioning": ++ draw.rectangle((188, 38, 212, 75), outline=1) ++ draw.arc((175, 55, 225, 105), 20, 160, fill=1) ++ for x in range(42, width - 42, 5): ++ y = cy + int(32 * math.sin(x * 0.045 + phase * 2)) ++ y2 = cy + int(14 * math.sin(x * 0.085 - phase * 3)) ++ draw.line((x, y, min(width - 1, x + 4), y2), fill=1) ++ elif state in {"waiting", "speaking"}: ++ for radius in (34, 64, 94): ++ pulse = radius + int(8 * math.sin(phase + radius)) ++ draw.ellipse((cx - pulse, cy - pulse, cx + pulse, cy + pulse), outline=1) ++ for i in range(7): ++ x = 93 + i * 34 ++ bar = int(18 + 42 * (0.5 + 0.5 * math.sin(phase * 2 + i))) ++ draw.rectangle((x, 228 - bar, x + 17, 228), fill=1) ++ elif state == "error": ++ draw.line((150, 88, 250, 188), fill=1) ++ draw.line((250, 88, 150, 188), fill=1) ++ draw.rectangle((145, 82, 255, 194), outline=1) ++ else: ++ r = 42 + int(16 * math.sin(phase)) ++ draw.ellipse((cx - r, cy - r, cx + r, cy + r), outline=1) ++ draw.ellipse((cx - 8, cy - 8, cx + 8, cy + 8), fill=1) ++ draw.text((max(8, (width - len(label) * 6) // 2), 24), label, fill=1) ++ if snippet: ++ draw.text((max(8, (width - len(snippet) * 6) // 2), 260), snippet, fill=1) ++ rendered.append(to_rlcd(img)) ++ return rendered ++ ++ async def _send_stream_animation( ++ self, ++ screen_url: Optional[str], ++ state: str, ++ *, ++ text: str = "", ++ frames: Optional[int] = None, ++ ) -> bool: ++ """Send a server-rendered voice animation over the screen streaming socket.""" ++ target = self._screen_stream_target(screen_url) ++ if not target: ++ return False ++ frame_count = frames if frames is not None else (12 if state == "ack" else 18) ++ try: ++ rendered = await asyncio.to_thread(self._voice_animation_udp_frames, state, text, frame_count) ++ sock = _socket.socket(_socket.AF_INET, _socket.SOCK_DGRAM) ++ try: ++ for frame_id, frame in enumerate(rendered): ++ fid = frame_id % 256 ++ for chunk_idx in range(15): ++ start = chunk_idx * 1000 ++ sock.sendto(bytes([fid, chunk_idx]) + frame[start : start + 1000], target) ++ await asyncio.sleep(0.045) ++ finally: ++ sock.close() ++ return True ++ except Exception as exc: ++ logger.warning("[api_server] screen stream animation failed: %s", exc) ++ return False ++ ++ async def _send_screen_animation( ++ self, ++ screen_url: Optional[str], ++ state: str, ++ *, ++ text: str = "", ++ frames: Optional[int] = None, ++ ) -> bool: ++ """Upload and run the ESP32 voice-state animation script on the initiating screen.""" ++ if not screen_url: ++ return False ++ if await self._send_stream_animation(screen_url, state, text=text, frames=frames): ++ return True ++ if screen_url not in self._esp32_voice_animation_uploaded: ++ uploaded = await self._call_screen_mcp( ++ screen_url, ++ "write_file", ++ {"path": ESP32_VOICE_ANIMATION_PATH, "content": ESP32_VOICE_ANIMATION_SCRIPT}, ++ ) ++ if not uploaded: ++ return False ++ self._esp32_voice_animation_uploaded.add(screen_url) ++ code = ( ++ f"exec(open({ESP32_VOICE_ANIMATION_PATH!r}).read()); " ++ f"run_state({str(state)!r}, text={str(text or '')[:240]!r}, frames={frames!r})" ++ ) ++ return await self._call_screen_mcp(screen_url, "execute_python", {"code": code}) ++ ++ async def _send_screen_text(self, screen_url: Optional[str], text: str, *, clear: bool = False) -> None: ++ """Best-effort text update to the initiating MCP screen.""" ++ if not screen_url: ++ return ++ if clear: ++ await self._call_screen_mcp(screen_url, "clear_screen", {"color": 0}) ++ # Most Reyna screens accept x/y/text. Extra wrapping is avoided here so ++ # firmware-specific clients can keep their own font/layout behavior. ++ await self._call_screen_mcp(screen_url, "draw_text", {"x": 8, "y": 8, "text": text[:900]}) ++ ++ @staticmethod ++ def _extract_first_image_reference(text: str) -> Optional[str]: ++ """Return the first local path, MEDIA path, data URL, or HTTP image URL in text.""" ++ if not text: ++ return None ++ patterns = [ ++ r"MEDIA:([^\s)]+)", ++ r"!\[[^\]]*\]\(([^)]+)\)", ++ r"(?Phttps?://[^\s)]+\.(?:png|jpe?g|webp|gif)(?:\?[^\s)]*)?)", ++ r"(?P/[^\s)]+\.(?:png|jpe?g|webp|gif|bmp))", ++ r"(?Pdata:image/[^\s)]+)", ++ ] ++ for pattern in patterns: ++ for match in re.finditer(pattern, text, flags=re.IGNORECASE): ++ ref = match.groupdict().get("url") or match.groupdict().get("path") or match.groupdict().get("data") or match.group(1) ++ if ref: ++ return ref.strip().strip('"\'') ++ return None ++ ++ async def _send_screen_image_response(self, screen_url: Optional[str], text: str) -> bool: ++ """Best-effort render of the first generated/linked image onto the ESP32 screen.""" ++ if not screen_url: ++ return False ++ image_ref = self._extract_first_image_reference(text) ++ if not image_ref: ++ return False ++ try: ++ import base64 ++ import io ++ from PIL import Image, ImageOps ++ ++ raw: bytes ++ lowered = image_ref.lower() ++ if lowered.startswith("data:image/"): ++ _, _, encoded = image_ref.partition(",") ++ raw = base64.b64decode(encoded) ++ elif self._looks_like_http_url(image_ref): ++ from aiohttp import ClientSession, ClientTimeout ++ async with ClientSession(timeout=ClientTimeout(total=20)) as session: ++ async with session.get(image_ref) as resp: ++ if resp.status >= 400: ++ logger.warning("[api_server] screen image fetch failed: HTTP %s", resp.status) ++ return False ++ raw = await resp.read() ++ else: ++ path = Path(image_ref).expanduser() ++ if not path.exists() or not path.is_file(): ++ return False ++ raw = await asyncio.to_thread(path.read_bytes) ++ if not raw: ++ return False ++ ++ def render_pbm() -> str: ++ with Image.open(io.BytesIO(raw)) as img: ++ img = ImageOps.exif_transpose(img).convert("RGB") ++ canvas = Image.new("RGB", (320, 240), "white") ++ img.thumbnail((320, 240), Image.Resampling.LANCZOS) ++ x = (320 - img.width) // 2 ++ y = (240 - img.height) // 2 ++ canvas.paste(img, (x, y)) ++ mono = canvas.convert("1", dither=Image.Dither.FLOYDSTEINBERG) ++ out = io.BytesIO() ++ mono.save(out, format="PPM") # mode=1 writes raw PBM (P4) ++ return base64.b64encode(out.getvalue()).decode("ascii") ++ ++ pbm_base64 = await asyncio.to_thread(render_pbm) ++ return await self._call_screen_mcp( ++ screen_url, ++ "draw_image", ++ {"pbm_base64": pbm_base64, "x": 0, "y": 0, "dither": False}, ++ timeout=15, ++ ) ++ except Exception: ++ logger.exception("[api_server] failed to render ESP32 screen image response") ++ return False ++ ++ async def _send_screen_audio_response( ++ self, ++ screen_url: Optional[str], ++ text: str, ++ *, ++ audio_dir: Path, ++ device_id: str, ++ volume: int = 80, ++ ) -> bool: ++ """Best-effort spoken follow-up for ACK-mode ESP32 voice turns.""" ++ if not screen_url or not text.strip(): ++ return False ++ try: ++ import base64 ++ from tools.tts_tool import text_to_speech_tool ++ ++ # Keep board playback short and payloads small. The ESP32 HTTP MCP ++ # endpoint resets on large base64 WAV bodies; MP3 is much smaller ++ # and the firmware exposes play_mp3_base64 for exactly this path. ++ spoken_text = self._short_voice_summary(text, 420) ++ tts_raw = await asyncio.to_thread(text_to_speech_tool, spoken_text) ++ tts_result = json.loads(tts_raw) ++ src = str(tts_result.get("file_path") or "") ++ if not tts_result.get("success") or not src or not Path(src).exists(): ++ logger.warning( ++ "[api_server] screen audio TTS failed: %s", ++ tts_result.get("error") or "missing TTS output", ++ ) ++ return False ++ ++ src_path = Path(src) ++ if src_path.suffix.lower() == ".mp3": ++ mp3_base64 = base64.b64encode(src_path.read_bytes()).decode("ascii") ++ if await self._call_screen_mcp( ++ screen_url, ++ "play_mp3_base64", ++ {"mp3_base64": mp3_base64, "volume": volume}, ++ timeout=75, ++ ): ++ return True ++ ++ wav_path = await asyncio.to_thread( ++ self._convert_audio_to_esp32_wav, ++ src, ++ output_dir=audio_dir, ++ device_id=f"{device_id}_response", ++ ) ++ wav_bytes = wav_path.read_bytes() ++ if len(wav_bytes) > 220_000: ++ logger.warning( ++ "[api_server] screen audio WAV too large for MCP payload: %s bytes", ++ len(wav_bytes), ++ ) ++ return False ++ wav_base64 = base64.b64encode(wav_bytes).decode("ascii") ++ return await self._call_screen_mcp( ++ screen_url, ++ "play_audio_base64", ++ {"wav_base64": wav_base64, "volume": volume}, ++ timeout=45, ++ ) ++ except Exception: ++ logger.exception("[api_server] failed to play ESP32 screen audio response") ++ return False ++ ++ @staticmethod ++ def _short_voice_summary(text: str, limit: int = 220) -> str: ++ """Local fallback summary for voice/screen snippets without another LLM call.""" ++ clean = " ".join(str(text or "").split()) ++ if len(clean) <= limit: ++ return clean ++ return clean[: max(0, limit - 1)].rstrip() + "…" ++ ++ async def _esp32_ack_wav(self, *, audio_dir: Path, device_id: str) -> Path: ++ """Return a reusable quiet ACK WAV; the visual screen animation is the acknowledgement.""" ++ ack_path = audio_dir / "quick_ack.wav" ++ if ack_path.exists(): ++ return ack_path ++ ++ # ACK-mode clients expect a valid audio/wav response immediately, but ++ # Reyna ESP32 screens now show the acknowledgement visually. Keep the ++ # HTTP contract while avoiding the old spoken "Got it" message. ++ import wave ++ sample_rate = 16000 ++ duration_s = 0.08 ++ frames = b"\x00\x00" * int(sample_rate * duration_s) ++ with wave.open(str(ack_path), "wb") as wav: ++ wav.setnchannels(1) ++ wav.setsampwidth(2) ++ wav.setframerate(sample_rate) ++ wav.writeframes(frames) ++ return ack_path ++ ++ def _append_esp32_voice_audit(self, audio_dir: Path, event: Dict[str, Any]) -> None: ++ """Append a best-effort ESP32 voice request audit event.""" ++ try: ++ log_path = audio_dir / "voice_requests.jsonl" ++ with log_path.open("a", encoding="utf-8") as handle: ++ handle.write(json.dumps(event, default=str) + "\n") ++ except Exception: ++ logger.debug("[api_server] failed to append ESP32 voice audit log", exc_info=True) ++ ++ async def _run_esp32_voice_audio_background( ++ self, ++ *, ++ audio_path: Path, ++ audit_event: Dict[str, Any], ++ device_id: str, ++ instructions: Optional[str], ++ audio_dir: Path, ++ screen_url: Optional[str], ++ ) -> None: ++ """Transcribe uploaded ESP32 audio and continue the voice turn after an immediate ACK.""" ++ try: ++ await self._send_screen_animation(screen_url, "captioning") ++ from tools.transcription_tools import transcribe_audio ++ stt_result = await asyncio.to_thread(transcribe_audio, str(audio_path)) ++ if not stt_result.get("success"): ++ failed_event = dict(audit_event) ++ failed_event.update({ ++ "status": "transcription_failed", ++ "error": stt_result.get("error") or "Transcription failed", ++ "timestamp": time.time(), ++ }) ++ self._append_esp32_voice_audit(audio_dir, failed_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but transcription failed. Try again.", ++ clear=True, ++ ) ++ return ++ transcript = str(stt_result.get("transcript") or "").strip() ++ if not transcript: ++ empty_event = dict(audit_event) ++ empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) ++ self._append_esp32_voice_audit(audio_dir, empty_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but got no words. Try again.", ++ clear=True, ++ ) ++ return ++ transcript_event = dict(audit_event) ++ transcript_event.update({"status": "transcribed", "transcript": transcript, "timestamp": time.time()}) ++ self._append_esp32_voice_audit(audio_dir, transcript_event) ++ await self._run_esp32_voice_background( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=instructions, ++ audio_dir=audio_dir, ++ screen_url=screen_url, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 ACK background voice processing failed") ++ failed_event = dict(audit_event) ++ failed_event.update({"status": "background_exception", "error": str(exc), "timestamp": time.time()}) ++ self._append_esp32_voice_audit(audio_dir, failed_event) ++ await self._send_screen_text( ++ screen_url, ++ "I got the audio, but background processing failed.", ++ clear=True, ++ ) ++ ++ async def _run_esp32_voice_background( ++ self, ++ *, ++ transcript: str, ++ device_id: str, ++ instructions: Optional[str], ++ audio_dir: Path, ++ screen_url: Optional[str], ++ ) -> None: ++ """Run the slow ESP32 voice turn after the device already heard an ACK.""" ++ try: ++ animation_ok = await self._send_screen_animation(screen_url, "waiting", text=transcript) ++ if not animation_ok: ++ await self._send_screen_text( ++ screen_url, ++ f"Heard: {self._short_voice_summary(transcript, 180)}\n\nWorking…", ++ clear=True, ++ ) ++ screen_hint = ( ++ "\n\nThis request came from an ESP32 voice/screen device. " ++ "Keep the spoken follow-up short. The gateway will display the full answer on the initiating screen when possible." ++ ) ++ merged_instructions = (instructions or "") + screen_hint ++ response_data, final_text = await self._run_esp32_voice_turn( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=merged_instructions, ++ ) ++ if not final_text.strip(): ++ final_text = self._extract_response_text(response_data) or "I finished, but I do not have a text answer." ++ display_text = final_text ++ if len(display_text) > 1200: ++ display_text = self._short_voice_summary(display_text, 900) + "\n\nFull answer is long; WhatsApp delivery is the next wiring step." ++ await self._send_screen_text(screen_url, display_text, clear=True) ++ await self._send_screen_image_response(screen_url, final_text) ++ await self._send_screen_audio_response( ++ screen_url, ++ final_text, ++ audio_dir=audio_dir, ++ device_id=device_id, ++ ) ++ except Exception: ++ logger.exception("[api_server] ESP32 background voice turn failed") ++ await self._send_screen_text(screen_url, "Hermes hit an error while preparing the full answer.", clear=True) ++ ++ @staticmethod ++ def _extract_response_text(response_data: Dict[str, Any]) -> str: ++ """Extract assistant text from an OpenAI Responses-shaped payload.""" ++ chunks: List[str] = [] ++ for item in response_data.get("output") or []: ++ if not isinstance(item, dict) or item.get("type") != "message": ++ continue ++ for part in item.get("content") or []: ++ if isinstance(part, dict) and part.get("type") in {"output_text", "text"}: ++ text = part.get("text") ++ if isinstance(text, str) and text: ++ chunks.append(text) ++ return "\n".join(chunks).strip() ++ ++ async def _run_esp32_voice_turn( ++ self, ++ *, ++ transcript: str, ++ device_id: str, ++ instructions: Optional[str] = None, ++ ) -> tuple[Dict[str, Any], str]: ++ """Run one persistent Responses-style turn for an ESP32 voice device.""" ++ conversation = f"esp32:{device_id}" ++ previous_response_id = self._response_store.get_conversation(conversation) ++ conversation_history: List[Dict[str, Any]] = [] ++ stored_session_id = None ++ stored_instructions = None ++ ++ if previous_response_id: ++ stored = self._response_store.get(previous_response_id) ++ if stored is not None: ++ conversation_history = list(stored.get("conversation_history", [])) ++ stored_session_id = stored.get("session_id") ++ stored_instructions = stored.get("instructions") ++ ++ if instructions is None: ++ instructions = stored_instructions ++ ++ session_id = stored_session_id or str(uuid.uuid4()) ++ user_message = ( ++ f'[Voice input from ESP32 device "{device_id}"]\n' ++ f'{transcript}' ++ ) ++ ++ result, usage = await self._run_agent( ++ user_message=user_message, ++ conversation_history=conversation_history, ++ ephemeral_system_prompt=instructions, ++ session_id=session_id, ++ gateway_session_key=conversation, ++ ) ++ ++ final_response = result.get("final_response", "") or result.get("error", "") or "(No response generated)" ++ response_id = f"resp_{uuid.uuid4().hex[:28]}" ++ created_at = int(time.time()) ++ full_history = self._build_response_conversation_history( ++ conversation_history, ++ user_message, ++ result, ++ final_response, ++ ) ++ output_start_index = self._response_messages_turn_start_index( ++ conversation_history, ++ user_message, ++ result, ++ ) ++ output_items = self._extract_output_items(result, start_index=output_start_index) ++ response_data = { ++ "id": response_id, ++ "object": "response", ++ "status": "completed", ++ "created_at": created_at, ++ "model": self._model_name, ++ "output": output_items, ++ "usage": { ++ "input_tokens": usage.get("input_tokens", 0), ++ "output_tokens": usage.get("output_tokens", 0), ++ "total_tokens": usage.get("total_tokens", 0), ++ }, ++ } ++ self._response_store.put(response_id, { ++ "response": response_data, ++ "conversation_history": full_history, ++ "instructions": instructions, ++ "session_id": session_id, ++ }) ++ self._response_store.set_conversation(conversation, response_id) ++ return response_data, final_response ++ ++ @staticmethod ++ def _esp32_audio_cache_dir() -> Path: ++ """Return the cache directory for ESP32 voice artifacts.""" ++ try: ++ from hermes_constants import get_hermes_dir ++ return get_hermes_dir("cache/audio", "audio_cache") / "esp32" ++ except Exception: ++ return Path.home() / ".hermes" / "cache" / "audio" / "esp32" ++ ++ @staticmethod ++ def _write_pcm_s16le_wav(raw_path: Path, wav_path: Path, *, sample_rate: int, channels: int = 1) -> None: ++ """Wrap a raw PCM s16le file in a minimal RIFF/WAVE container.""" ++ import wave ++ ++ with wave.open(str(wav_path), "wb") as wav: ++ wav.setnchannels(channels) ++ wav.setsampwidth(2) ++ wav.setframerate(sample_rate) ++ with raw_path.open("rb") as raw_handle: ++ while True: ++ chunk = raw_handle.read(64 * 1024) ++ if not chunk: ++ break ++ wav.writeframes(chunk) ++ ++ # --- Live draft helpers (from your whisper-translation/apple_speech pipe protocol) --- ++ def _load_macmini_mcp_config(self) -> dict: ++ try: ++ if ESP32_MACMINI_MCP_URL and ESP32_MACMINI_MCP_TOKEN: ++ return {"url": ESP32_MACMINI_MCP_URL, "token": ESP32_MACMINI_MCP_TOKEN} ++ cfg_path = Path.home() / ".hermes" / "config.yaml" ++ if cfg_path.exists(): ++ import yaml ++ cfg = yaml.safe_load(cfg_path.read_text()) or {} ++ m = (cfg.get("mcp_servers") or {}).get("macmini") or {} ++ url = m.get("url") or "" ++ token = "" ++ headers = m.get("headers") or {} ++ auth = headers.get("Authorization") or "" ++ if auth.startswith("Bearer "): ++ token = auth[7:] ++ if url: ++ return {"url": url, "token": token} ++ except Exception: ++ pass ++ return {} ++ ++ @staticmethod ++ def _s16le_to_wav_bytes(pcm_bytes: bytes, sample_rate: int = 16000, channels: int = 1) -> bytes: ++ import io, wave ++ bio = io.BytesIO() ++ with wave.open(bio, "wb") as wf: ++ wf.setnchannels(channels) ++ wf.setsampwidth(2) ++ wf.setframerate(sample_rate) ++ wf.writeframes(pcm_bytes) ++ return bio.getvalue() ++ ++ async def _transcribe_draft_via_macmini(self, wav_bytes: bytes, locale: str = "en-US") -> str | None: ++ if not ESP32_LIVE_DRAFT_USE_MACMINI: ++ return None ++ cfg = self._load_macmini_mcp_config() ++ url = cfg.get("url") ++ token = cfg.get("token") ++ if not url: ++ return None ++ try: ++ import base64 ++ b64 = base64.b64encode(wav_bytes).decode("ascii") ++ payload = { ++ "jsonrpc": "2.0", ++ "id": 1, ++ "method": "tools/call", ++ "params": {"name": "speech_live_transcribe", "arguments": {"audioBase64": b64, "locale": locale}}, ++ } ++ headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"} ++ if token: ++ headers["Authorization"] = f"Bearer {token}" ++ import aiohttp ++ async with aiohttp.ClientSession() as session: ++ async with session.post(url, json=payload, headers=headers, timeout=aiohttp.ClientTimeout(total=8)) as resp: ++ if resp.status != 200: ++ return None ++ txt = await resp.text() ++ for line in txt.splitlines(): ++ if not line.startswith("data: "): ++ continue ++ try: ++ data = json.loads(line[6:]) ++ content = (data.get("result", {}).get("content") or [{}])[0].get("text", "") or "" ++ if not content: ++ continue ++ inner = json.loads(content) ++ t = inner.get("text") or "" ++ if t: ++ return t.strip() ++ except Exception: ++ continue ++ except Exception as e: ++ logger.debug(f"[api_server] macmini live draft failed: {e}") ++ return None ++ ++ async def _transcribe_draft_local(self, wav_bytes: bytes) -> str | None: ++ try: ++ import tempfile ++ fd, tmp_path = tempfile.mkstemp(suffix=".wav") ++ os.close(fd) ++ Path(tmp_path).write_bytes(wav_bytes) ++ def _do(): ++ try: ++ from faster_whisper import WhisperModel ++ model = None ++ try: ++ from tools.transcription_tools import _local_model ++ if _local_model is not None: ++ model = _local_model ++ except Exception: ++ pass ++ if model is None: ++ model = WhisperModel("tiny", device="cpu", compute_type="int8", num_workers=1) ++ segs, _ = model.transcribe(tmp_path, beam_size=1, best_of=1, language="en", condition_on_previous_text=False, vad_filter=True) ++ return " ".join(s.text.strip() for s in segs).strip() ++ except Exception as ex: ++ logger.debug(f"[api_server] local draft err: {ex}") ++ return None ++ finally: ++ try: ++ Path(tmp_path).unlink(missing_ok=True) ++ except Exception: ++ pass ++ return await asyncio.to_thread(_do) ++ except Exception: ++ return None ++ ++ async def _handle_esp32_voice_ws(self, request: "web.Request") -> "web.WebSocketResponse": ++ """GET /api/esp32/voice/ws — ESP32 PCM streaming with LIVE draft captioning. ++ LIVE DRAFT MODE (from whisper-translation/engine_apple_transcribe.py): ++ - While ingesting binary s16le, every ~800ms snapshots current PCM -> in-mem WAV ++ -> tries Mac mini speech_live_transcribe (ApplePipeTranscriber --pipe volatile, ++ <200ms drafts) fallback to local tiny for fast feedback. ++ - Emits WS events: draft + interim_transcript for UI. Final via macmini or local base. ++ """ ++ auth_err = self._check_auth(request) ++ if auth_err: ++ return auth_err # type: ignore[return-value] ++ ++ ws = web.WebSocketResponse(heartbeat=None, max_msg_size=512 * 1024) ++ await ws.prepare(request) ++ ++ async def _send_event(event: str, **payload): ++ if not ws.closed: ++ await ws.send_str(json.dumps({"event": event, **payload})) ++ ++ device_id = ( ++ request.headers.get("X-Device-ID") ++ or request.query.get("device_id") ++ or "default" ++ ).strip()[:80] or "default" ++ safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" ++ audio_dir = self._esp32_audio_cache_dir() ++ audio_dir.mkdir(parents=True, exist_ok=True) ++ stamp = f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}" ++ raw_path = audio_dir / f"{stamp}.s16le" ++ wav_path = audio_dir / f"{stamp}.wav" ++ sample_rate = 16000 ++ channels = 1 ++ sample_width = 2 ++ total_audio_bytes = 0 ++ started = False ++ stopped = False ++ first_audio_at: float | None = None ++ started_at = time.time() ++ # live draft state (directly from your whisper-translation ApplePipeTranscriber pattern) ++ live_enabled = ESP32_LIVE_DRAFT_ENABLED ++ pcm_buffer = bytearray() ++ last_draft_at = 0.0 ++ last_draft_bytes = 0 ++ last_draft_text = "" ++ draft_in_progress = False ++ draft_count = 0 ++ locale_req = request.headers.get("X-Hermes-Locale") or request.query.get("locale") or "en-US" ++ ++ audit_ctx = self._request_audit_context(request) ++ audit_event = { ++ "timestamp": started_at, ++ "device_id": device_id, ++ "source_ip": audit_ctx.get("remote") or audit_ctx.get("peer_ip") or "", ++ "peer_ip": audit_ctx.get("peer_ip") or "", ++ "forwarded_for": audit_ctx.get("forwarded_for") or "", ++ "real_ip": audit_ctx.get("real_ip") or "", ++ "user_agent": audit_ctx.get("user_agent") or "", ++ "transport": "websocket", ++ "raw_path": str(raw_path), ++ "audio_path": str(wav_path), ++ "status": "accepted", ++ "live_draft_enabled": live_enabled, ++ } ++ self._append_esp32_voice_audit(audio_dir, audit_event) ++ screen_url = self._resolve_esp32_screen_url(request, device_id) ++ visual_listening_sent = False ++ ++ def _schedule_voice_state(state: str, *, text: str = "", frames: int | None = None) -> None: ++ if not screen_url: ++ return ++ task = asyncio.create_task(self._send_screen_animation(screen_url, state, text=text, frames=frames)) ++ try: ++ self._background_tasks.add(task) ++ except TypeError: ++ pass ++ if hasattr(task, "add_done_callback"): ++ task.add_done_callback(self._background_tasks.discard) ++ ++ async def _do_live_draft(snapshot_bytes: bytes): ++ nonlocal last_draft_text, draft_in_progress, draft_count ++ if draft_in_progress: ++ return ++ if len(snapshot_bytes) < int(sample_rate * 0.5 * 2): # at least 500ms of s16le at 16k mono ++ return ++ draft_in_progress = True ++ try: ++ wav_b = self._s16le_to_wav_bytes(snapshot_bytes, sample_rate=sample_rate, channels=channels) ++ txt = None ++ if ESP32_LIVE_DRAFT_USE_MACMINI: ++ txt = await self._transcribe_draft_via_macmini(wav_b, locale=locale_req) ++ if not txt: ++ txt = await self._transcribe_draft_local(wav_b) ++ if not txt: ++ return ++ txt = txt.strip() ++ if not txt or txt == last_draft_text or len(txt) < 2: ++ return ++ last_draft_text = txt ++ draft_count += 1 ++ await _send_event("draft", text=txt, is_final=False, isFinal=False, chunk=draft_count, bytes=len(snapshot_bytes)) ++ await _send_event("interim_transcript", text=txt, is_final=False, draft_count=draft_count) ++ if screen_url and len(txt) > 1: ++ try: ++ t = asyncio.create_task(self._send_screen_text(screen_url, txt, clear=False)) ++ self._background_tasks.add(t) ++ t.add_done_callback(self._background_tasks.discard) ++ except Exception: ++ pass ++ finally: ++ draft_in_progress = False ++ ++ await _send_event("ready", device_id=device_id, screen_url=screen_url or "", live_draft=live_enabled, live_model="apple-pipe-volatile") ++ ++ try: ++ raw_handle = raw_path.open("wb") ++ try: ++ async for msg in ws: ++ if msg.type == web.WSMsgType.TEXT: ++ try: ++ payload = json.loads(msg.data) ++ except json.JSONDecodeError: ++ await _send_event("error", code="bad_json", message="Text frames must be JSON") ++ continue ++ event = str(payload.get("event") or payload.get("type") or "").strip().lower() ++ if event == "start": ++ if started: ++ await _send_event("error", code="already_started", message="Audio stream already started") ++ continue ++ fmt = str(payload.get("format") or "pcm_s16le").strip().lower() ++ channels = int(payload.get("channels") or 1) ++ sample_width = int(payload.get("sample_width") or 2) ++ sample_rate = int(payload.get("sample_rate") or 16000) ++ device_id = str(payload.get("device_id") or device_id).strip()[:80] or device_id ++ if fmt not in {"pcm_s16le", "s16le", "pcm"} or channels != 1 or sample_width != 2: ++ await _send_event("error", code="unsupported_audio_format", message="Use 16 kHz mono signed 16-bit little-endian PCM") ++ await ws.close() ++ return ws ++ if sample_rate < 8000 or sample_rate > 48000: ++ await _send_event("error", code="bad_sample_rate", message="sample_rate must be 8000-48000") ++ await ws.close() ++ return ws ++ started = True ++ if not visual_listening_sent: ++ _schedule_voice_state("captioning", frames=8) ++ visual_listening_sent = True ++ await _send_event("listening", sample_rate=sample_rate, channels=channels, sample_width=sample_width) ++ elif event in {"stop", "end"}: ++ stopped = True ++ break ++ elif event in {"ping", "keepalive"}: ++ await _send_event("pong") ++ else: ++ await _send_event("error", code="unknown_event", message=f"Unknown event: {event or ''}") ++ elif msg.type == web.WSMsgType.BINARY: ++ if not started: ++ started = True ++ if not visual_listening_sent: ++ _schedule_voice_state("captioning", frames=8) ++ visual_listening_sent = True ++ await _send_event("listening", sample_rate=sample_rate, channels=channels, sample_width=sample_width) ++ chunk = bytes(msg.data or b"") ++ if not chunk: ++ continue ++ if first_audio_at is None: ++ first_audio_at = time.time() ++ total_audio_bytes += len(chunk) ++ pcm_buffer.extend(chunk) ++ if total_audio_bytes > ESP32_WS_MAX_AUDIO_BYTES: ++ await _send_event("error", code="audio_too_large", message=f"Audio stream exceeded {ESP32_WS_MAX_AUDIO_BYTES} bytes") ++ await ws.close() ++ return ws ++ raw_handle.write(chunk) ++ # live draft trigger (every 800ms & 8KB new data) — direct captioning fluff stripped ++ if live_enabled: ++ now = time.time() ++ elapsed_ms = (now - last_draft_at) * 1000 ++ new_bytes = len(pcm_buffer) - last_draft_bytes ++ total_ms = len(pcm_buffer) / 2 / sample_rate * 1000 ++ if total_ms >= ESP32_LIVE_DRAFT_MIN_TOTAL_MS and elapsed_ms >= ESP32_LIVE_DRAFT_INTERVAL_MS and new_bytes >= ESP32_LIVE_DRAFT_MIN_BYTES: ++ last_draft_at = now ++ last_draft_bytes = len(pcm_buffer) ++ # copy snapshot, fire-and-forget draft (but guard concurrent) ++ snapshot = bytes(pcm_buffer) ++ asyncio.create_task(_do_live_draft(snapshot)) ++ elif msg.type == web.WSMsgType.ERROR: ++ logger.warning("[api_server] ESP32 voice websocket error: %s", ws.exception()) ++ break ++ finally: ++ raw_handle.close() ++ ++ if not stopped: ++ await _send_event("error", code="missing_stop", message="Send a stop event after audio chunks") ++ await ws.close() ++ return ws ++ if total_audio_bytes < ESP32_WS_MIN_AUDIO_BYTES: ++ await _send_event("error", code="audio_too_short", message="Not enough audio received") ++ await ws.close() ++ return ws ++ ++ await asyncio.to_thread( ++ self._write_pcm_s16le_wav, ++ raw_path, ++ wav_path, ++ sample_rate=sample_rate, ++ channels=channels, ++ ) ++ audit_event.update({ ++ "status": "received", ++ "audio_bytes": total_audio_bytes, ++ "sample_rate": sample_rate, ++ "first_audio_latency_ms": int(((first_audio_at or started_at) - started_at) * 1000), ++ "stop_latency_ms": int((time.time() - started_at) * 1000), ++ "draft_count": draft_count, ++ "last_draft": last_draft_text, ++ }) ++ self._append_esp32_voice_audit(audio_dir, dict(audit_event)) ++ ++ _schedule_voice_state("captioning", frames=10) ++ await _send_event("thinking", stage="transcribing", draft_count=draft_count) ++ ++ # Final transcription: prefer Mac mini live for accuracy (not just draft, full file via speech_transcribe_file) ++ transcript = None ++ if ESP32_LIVE_DRAFT_USE_MACMINI: ++ try: ++ cfg = self._load_macmini_mcp_config() ++ if cfg.get("url"): ++ # Try final via speech_transcribe_file (file mode, more accurate than draft) ++ # Need to copy wav to mac? Actually speech_transcribe_file expects file path on Mac ++ # So we keep local transcription as final for now, but if draft == good use it. ++ # Future: upload via SCP or use live transcribe for final with full buffer. ++ pass ++ except Exception: ++ pass ++ ++ from tools.transcription_tools import transcribe_audio ++ stt_result = await asyncio.to_thread(transcribe_audio, str(wav_path)) ++ if not stt_result.get("success"): ++ message = stt_result.get("error") or "Transcription failed" ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "transcription_failed", "error": message, "timestamp": time.time()}) ++ await _send_event("error", code="transcription_failed", message=message) ++ await ws.close() ++ return ws ++ final_txt = str(stt_result.get("transcript") or "").strip() ++ if not final_txt: ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "empty_transcript", "timestamp": time.time()}) ++ await _send_event("error", code="empty_transcript", message="Transcription was empty") ++ await ws.close() ++ return ws ++ transcript = final_txt ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "transcribed", "transcript": transcript, "timestamp": time.time(), "draft_count": draft_count}) ++ await _send_event("transcript", text=transcript, is_final=True, isFinal=True, draft_count=draft_count) ++ await _send_event("final", text=transcript, isFinal=True) ++ ++ summary = self._short_voice_summary(transcript, 180) ++ ack_text = f"Heard: {summary}\n\nWorking on the full answer now." ++ await _send_event("response_text", text=ack_text, response_id="", final=False) ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "queued_background", "transcript": transcript, "timestamp": time.time()}) ++ ++ try: ++ await _send_event("done", mode="background", screen_url=screen_url or "", draft_count=draft_count) ++ await ws.close() ++ except Exception: ++ pass ++ background_task = asyncio.create_task( ++ self._run_esp32_voice_background( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=request.headers.get("X-Hermes-Instructions") or None, ++ audio_dir=audio_dir, ++ screen_url=screen_url, ++ ) ++ ) ++ try: ++ self._background_tasks.add(background_task) ++ except TypeError: ++ pass ++ if hasattr(background_task, "add_done_callback"): ++ background_task.add_done_callback(self._background_tasks.discard) ++ return ws ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 voice websocket processing failed") ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "websocket_exception", "error": str(exc), "timestamp": time.time()}) ++ try: ++ await _send_event("error", code="internal_error", message=str(exc)) ++ await ws.close() ++ except Exception: ++ pass ++ return ws ++ finally: ++ try: ++ raw_path.unlink(missing_ok=True) ++ except Exception: ++ pass ++ ++ async def _handle_esp32_voice(self, request: "web.Request") -> "web.Response": ++ ++ """POST /api/esp32/voice — receive audio, run Hermes, return TTS audio.""" ++ auth_err = self._check_auth(request) ++ if auth_err: ++ return auth_err ++ ++ device_id = ( ++ request.headers.get("X-Device-ID") ++ or request.query.get("device_id") ++ or "default" ++ ).strip()[:80] or "default" ++ ++ try: ++ audio_bytes = await request.read() ++ except Exception as exc: ++ return web.json_response(_openai_error(f"Failed to read audio body: {exc}"), status=400) ++ ++ if not audio_bytes: ++ return web.json_response(_openai_error("Empty audio body", code="empty_audio"), status=400) ++ if len(audio_bytes) > ESP32_AUDIO_MAX_BYTES: ++ return web.json_response( ++ _openai_error( ++ f"Audio body is too large ({len(audio_bytes)} bytes); max is {ESP32_AUDIO_MAX_BYTES}", ++ code="audio_too_large", ++ ), ++ status=413, ++ ) ++ ++ suffix = self._audio_extension_for_content_type(request.headers.get("Content-Type", "")) ++ try: ++ from hermes_constants import get_hermes_dir ++ audio_dir = get_hermes_dir("cache/audio", "audio_cache") / "esp32" ++ except Exception: ++ audio_dir = Path.home() / ".hermes" / "cache" / "audio" / "esp32" ++ audio_dir.mkdir(parents=True, exist_ok=True) ++ safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" ++ audio_path = audio_dir / f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}{suffix}" ++ audio_path.write_bytes(audio_bytes) ++ ++ audit_ctx = self._request_audit_context(request) ++ audit_event: Dict[str, Any] = { ++ "timestamp": time.time(), ++ "device_id": device_id, ++ "source_ip": audit_ctx.get("remote") or audit_ctx.get("peer_ip") or "", ++ "peer_ip": audit_ctx.get("peer_ip") or "", ++ "forwarded_for": audit_ctx.get("forwarded_for") or "", ++ "real_ip": audit_ctx.get("real_ip") or "", ++ "user_agent": audit_ctx.get("user_agent") or "", ++ "reply_mode": self._esp32_reply_mode(request), ++ "screen_device": request.headers.get("X-Hermes-Screen-Device") or request.query.get("screen_device") or "", ++ "screen_url": request.headers.get("X-Hermes-Screen-Url") or request.headers.get("X-Screen-MCP-Url") or request.query.get("screen_url") or "", ++ "content_type": request.headers.get("Content-Type", ""), ++ "audio_bytes": len(audio_bytes), ++ "audio_path": str(audio_path), ++ "status": "received", ++ } ++ ++ def _append_audit(event: Dict[str, Any]) -> None: ++ try: ++ log_path = audio_dir / "voice_requests.jsonl" ++ with log_path.open("a", encoding="utf-8") as handle: ++ handle.write(json.dumps(event, default=str) + "\n") ++ except Exception: ++ logger.debug("[api_server] failed to append ESP32 voice audit log", exc_info=True) ++ ++ _append_audit(audit_event) ++ ++ if self._esp32_reply_mode(request) == "ack": ++ screen_url = self._resolve_esp32_screen_url(request, device_id) ++ ++ async def _ack_background() -> None: ++ try: ++ await self._send_screen_animation(screen_url, "ack") ++ await self._send_screen_animation(screen_url, "captioning") ++ from tools.transcription_tools import transcribe_audio ++ stt_result_bg = await asyncio.wait_for( ++ asyncio.to_thread(transcribe_audio, str(audio_path)), ++ timeout=45, ++ ) ++ if not stt_result_bg.get("success"): ++ failed_event = dict(audit_event) ++ failed_event.update({ ++ "status": "transcription_failed", ++ "error": stt_result_bg.get("error") or "Transcription failed", ++ "timestamp": time.time(), ++ }) ++ _append_audit(failed_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but transcription failed. Try again.", ++ clear=True, ++ ) ++ return ++ transcript_bg = str(stt_result_bg.get("transcript") or "").strip() ++ if not transcript_bg: ++ empty_event = dict(audit_event) ++ empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) ++ _append_audit(empty_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but got no words. Try again.", ++ clear=True, ++ ) ++ return ++ transcript_event = dict(audit_event) ++ transcript_event.update({"status": "transcribed", "transcript": transcript_bg, "timestamp": time.time()}) ++ _append_audit(transcript_event) ++ await self._run_esp32_voice_background( ++ transcript=transcript_bg, ++ device_id=device_id, ++ instructions=request.headers.get("X-Hermes-Instructions") or None, ++ audio_dir=audio_dir, ++ screen_url=screen_url, ++ ) ++ except asyncio.TimeoutError: ++ timeout_event = dict(audit_event) ++ timeout_event.update({"status": "transcription_timeout", "error": "Transcription exceeded 45s", "timestamp": time.time()}) ++ _append_audit(timeout_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but captioning timed out. Try again.", ++ clear=True, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 ack-mode background processing failed") ++ failed_event = dict(audit_event) ++ failed_event.update({"status": "background_exception", "error": str(exc), "timestamp": time.time()}) ++ _append_audit(failed_event) ++ ++ task = asyncio.create_task(_ack_background()) ++ try: ++ self._background_tasks.add(task) ++ except TypeError: ++ pass ++ if hasattr(task, "add_done_callback"): ++ task.add_done_callback(self._background_tasks.discard) ++ ack_path = await self._esp32_ack_wav(audio_dir=audio_dir, device_id=device_id) ++ headers = { ++ "X-Hermes-Conversation": f"esp32:{device_id}", ++ "X-Hermes-Reply-Mode": "ack", ++ "X-Hermes-Screen-Url": screen_url or "", ++ "Content-Type": "audio/wav", ++ } ++ return web.FileResponse(ack_path, headers=headers) ++ ++ try: ++ from tools.transcription_tools import transcribe_audio ++ stt_result = await asyncio.to_thread(transcribe_audio, str(audio_path)) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 transcription failed") ++ failed_event = dict(audit_event) ++ failed_event.update({"status": "transcription_exception", "error": str(exc), "timestamp": time.time()}) ++ _append_audit(failed_event) ++ return web.json_response(_openai_error(f"Transcription failed: {exc}"), status=500) ++ ++ if not stt_result.get("success"): ++ failed_event = dict(audit_event) ++ failed_event.update({ ++ "status": "transcription_failed", ++ "error": stt_result.get("error") or "Transcription failed", ++ "timestamp": time.time(), ++ }) ++ _append_audit(failed_event) ++ return web.json_response( ++ _openai_error(stt_result.get("error") or "Transcription failed", code="transcription_failed"), ++ status=502, ++ ) ++ ++ transcript = str(stt_result.get("transcript") or "").strip() ++ if not transcript: ++ empty_event = dict(audit_event) ++ empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) ++ _append_audit(empty_event) ++ return web.json_response(_openai_error("Transcription was empty", code="empty_transcript"), status=422) ++ ++ transcript_event = dict(audit_event) ++ transcript_event.update({"status": "transcribed", "transcript": transcript, "timestamp": time.time()}) ++ _append_audit(transcript_event) ++ ++ instructions = request.headers.get("X-Hermes-Instructions") or None ++ try: ++ response_data, final_text = await self._run_esp32_voice_turn( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=instructions, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 agent turn failed") ++ return web.json_response(_openai_error(f"Agent turn failed: {exc}"), status=500) ++ ++ if not final_text.strip(): ++ final_text = self._extract_response_text(response_data) or "I heard you, but I do not have a spoken answer." ++ ++ try: ++ from tools.tts_tool import text_to_speech_tool ++ tts_raw = await asyncio.to_thread(text_to_speech_tool, final_text) ++ tts_result = json.loads(tts_raw) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 TTS failed") ++ return web.json_response( ++ _openai_error(f"TTS failed after response was generated: {exc}", code="tts_failed"), ++ status=500, ++ headers={ ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ }, ++ ) ++ ++ if not tts_result.get("success"): ++ return web.json_response( ++ _openai_error(tts_result.get("error") or "TTS failed", code="tts_failed"), ++ status=502, ++ headers={ ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ }, ++ ) ++ ++ tts_path = str(tts_result.get("file_path") or "") ++ if not tts_path or not Path(tts_path).exists(): ++ return web.json_response( ++ _openai_error("TTS succeeded but no audio file was produced", code="tts_file_missing"), ++ status=500, ++ ) ++ ++ try: ++ response_audio_path = self._convert_audio_to_esp32_wav( ++ tts_path, ++ output_dir=audio_dir, ++ device_id=device_id, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 WAV normalization failed") ++ return web.json_response( ++ _openai_error(f"Could not convert TTS audio to ESP32 WAV: {exc}", code="wav_conversion_failed"), ++ status=500, ++ headers={ ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ }, ++ ) ++ ++ headers = { ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ "X-Hermes-Response-Id": str(response_data.get("id") or ""), ++ "X-Hermes-Conversation": f"esp32:{device_id}", ++ "X-Hermes-TTS-Source-Type": self._audio_response_mime(tts_path), ++ "Content-Type": "audio/wav", ++ } ++ return web.FileResponse( ++ response_audio_path, ++ headers=headers, ++ ) ++ ++ # ------------------------------------------------------------------ ++ # Cron jobs API ++ # ------------------------------------------------------------------ ++ ++ _JOB_ID_RE = __import__("re").compile(r"[a-f0-9]{12}") ++ # Allowed fields for update — prevents clients injecting arbitrary keys ++ _UPDATE_ALLOWED_FIELDS = {"name", "schedule", "prompt", "deliver", "skills", "skill", "repeat", "enabled"} ++ _MAX_NAME_LENGTH = 200 ++ _MAX_PROMPT_LENGTH = 5000 ++ ++ @staticmethod ++ ++ + async def _handle_models(self, request: "web.Request") -> "web.Response": + """GET /v1/models — list hermes-agent and any configured model_routes aliases.""" + auth_err = self._check_auth(request) +@@ -5018,6 +6548,9 @@ class APIServerAdapter(BasePlatformAdapter): + # bootstrap shims use this key as a feature-detection hook; registering + # native routes first lets those shims no-op instead of shadowing the + # upstream session-control handlers. ++ # Local ESP32 push-to-talk endpoints: POST complete audio, or stream PCM over WebSocket. ++ self._app.router.add_post("/api/esp32/voice", self._handle_esp32_voice) ++ self._app.router.add_get("/api/esp32/voice/ws", self._handle_esp32_voice_ws) + self._app["api_server_adapter"] = self + + # Start background sweep to clean up orphaned (unconsumed) run streams diff --git a/patch/esp32_voice_patch.patch b/patch/esp32_voice_patch.patch new file mode 100644 index 0000000..71b1852 --- /dev/null +++ b/patch/esp32_voice_patch.patch @@ -0,0 +1,1383 @@ +diff --git a/gateway/platforms/api_server.py b/gateway/platforms/api_server.py +index 628713224..2691159a9 100644 +--- a/gateway/platforms/api_server.py ++++ b/gateway/platforms/api_server.py +@@ -34,6 +34,7 @@ Requires: + import asyncio + import hashlib + import hmac ++import ipaddress + import json + from contextlib import contextmanager + from contextvars import ContextVar +@@ -47,6 +48,7 @@ import time + import uuid + from pathlib import Path + from typing import Any, Dict, List, Optional ++from urllib.parse import urlparse + + def _approval_event_choices(*, smart_denied: bool, allow_permanent: bool) -> list[str]: + if smart_denied: +@@ -106,6 +108,38 @@ CHAT_COMPLETIONS_SSE_KEEPALIVE_SECONDS = 30.0 + MAX_NORMALIZED_TEXT_LENGTH = 65_536 # 64 KB cap for normalized content parts + MAX_CONTENT_LIST_SIZE = 1_000 # Max items when content is an array + ++ESP32_AUDIO_MAX_BYTES = 8_000_000 # ~4 minutes of 16 kHz mono PCM WAV; enough for push-to-talk tests ++ESP32_WS_MAX_AUDIO_BYTES = ESP32_AUDIO_MAX_BYTES ++ESP32_WS_MIN_AUDIO_BYTES = 1_600 # 50 ms at 16 kHz mono s16le; rejects empty accidental taps ++ESP32_WS_TTS_CHUNK_BYTES = 16_384 ++ ++_ESP32_AUDIO_EXTENSIONS = { ++ "audio/wav": ".wav", ++ "audio/wave": ".wav", ++ "audio/x-wav": ".wav", ++ "audio/mpeg": ".mp3", ++ "audio/mp3": ".mp3", ++ "audio/ogg": ".ogg", ++ "audio/opus": ".ogg", ++ "audio/webm": ".webm", ++ "audio/mp4": ".m4a", ++ "audio/x-m4a": ".m4a", ++ "application/octet-stream": ".wav", ++} ++ ++_AUDIO_RESPONSE_MIME_BY_SUFFIX = { ++ ".mp3": "audio/mpeg", ++ ".wav": "audio/wav", ++ ".ogg": "audio/ogg", ++ ".opus": "audio/ogg", ++ ".m4a": "audio/mp4", ++ ".mp4": "audio/mp4", ++ ".flac": "audio/flac", ++} ++ ++ESP32_VOICE_ANIMATION_PATH = "hermes_voice_animations.py" ++ESP32_VOICE_ANIMATION_SCRIPT = "# Hermes ESP32 voice-state animations for MicroPython screen boards.\n# Uploaded by the API server; can also be run manually with:\n# exec(open('hermes_voice_animations.py').read()); run_state('ack')\nimport time\ntry:\n import board_config\nexcept Exception:\n board_config = None\n\n\ndef _display():\n return getattr(board_config, 'display_instance', None) if board_config else None\n\n\ndef _sleep(ms):\n try:\n time.sleep_ms(ms)\n except AttributeError:\n time.sleep(ms / 1000)\n\n\ndef _text(d, msg, x, y, scale=1, color=1):\n if hasattr(d, 'text_large') and scale > 1:\n try:\n d.text_large(msg, x, y, scale=scale, color=color)\n return\n except TypeError:\n pass\n d.text(msg, x, y, color)\n\n\ndef _center_text(d, msg, y, scale=1, color=1):\n width = getattr(d, 'width', 320)\n char_w = 8 * scale\n x = max(0, (width - len(msg) * char_w) // 2)\n _text(d, msg, x, y, scale=scale, color=color)\n\n\ndef _wrap(text, width_chars=34, max_lines=4):\n words = str(text or '').replace('\\n', ' ').split()\n lines, line = [], ''\n for w in words:\n candidate = (line + ' ' + w).strip()\n if len(candidate) > width_chars and line:\n lines.append(line)\n line = w\n if len(lines) >= max_lines:\n break\n else:\n line = candidate\n if line and len(lines) < max_lines:\n lines.append(line)\n return lines\n\n\ndef _spinner(d, title, subtitle='', frames=24, delay=70):\n width, height = getattr(d, 'width', 320), getattr(d, 'height', 240)\n cx, cy = width // 2, height // 2 - 8\n spokes = [(-22,0),(-16,-16),(0,-22),(16,-16),(22,0),(16,16),(0,22),(-16,16)]\n for frame in range(frames):\n d.clear(0)\n _center_text(d, title, 18, scale=2)\n for i, (dx, dy) in enumerate(spokes):\n color = 1 if (i + frame) % len(spokes) in (0, 1, 2, 3) else 0\n size = 4 if i == frame % len(spokes) else 2\n d.fill_rect(cx + dx - size // 2, cy + dy - size // 2, size, size, color)\n d.rect(8, height - 34, width - 16, 18, 1)\n d.fill_rect(10, height - 32, ((width - 20) * ((frame % 16) + 1)) // 16, 14, 1)\n if subtitle:\n _center_text(d, subtitle[:32], height - 56, scale=1)\n d.show()\n _sleep(delay)\n\n\ndef ack(frames=18):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'GOT IT', 18, scale=2)\n # expanding check pulse\n box = 48 + (i % 6) * 8\n x, y = width // 2 - box // 2, height // 2 - box // 2\n d.rect(x, y, box, box, 1)\n d.line(width//2 - 24, height//2, width//2 - 6, height//2 + 18, 1)\n d.line(width//2 - 6, height//2 + 18, width//2 + 30, height//2 - 22, 1)\n _center_text(d, 'Listening captured', height - 36, scale=1)\n d.show()\n _sleep(55)\n\n\ndef captioning(frames=36):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'CAPTIONING', 16, scale=2)\n for bar in range(9):\n h = 8 + ((i * 5 + bar * 13) % 42)\n x = width // 2 - 72 + bar * 18\n d.fill_rect(x, height // 2 - h // 2, 10, h, 1)\n dots = '.' * ((i // 5) % 4)\n _center_text(d, 'Transcribing' + dots, height - 42, scale=1)\n d.show()\n _sleep(65)\n\n\ndef waiting(text='', frames=44):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'THINKING', 12, scale=2)\n r = 18 + (i % 10) * 2\n cx, cy = width // 2, height // 2 - 10\n d.rect(cx - r, cy - r, r * 2, r * 2, 1)\n d.rect(cx - r//2, cy - r//2, r, r, 1)\n if text:\n d.text('Heard:', 10, height - 68, 1)\n for idx, line in enumerate(_wrap(text, 36, 3)):\n d.text(line[:38], 10, height - 54 + idx * 12, 1)\n else:\n _center_text(d, 'Waiting for reply...', height - 42, scale=1)\n d.show()\n _sleep(75)\n\n\ndef run_state(state='ack', text='', frames=None):\n state = str(state or 'ack').lower()\n if state in ('ack', 'listening', 'got_it'):\n ack(frames or 18)\n elif state in ('caption', 'captioning', 'transcribing'):\n captioning(frames or 36)\n elif state in ('wait', 'waiting', 'thinking', 'reply'):\n waiting(text, frames or 44)\n else:\n _spinner(_display(), 'HERMES', state[:28], frames or 20)\n" ++ + + def _coerce_port(value: Any, default: int = DEFAULT_PORT) -> int: + """Parse a listen port without letting malformed env/config values crash startup.""" +@@ -1585,6 +1619,1313 @@ class APIServerAdapter(BasePlatformAdapter): + "pid": os.getpid(), + }) + ++ @staticmethod ++ def _audio_extension_for_content_type(content_type: str) -> str: ++ """Map an inbound audio Content-Type to a safe cache-file suffix.""" ++ media_type = (content_type or "").split(";", 1)[0].strip().lower() ++ return _ESP32_AUDIO_EXTENSIONS.get(media_type, ".wav") ++ ++ @staticmethod ++ def _audio_response_mime(path: str) -> str: ++ """Return an HTTP Content-Type for a generated audio file path.""" ++ suffix = Path(path).suffix.lower() ++ return _AUDIO_RESPONSE_MIME_BY_SUFFIX.get(suffix, "application/octet-stream") ++ ++ @staticmethod ++ ++ def _convert_audio_to_esp32_wav(input_path: str, *, output_dir: Path, device_id: str) -> Path: ++ """Normalize generated TTS audio to ESP32-friendly WAV. ++ ++ Hermes TTS providers commonly emit MP3 (Edge) or OGG. The ESP32 test ++ client expects a small, predictable RIFF/WAVE file, so decode with ++ ffmpeg to raw PCM and rewrite the WAV container with Python's ``wave`` ++ module. That produces a minimal ``fmt`` + ``data`` WAV: mono, 16 kHz, ++ signed 16-bit little-endian PCM. ++ """ ++ import subprocess ++ import wave ++ ++ safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" ++ output_dir.mkdir(parents=True, exist_ok=True) ++ raw_path = output_dir / f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}.s16le" ++ wav_path = raw_path.with_suffix(".wav") ++ cmd = [ ++ "ffmpeg", ++ "-y", ++ "-hide_banner", ++ "-loglevel", ++ "error", ++ "-i", ++ input_path, ++ "-ac", ++ "1", ++ "-ar", ++ "16000", ++ "-f", ++ "s16le", ++ str(raw_path), ++ ] ++ try: ++ subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ++ raw = raw_path.read_bytes() ++ with wave.open(str(wav_path), "wb") as wav: ++ wav.setnchannels(1) ++ wav.setsampwidth(2) ++ wav.setframerate(16000) ++ wav.writeframes(raw) ++ return wav_path ++ finally: ++ try: ++ raw_path.unlink(missing_ok=True) ++ except Exception: ++ pass ++ ++ @staticmethod ++ def _safe_audio_header(value: Any, max_length: int) -> str: ++ """Make diagnostic text safe for HTTP response headers.""" ++ text = str(value or "")[:max_length] ++ return text.replace("\r", " ").replace("\n", " ").replace("\x00", " ") ++ ++ @staticmethod ++ def _esp32_reply_mode(request: "web.Request") -> str: ++ """Return 'sync' or 'ack' for the ESP32 voice response contract.""" ++ value = ( ++ request.headers.get("X-Hermes-Reply-Mode") ++ or request.query.get("reply_mode") ++ or request.query.get("mode") ++ or "sync" ++ ) ++ value = str(value).strip().lower() ++ if value in {"ack", "async", "background", "quick_ack", "quick-ack"}: ++ return "ack" ++ return "sync" ++ ++ @staticmethod ++ def _looks_like_http_url(value: str) -> bool: ++ try: ++ parsed = urlparse(value) ++ return parsed.scheme in {"http", "https"} and bool(parsed.netloc) ++ except Exception: ++ return False ++ ++ @staticmethod ++ def _load_local_device_url(device_name: str) -> Optional[str]: ++ """Best-effort local MCP screen lookup from ~/.hermes/local_devices.yaml.""" ++ if not device_name: ++ return None ++ try: ++ import yaml ++ registry = Path.home() / ".hermes" / "local_devices.yaml" ++ data = yaml.safe_load(registry.read_text()) if registry.exists() else {} ++ aliases = data.get("aliases") if isinstance(data, dict) else None ++ devices = data.get("devices") if isinstance(data, dict) else None ++ names = [str(device_name).strip(), str(device_name).strip().replace("-", "_")] ++ seen: set[str] = set() ++ while names: ++ name = names.pop(0) ++ if not name or name in seen: ++ continue ++ seen.add(name) ++ if isinstance(aliases, dict): ++ alias = aliases.get(name) ++ if isinstance(alias, str) and alias: ++ if alias.startswith("http"): ++ return alias ++ names.append(alias) ++ info = devices.get(name) if isinstance(devices, dict) else None ++ if isinstance(info, dict): ++ return str(info.get("fallback_url") or info.get("url") or "") or None ++ except Exception: ++ return None ++ return None ++ ++ @staticmethod ++ def _load_local_device_url_by_ip(source_ip: str) -> Optional[str]: ++ """Best-effort local MCP screen lookup by observed device IP. ++ ++ Some MCP-capable devices, notably the iPhone companion app, do not run ++ their MCP endpoint on port 80. When the voice gateway sees traffic from ++ one of those devices, prefer the registry URL over synthesizing ++ http:///api/mcp. ++ """ ++ value = str(source_ip or "").strip() ++ if not value: ++ return None ++ if value.startswith("::ffff:"): ++ value = value.removeprefix("::ffff:") ++ try: ++ import yaml ++ registry = Path.home() / ".hermes" / "local_devices.yaml" ++ data = yaml.safe_load(registry.read_text()) if registry.exists() else {} ++ devices = data.get("devices") if isinstance(data, dict) else None ++ if not isinstance(devices, dict): ++ return None ++ for info in devices.values(): ++ if not isinstance(info, dict): ++ continue ++ candidates = { ++ str(info.get("reserved_ip") or "").strip(), ++ str(info.get("host") or "").strip(), ++ } ++ for url_key in ("fallback_url", "url"): ++ raw_url = str(info.get(url_key) or "").strip() ++ if raw_url: ++ parsed = urlparse(raw_url) ++ if parsed.hostname: ++ candidates.add(parsed.hostname) ++ if value in candidates: ++ return str(info.get("fallback_url") or info.get("url") or "") or None ++ except Exception: ++ return None ++ return None ++ ++ @staticmethod ++ def _esp32_screen_url_from_source_ip(source_ip: str) -> Optional[str]: ++ """Build the MCP endpoint for the ESP32 that made this request. ++ ++ ESP32 boards can move between DHCP leases. For voice ACK follow-ups the ++ initiating board is the safest target, so prefer its observed peer IP ++ instead of a possibly stale local-device registry entry. ++ """ ++ value = str(source_ip or "").strip() ++ if not value: ++ return None ++ if value.startswith("::ffff:"): ++ value = value.removeprefix("::ffff:") ++ # Do not route to loopback/proxy/invalid addresses; registry fallback is ++ # better in those cases. ++ try: ++ ip = ipaddress.ip_address(value) ++ except ValueError: ++ return None ++ if ip.is_loopback or ip.is_unspecified or ip.is_multicast: ++ return None ++ host = f"[{value}]" if ip.version == 6 else value ++ return f"http://{host}/api/mcp" ++ ++ def _resolve_esp32_screen_url(self, request: "web.Request", device_id: str) -> Optional[str]: ++ """Resolve the screen MCP endpoint associated with this voice turn.""" ++ explicit = ( ++ request.headers.get("X-Hermes-Screen-Url") ++ or request.headers.get("X-Screen-MCP-Url") ++ or request.query.get("screen_url") ++ ) ++ if explicit and self._looks_like_http_url(explicit.strip()): ++ return explicit.strip() ++ ++ if str(device_id or "").strip().lower().startswith("iphone"): ++ iphone_url = self._load_local_device_url(str(device_id).strip()) ++ if iphone_url and self._looks_like_http_url(iphone_url): ++ return iphone_url ++ ++ audit_ctx = self._request_audit_context(request) ++ for source_ip in (audit_ctx.get("peer_ip"), audit_ctx.get("remote")): ++ registry_url = self._load_local_device_url_by_ip(source_ip or "") ++ if registry_url and self._looks_like_http_url(registry_url): ++ return registry_url ++ ++ audit_ctx = self._request_audit_context(request) ++ for source_ip in (audit_ctx.get("peer_ip"), audit_ctx.get("remote")): ++ source_url = self._esp32_screen_url_from_source_ip(source_ip or "") ++ if source_url: ++ return source_url ++ ++ screen_device = ( ++ request.headers.get("X-Hermes-Screen-Device") ++ or request.query.get("screen_device") ++ or device_id ++ ) ++ candidates = [screen_device, "little32", "esp32_screen"] ++ for candidate in candidates: ++ url = self._load_local_device_url(str(candidate).strip()) ++ if url and self._looks_like_http_url(url): ++ return url ++ return None ++ ++ async def _call_screen_mcp( ++ self, ++ screen_url: Optional[str], ++ tool_name: str, ++ arguments: Dict[str, Any], ++ *, ++ timeout: float = 8.0, ++ ) -> bool: ++ """Call a simple HTTP JSON-RPC MCP screen tool directly.""" ++ if not screen_url: ++ return False ++ try: ++ from aiohttp import ClientSession, ClientTimeout ++ payload = { ++ "jsonrpc": "2.0", ++ "id": uuid.uuid4().hex[:12], ++ "method": "tools/call", ++ "params": {"name": tool_name, "arguments": arguments}, ++ } ++ async with ClientSession(timeout=ClientTimeout(total=timeout)) as session: ++ async with session.post(screen_url, json=payload) as resp: ++ body = await resp.text() ++ if resp.status >= 400: ++ logger.warning( ++ "[api_server] screen MCP %s failed: HTTP %s %s", ++ tool_name, ++ resp.status, ++ body[:300], ++ ) ++ return False ++ try: ++ data = json.loads(body) if body else {} ++ if isinstance(data, dict) and data.get("error"): ++ logger.warning("[api_server] screen MCP %s returned error: %s", tool_name, data.get("error")) ++ return False ++ except Exception: ++ pass ++ return True ++ except Exception as exc: ++ logger.warning("[api_server] screen MCP %s failed: %s", tool_name, exc) ++ return False ++ ++ @staticmethod ++ def _screen_stream_target(screen_url: Optional[str]) -> Optional[tuple[str, int]]: ++ """Return the raw UDP video-stream target for MCP screens that expose one.""" ++ if not screen_url: ++ return None ++ try: ++ parsed = urlparse(screen_url) ++ host = parsed.hostname ++ if not host: ++ return None ++ return (host, 8082) ++ except Exception: ++ return None ++ ++ @staticmethod ++ def _voice_animation_udp_frames(state: str, text: str = "", frames: int = 18) -> List[bytes]: ++ """Render small Hermes voice-state animations as RLCD-compatible frames.""" ++ from PIL import Image, ImageDraw ++ import math ++ ++ width, height = 400, 300 ++ ++ def to_rlcd(img: "Image.Image") -> bytes: ++ img_1bit = img.convert("1", dither=Image.Dither.FLOYDSTEINBERG) ++ px = img_1bit.load() ++ buf = bytearray(15000) ++ for y in range(height): ++ for x in range(width): ++ if px[x, y]: ++ inv_y = 299 - y ++ bx = x // 2 ++ by = inv_y // 4 ++ idx = bx * 75 + by ++ lx = x % 2 ++ ly = inv_y % 4 ++ bit = 7 - (ly * 2 + lx) ++ buf[idx] |= (1 << bit) ++ return bytes(buf) ++ ++ label_by_state = { ++ "ack": "Got it", ++ "captioning": "Listening", ++ "waiting": "Thinking", ++ "speaking": "Replying", ++ "success": "Done", ++ "error": "Try again", ++ } ++ label = label_by_state.get(str(state), str(state or "Hermes").title()) ++ snippet = " ".join(str(text or "").split())[:42] ++ rendered: List[bytes] = [] ++ for frame in range(max(1, int(frames))): ++ img = Image.new("1", (width, height), 0) ++ draw = ImageDraw.Draw(img) ++ cx, cy = width // 2, height // 2 ++ phase = frame * 0.35 ++ if state == "captioning": ++ draw.rectangle((188, 38, 212, 75), outline=1) ++ draw.arc((175, 55, 225, 105), 20, 160, fill=1) ++ for x in range(42, width - 42, 5): ++ y = cy + int(32 * math.sin(x * 0.045 + phase * 2)) ++ y2 = cy + int(14 * math.sin(x * 0.085 - phase * 3)) ++ draw.line((x, y, min(width - 1, x + 4), y2), fill=1) ++ elif state in {"waiting", "speaking"}: ++ for radius in (34, 64, 94): ++ pulse = radius + int(8 * math.sin(phase + radius)) ++ draw.ellipse((cx - pulse, cy - pulse, cx + pulse, cy + pulse), outline=1) ++ for i in range(7): ++ x = 93 + i * 34 ++ bar = int(18 + 42 * (0.5 + 0.5 * math.sin(phase * 2 + i))) ++ draw.rectangle((x, 228 - bar, x + 17, 228), fill=1) ++ elif state == "error": ++ draw.line((150, 88, 250, 188), fill=1) ++ draw.line((250, 88, 150, 188), fill=1) ++ draw.rectangle((145, 82, 255, 194), outline=1) ++ else: ++ r = 42 + int(16 * math.sin(phase)) ++ draw.ellipse((cx - r, cy - r, cx + r, cy + r), outline=1) ++ draw.ellipse((cx - 8, cy - 8, cx + 8, cy + 8), fill=1) ++ draw.text((max(8, (width - len(label) * 6) // 2), 24), label, fill=1) ++ if snippet: ++ draw.text((max(8, (width - len(snippet) * 6) // 2), 260), snippet, fill=1) ++ rendered.append(to_rlcd(img)) ++ return rendered ++ ++ async def _send_stream_animation( ++ self, ++ screen_url: Optional[str], ++ state: str, ++ *, ++ text: str = "", ++ frames: Optional[int] = None, ++ ) -> bool: ++ """Send a server-rendered voice animation over the screen streaming socket.""" ++ target = self._screen_stream_target(screen_url) ++ if not target: ++ return False ++ frame_count = frames if frames is not None else (12 if state == "ack" else 18) ++ try: ++ rendered = await asyncio.to_thread(self._voice_animation_udp_frames, state, text, frame_count) ++ sock = _socket.socket(_socket.AF_INET, _socket.SOCK_DGRAM) ++ try: ++ for frame_id, frame in enumerate(rendered): ++ fid = frame_id % 256 ++ for chunk_idx in range(15): ++ start = chunk_idx * 1000 ++ sock.sendto(bytes([fid, chunk_idx]) + frame[start : start + 1000], target) ++ await asyncio.sleep(0.045) ++ finally: ++ sock.close() ++ return True ++ except Exception as exc: ++ logger.warning("[api_server] screen stream animation failed: %s", exc) ++ return False ++ ++ async def _send_screen_animation( ++ self, ++ screen_url: Optional[str], ++ state: str, ++ *, ++ text: str = "", ++ frames: Optional[int] = None, ++ ) -> bool: ++ """Upload and run the ESP32 voice-state animation script on the initiating screen.""" ++ if not screen_url: ++ return False ++ if await self._send_stream_animation(screen_url, state, text=text, frames=frames): ++ return True ++ if screen_url not in self._esp32_voice_animation_uploaded: ++ uploaded = await self._call_screen_mcp( ++ screen_url, ++ "write_file", ++ {"path": ESP32_VOICE_ANIMATION_PATH, "content": ESP32_VOICE_ANIMATION_SCRIPT}, ++ ) ++ if not uploaded: ++ return False ++ self._esp32_voice_animation_uploaded.add(screen_url) ++ code = ( ++ f"exec(open({ESP32_VOICE_ANIMATION_PATH!r}).read()); " ++ f"run_state({str(state)!r}, text={str(text or '')[:240]!r}, frames={frames!r})" ++ ) ++ return await self._call_screen_mcp(screen_url, "execute_python", {"code": code}) ++ ++ async def _send_screen_text(self, screen_url: Optional[str], text: str, *, clear: bool = False) -> None: ++ """Best-effort text update to the initiating MCP screen.""" ++ if not screen_url: ++ return ++ if clear: ++ await self._call_screen_mcp(screen_url, "clear_screen", {"color": 0}) ++ # Most Reyna screens accept x/y/text. Extra wrapping is avoided here so ++ # firmware-specific clients can keep their own font/layout behavior. ++ await self._call_screen_mcp(screen_url, "draw_text", {"x": 8, "y": 8, "text": text[:900]}) ++ ++ @staticmethod ++ def _extract_first_image_reference(text: str) -> Optional[str]: ++ """Return the first local path, MEDIA path, data URL, or HTTP image URL in text.""" ++ if not text: ++ return None ++ patterns = [ ++ r"MEDIA:([^\s)]+)", ++ r"!\[[^\]]*\]\(([^)]+)\)", ++ r"(?Phttps?://[^\s)]+\.(?:png|jpe?g|webp|gif)(?:\?[^\s)]*)?)", ++ r"(?P/[^\s)]+\.(?:png|jpe?g|webp|gif|bmp))", ++ r"(?Pdata:image/[^\s)]+)", ++ ] ++ for pattern in patterns: ++ for match in re.finditer(pattern, text, flags=re.IGNORECASE): ++ ref = match.groupdict().get("url") or match.groupdict().get("path") or match.groupdict().get("data") or match.group(1) ++ if ref: ++ return ref.strip().strip('"\'') ++ return None ++ ++ async def _send_screen_image_response(self, screen_url: Optional[str], text: str) -> bool: ++ """Best-effort render of the first generated/linked image onto the ESP32 screen.""" ++ if not screen_url: ++ return False ++ image_ref = self._extract_first_image_reference(text) ++ if not image_ref: ++ return False ++ try: ++ import base64 ++ import io ++ from PIL import Image, ImageOps ++ ++ raw: bytes ++ lowered = image_ref.lower() ++ if lowered.startswith("data:image/"): ++ _, _, encoded = image_ref.partition(",") ++ raw = base64.b64decode(encoded) ++ elif self._looks_like_http_url(image_ref): ++ from aiohttp import ClientSession, ClientTimeout ++ async with ClientSession(timeout=ClientTimeout(total=20)) as session: ++ async with session.get(image_ref) as resp: ++ if resp.status >= 400: ++ logger.warning("[api_server] screen image fetch failed: HTTP %s", resp.status) ++ return False ++ raw = await resp.read() ++ else: ++ path = Path(image_ref).expanduser() ++ if not path.exists() or not path.is_file(): ++ return False ++ raw = await asyncio.to_thread(path.read_bytes) ++ if not raw: ++ return False ++ ++ def render_pbm() -> str: ++ with Image.open(io.BytesIO(raw)) as img: ++ img = ImageOps.exif_transpose(img).convert("RGB") ++ canvas = Image.new("RGB", (320, 240), "white") ++ img.thumbnail((320, 240), Image.Resampling.LANCZOS) ++ x = (320 - img.width) // 2 ++ y = (240 - img.height) // 2 ++ canvas.paste(img, (x, y)) ++ mono = canvas.convert("1", dither=Image.Dither.FLOYDSTEINBERG) ++ out = io.BytesIO() ++ mono.save(out, format="PPM") # mode=1 writes raw PBM (P4) ++ return base64.b64encode(out.getvalue()).decode("ascii") ++ ++ pbm_base64 = await asyncio.to_thread(render_pbm) ++ return await self._call_screen_mcp( ++ screen_url, ++ "draw_image", ++ {"pbm_base64": pbm_base64, "x": 0, "y": 0, "dither": False}, ++ timeout=15, ++ ) ++ except Exception: ++ logger.exception("[api_server] failed to render ESP32 screen image response") ++ return False ++ ++ async def _send_screen_audio_response( ++ self, ++ screen_url: Optional[str], ++ text: str, ++ *, ++ audio_dir: Path, ++ device_id: str, ++ volume: int = 80, ++ ) -> bool: ++ """Best-effort spoken follow-up for ACK-mode ESP32 voice turns.""" ++ if not screen_url or not text.strip(): ++ return False ++ try: ++ import base64 ++ from tools.tts_tool import text_to_speech_tool ++ ++ # Keep board playback short and payloads small. The ESP32 HTTP MCP ++ # endpoint resets on large base64 WAV bodies; MP3 is much smaller ++ # and the firmware exposes play_mp3_base64 for exactly this path. ++ spoken_text = self._short_voice_summary(text, 420) ++ tts_raw = await asyncio.to_thread(text_to_speech_tool, spoken_text) ++ tts_result = json.loads(tts_raw) ++ src = str(tts_result.get("file_path") or "") ++ if not tts_result.get("success") or not src or not Path(src).exists(): ++ logger.warning( ++ "[api_server] screen audio TTS failed: %s", ++ tts_result.get("error") or "missing TTS output", ++ ) ++ return False ++ ++ src_path = Path(src) ++ if src_path.suffix.lower() == ".mp3": ++ mp3_base64 = base64.b64encode(src_path.read_bytes()).decode("ascii") ++ if await self._call_screen_mcp( ++ screen_url, ++ "play_mp3_base64", ++ {"mp3_base64": mp3_base64, "volume": volume}, ++ timeout=75, ++ ): ++ return True ++ ++ wav_path = await asyncio.to_thread( ++ self._convert_audio_to_esp32_wav, ++ src, ++ output_dir=audio_dir, ++ device_id=f"{device_id}_response", ++ ) ++ wav_bytes = wav_path.read_bytes() ++ if len(wav_bytes) > 220_000: ++ logger.warning( ++ "[api_server] screen audio WAV too large for MCP payload: %s bytes", ++ len(wav_bytes), ++ ) ++ return False ++ wav_base64 = base64.b64encode(wav_bytes).decode("ascii") ++ return await self._call_screen_mcp( ++ screen_url, ++ "play_audio_base64", ++ {"wav_base64": wav_base64, "volume": volume}, ++ timeout=45, ++ ) ++ except Exception: ++ logger.exception("[api_server] failed to play ESP32 screen audio response") ++ return False ++ ++ @staticmethod ++ def _short_voice_summary(text: str, limit: int = 220) -> str: ++ """Local fallback summary for voice/screen snippets without another LLM call.""" ++ clean = " ".join(str(text or "").split()) ++ if len(clean) <= limit: ++ return clean ++ return clean[: max(0, limit - 1)].rstrip() + "…" ++ ++ async def _esp32_ack_wav(self, *, audio_dir: Path, device_id: str) -> Path: ++ """Return a reusable quiet ACK WAV; the visual screen animation is the acknowledgement.""" ++ ack_path = audio_dir / "quick_ack.wav" ++ if ack_path.exists(): ++ return ack_path ++ ++ # ACK-mode clients expect a valid audio/wav response immediately, but ++ # Reyna ESP32 screens now show the acknowledgement visually. Keep the ++ # HTTP contract while avoiding the old spoken "Got it" message. ++ import wave ++ sample_rate = 16000 ++ duration_s = 0.08 ++ frames = b"\x00\x00" * int(sample_rate * duration_s) ++ with wave.open(str(ack_path), "wb") as wav: ++ wav.setnchannels(1) ++ wav.setsampwidth(2) ++ wav.setframerate(sample_rate) ++ wav.writeframes(frames) ++ return ack_path ++ ++ def _append_esp32_voice_audit(self, audio_dir: Path, event: Dict[str, Any]) -> None: ++ """Append a best-effort ESP32 voice request audit event.""" ++ try: ++ log_path = audio_dir / "voice_requests.jsonl" ++ with log_path.open("a", encoding="utf-8") as handle: ++ handle.write(json.dumps(event, default=str) + "\n") ++ except Exception: ++ logger.debug("[api_server] failed to append ESP32 voice audit log", exc_info=True) ++ ++ async def _run_esp32_voice_audio_background( ++ self, ++ *, ++ audio_path: Path, ++ audit_event: Dict[str, Any], ++ device_id: str, ++ instructions: Optional[str], ++ audio_dir: Path, ++ screen_url: Optional[str], ++ ) -> None: ++ """Transcribe uploaded ESP32 audio and continue the voice turn after an immediate ACK.""" ++ try: ++ await self._send_screen_animation(screen_url, "captioning") ++ from tools.transcription_tools import transcribe_audio ++ stt_result = await asyncio.to_thread(transcribe_audio, str(audio_path)) ++ if not stt_result.get("success"): ++ failed_event = dict(audit_event) ++ failed_event.update({ ++ "status": "transcription_failed", ++ "error": stt_result.get("error") or "Transcription failed", ++ "timestamp": time.time(), ++ }) ++ self._append_esp32_voice_audit(audio_dir, failed_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but transcription failed. Try again.", ++ clear=True, ++ ) ++ return ++ transcript = str(stt_result.get("transcript") or "").strip() ++ if not transcript: ++ empty_event = dict(audit_event) ++ empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) ++ self._append_esp32_voice_audit(audio_dir, empty_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but got no words. Try again.", ++ clear=True, ++ ) ++ return ++ transcript_event = dict(audit_event) ++ transcript_event.update({"status": "transcribed", "transcript": transcript, "timestamp": time.time()}) ++ self._append_esp32_voice_audit(audio_dir, transcript_event) ++ await self._run_esp32_voice_background( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=instructions, ++ audio_dir=audio_dir, ++ screen_url=screen_url, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 ACK background voice processing failed") ++ failed_event = dict(audit_event) ++ failed_event.update({"status": "background_exception", "error": str(exc), "timestamp": time.time()}) ++ self._append_esp32_voice_audit(audio_dir, failed_event) ++ await self._send_screen_text( ++ screen_url, ++ "I got the audio, but background processing failed.", ++ clear=True, ++ ) ++ ++ async def _run_esp32_voice_background( ++ self, ++ *, ++ transcript: str, ++ device_id: str, ++ instructions: Optional[str], ++ audio_dir: Path, ++ screen_url: Optional[str], ++ ) -> None: ++ """Run the slow ESP32 voice turn after the device already heard an ACK.""" ++ try: ++ animation_ok = await self._send_screen_animation(screen_url, "waiting", text=transcript) ++ if not animation_ok: ++ await self._send_screen_text( ++ screen_url, ++ f"Heard: {self._short_voice_summary(transcript, 180)}\n\nWorking…", ++ clear=True, ++ ) ++ screen_hint = ( ++ "\n\nThis request came from an ESP32 voice/screen device. " ++ "Keep the spoken follow-up short. The gateway will display the full answer on the initiating screen when possible." ++ ) ++ merged_instructions = (instructions or "") + screen_hint ++ response_data, final_text = await self._run_esp32_voice_turn( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=merged_instructions, ++ ) ++ if not final_text.strip(): ++ final_text = self._extract_response_text(response_data) or "I finished, but I do not have a text answer." ++ display_text = final_text ++ if len(display_text) > 1200: ++ display_text = self._short_voice_summary(display_text, 900) + "\n\nFull answer is long; WhatsApp delivery is the next wiring step." ++ await self._send_screen_text(screen_url, display_text, clear=True) ++ await self._send_screen_image_response(screen_url, final_text) ++ await self._send_screen_audio_response( ++ screen_url, ++ final_text, ++ audio_dir=audio_dir, ++ device_id=device_id, ++ ) ++ except Exception: ++ logger.exception("[api_server] ESP32 background voice turn failed") ++ await self._send_screen_text(screen_url, "Hermes hit an error while preparing the full answer.", clear=True) ++ ++ @staticmethod ++ def _extract_response_text(response_data: Dict[str, Any]) -> str: ++ """Extract assistant text from an OpenAI Responses-shaped payload.""" ++ chunks: List[str] = [] ++ for item in response_data.get("output") or []: ++ if not isinstance(item, dict) or item.get("type") != "message": ++ continue ++ for part in item.get("content") or []: ++ if isinstance(part, dict) and part.get("type") in {"output_text", "text"}: ++ text = part.get("text") ++ if isinstance(text, str) and text: ++ chunks.append(text) ++ return "\n".join(chunks).strip() ++ ++ async def _run_esp32_voice_turn( ++ self, ++ *, ++ transcript: str, ++ device_id: str, ++ instructions: Optional[str] = None, ++ ) -> tuple[Dict[str, Any], str]: ++ """Run one persistent Responses-style turn for an ESP32 voice device.""" ++ conversation = f"esp32:{device_id}" ++ previous_response_id = self._response_store.get_conversation(conversation) ++ conversation_history: List[Dict[str, Any]] = [] ++ stored_session_id = None ++ stored_instructions = None ++ ++ if previous_response_id: ++ stored = self._response_store.get(previous_response_id) ++ if stored is not None: ++ conversation_history = list(stored.get("conversation_history", [])) ++ stored_session_id = stored.get("session_id") ++ stored_instructions = stored.get("instructions") ++ ++ if instructions is None: ++ instructions = stored_instructions ++ ++ session_id = stored_session_id or str(uuid.uuid4()) ++ user_message = ( ++ f'[Voice input from ESP32 device "{device_id}"]\n' ++ f'{transcript}' ++ ) ++ ++ result, usage = await self._run_agent( ++ user_message=user_message, ++ conversation_history=conversation_history, ++ ephemeral_system_prompt=instructions, ++ session_id=session_id, ++ gateway_session_key=conversation, ++ ) ++ ++ final_response = result.get("final_response", "") or result.get("error", "") or "(No response generated)" ++ response_id = f"resp_{uuid.uuid4().hex[:28]}" ++ created_at = int(time.time()) ++ full_history = self._build_response_conversation_history( ++ conversation_history, ++ user_message, ++ result, ++ final_response, ++ ) ++ output_start_index = self._response_messages_turn_start_index( ++ conversation_history, ++ user_message, ++ result, ++ ) ++ output_items = self._extract_output_items(result, start_index=output_start_index) ++ response_data = { ++ "id": response_id, ++ "object": "response", ++ "status": "completed", ++ "created_at": created_at, ++ "model": self._model_name, ++ "output": output_items, ++ "usage": { ++ "input_tokens": usage.get("input_tokens", 0), ++ "output_tokens": usage.get("output_tokens", 0), ++ "total_tokens": usage.get("total_tokens", 0), ++ }, ++ } ++ self._response_store.put(response_id, { ++ "response": response_data, ++ "conversation_history": full_history, ++ "instructions": instructions, ++ "session_id": session_id, ++ }) ++ self._response_store.set_conversation(conversation, response_id) ++ return response_data, final_response ++ ++ @staticmethod ++ def _esp32_audio_cache_dir() -> Path: ++ """Return the cache directory for ESP32 voice artifacts.""" ++ try: ++ from hermes_constants import get_hermes_dir ++ return get_hermes_dir("cache/audio", "audio_cache") / "esp32" ++ except Exception: ++ return Path.home() / ".hermes" / "cache" / "audio" / "esp32" ++ ++ @staticmethod ++ def _write_pcm_s16le_wav(raw_path: Path, wav_path: Path, *, sample_rate: int, channels: int = 1) -> None: ++ """Wrap a raw PCM s16le file in a minimal RIFF/WAVE container.""" ++ import wave ++ ++ with wave.open(str(wav_path), "wb") as wav: ++ wav.setnchannels(channels) ++ wav.setsampwidth(2) ++ wav.setframerate(sample_rate) ++ with raw_path.open("rb") as raw_handle: ++ while True: ++ chunk = raw_handle.read(64 * 1024) ++ if not chunk: ++ break ++ wav.writeframes(chunk) ++ ++ async def _handle_esp32_voice_ws(self, request: "web.Request") -> "web.WebSocketResponse": ++ """GET /api/esp32/voice/ws — stream ESP32 PCM over WebSocket, return TTS audio chunks.""" ++ auth_err = self._check_auth(request) ++ if auth_err: ++ return auth_err # type: ignore[return-value] ++ ++ ws = web.WebSocketResponse(heartbeat=None, max_msg_size=512 * 1024) ++ await ws.prepare(request) ++ ++ async def _send_event(event: str, **payload: Any) -> None: ++ if not ws.closed: ++ await ws.send_str(json.dumps({"event": event, **payload})) ++ ++ device_id = ( ++ request.headers.get("X-Device-ID") ++ or request.query.get("device_id") ++ or "default" ++ ).strip()[:80] or "default" ++ safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" ++ audio_dir = self._esp32_audio_cache_dir() ++ audio_dir.mkdir(parents=True, exist_ok=True) ++ stamp = f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}" ++ raw_path = audio_dir / f"{stamp}.s16le" ++ wav_path = audio_dir / f"{stamp}.wav" ++ sample_rate = 16000 ++ channels = 1 ++ sample_width = 2 ++ total_audio_bytes = 0 ++ started = False ++ stopped = False ++ first_audio_at: Optional[float] = None ++ started_at = time.time() ++ audit_ctx = self._request_audit_context(request) ++ audit_event: Dict[str, Any] = { ++ "timestamp": started_at, ++ "device_id": device_id, ++ "source_ip": audit_ctx.get("remote") or audit_ctx.get("peer_ip") or "", ++ "peer_ip": audit_ctx.get("peer_ip") or "", ++ "forwarded_for": audit_ctx.get("forwarded_for") or "", ++ "real_ip": audit_ctx.get("real_ip") or "", ++ "user_agent": audit_ctx.get("user_agent") or "", ++ "transport": "websocket", ++ "raw_path": str(raw_path), ++ "audio_path": str(wav_path), ++ "status": "accepted", ++ } ++ self._append_esp32_voice_audit(audio_dir, audit_event) ++ screen_url = self._resolve_esp32_screen_url(request, device_id) ++ visual_listening_sent = False ++ ++ def _schedule_voice_state(state: str, *, text: str = "", frames: Optional[int] = None) -> None: ++ if not screen_url: ++ return ++ task = asyncio.create_task(self._send_screen_animation(screen_url, state, text=text, frames=frames)) ++ try: ++ self._background_tasks.add(task) ++ except TypeError: ++ pass ++ if hasattr(task, "add_done_callback"): ++ task.add_done_callback(self._background_tasks.discard) ++ ++ await _send_event("ready", device_id=device_id, screen_url=screen_url or "") ++ ++ try: ++ raw_handle = raw_path.open("wb") ++ try: ++ async for msg in ws: ++ if msg.type == web.WSMsgType.TEXT: ++ try: ++ payload = json.loads(msg.data) ++ except json.JSONDecodeError: ++ await _send_event("error", code="bad_json", message="Text frames must be JSON") ++ continue ++ event = str(payload.get("event") or payload.get("type") or "").strip().lower() ++ if event == "start": ++ if started: ++ await _send_event("error", code="already_started", message="Audio stream already started") ++ continue ++ fmt = str(payload.get("format") or "pcm_s16le").strip().lower() ++ channels = int(payload.get("channels") or 1) ++ sample_width = int(payload.get("sample_width") or 2) ++ sample_rate = int(payload.get("sample_rate") or 16000) ++ device_id = str(payload.get("device_id") or device_id).strip()[:80] or device_id ++ if fmt not in {"pcm_s16le", "s16le", "pcm"} or channels != 1 or sample_width != 2: ++ await _send_event( ++ "error", ++ code="unsupported_audio_format", ++ message="Use 16 kHz mono signed 16-bit little-endian PCM", ++ ) ++ await ws.close() ++ return ws ++ if sample_rate < 8000 or sample_rate > 48000: ++ await _send_event("error", code="bad_sample_rate", message="sample_rate must be 8000-48000") ++ await ws.close() ++ return ws ++ started = True ++ if not visual_listening_sent: ++ _schedule_voice_state("captioning", frames=8) ++ visual_listening_sent = True ++ await _send_event("listening", sample_rate=sample_rate, channels=channels, sample_width=sample_width) ++ elif event in {"stop", "end"}: ++ stopped = True ++ break ++ elif event in {"ping", "keepalive"}: ++ await _send_event("pong") ++ else: ++ await _send_event("error", code="unknown_event", message=f"Unknown event: {event or ''}") ++ elif msg.type == web.WSMsgType.BINARY: ++ if not started: ++ started = True ++ if not visual_listening_sent: ++ _schedule_voice_state("captioning", frames=8) ++ visual_listening_sent = True ++ await _send_event("listening", sample_rate=sample_rate, channels=channels, sample_width=sample_width) ++ chunk = bytes(msg.data or b"") ++ if not chunk: ++ continue ++ if first_audio_at is None: ++ first_audio_at = time.time() ++ total_audio_bytes += len(chunk) ++ if total_audio_bytes > ESP32_WS_MAX_AUDIO_BYTES: ++ await _send_event( ++ "error", ++ code="audio_too_large", ++ message=f"Audio stream exceeded {ESP32_WS_MAX_AUDIO_BYTES} bytes", ++ ) ++ await ws.close() ++ return ws ++ raw_handle.write(chunk) ++ elif msg.type == web.WSMsgType.ERROR: ++ logger.warning("[api_server] ESP32 voice websocket error: %s", ws.exception()) ++ break ++ finally: ++ raw_handle.close() ++ ++ if not stopped: ++ await _send_event("error", code="missing_stop", message="Send a stop event after audio chunks") ++ await ws.close() ++ return ws ++ if total_audio_bytes < ESP32_WS_MIN_AUDIO_BYTES: ++ await _send_event("error", code="audio_too_short", message="Not enough audio received") ++ await ws.close() ++ return ws ++ ++ await asyncio.to_thread( ++ self._write_pcm_s16le_wav, ++ raw_path, ++ wav_path, ++ sample_rate=sample_rate, ++ channels=channels, ++ ) ++ audit_event.update({ ++ "status": "received", ++ "audio_bytes": total_audio_bytes, ++ "sample_rate": sample_rate, ++ "first_audio_latency_ms": int(((first_audio_at or started_at) - started_at) * 1000), ++ "stop_latency_ms": int((time.time() - started_at) * 1000), ++ }) ++ self._append_esp32_voice_audit(audio_dir, dict(audit_event)) ++ ++ _schedule_voice_state("captioning", frames=10) ++ await _send_event("thinking", stage="transcribing") ++ from tools.transcription_tools import transcribe_audio ++ stt_result = await asyncio.to_thread(transcribe_audio, str(wav_path)) ++ if not stt_result.get("success"): ++ message = stt_result.get("error") or "Transcription failed" ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "transcription_failed", "error": message, "timestamp": time.time()}) ++ await _send_event("error", code="transcription_failed", message=message) ++ await ws.close() ++ return ws ++ transcript = str(stt_result.get("transcript") or "").strip() ++ if not transcript: ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "empty_transcript", "timestamp": time.time()}) ++ await _send_event("error", code="empty_transcript", message="Transcription was empty") ++ await ws.close() ++ return ws ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "transcribed", "transcript": transcript, "timestamp": time.time()}) ++ await _send_event("transcript", text=transcript) ++ ++ ack_text = f"Heard: {self._short_voice_summary(transcript, 180)}\n\nWorking on the full answer now." ++ await _send_event("response_text", text=ack_text, response_id="", final=False) ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "queued_background", "transcript": transcript, "timestamp": time.time()}) ++ ++ # Close the client WebSocket before making any agent/tool/TTS/MCP calls. ++ # Some Tactility boards cannot service their HTTP MCP endpoint while the ++ # voice WebSocket is still open and waiting for the final answer. The ++ # slow work continues below as an owned background task and sends text, ++ # audio, and future image output via the board's HTTP MCP endpoint. ++ try: ++ await _send_event("done", mode="background", screen_url=screen_url or "") ++ await ws.close() ++ except Exception: ++ pass ++ background_task = asyncio.create_task( ++ self._run_esp32_voice_background( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=request.headers.get("X-Hermes-Instructions") or None, ++ audio_dir=audio_dir, ++ screen_url=screen_url, ++ ) ++ ) ++ try: ++ self._background_tasks.add(background_task) ++ except TypeError: ++ pass ++ if hasattr(background_task, "add_done_callback"): ++ background_task.add_done_callback(self._background_tasks.discard) ++ return ws ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 voice websocket processing failed") ++ self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "websocket_exception", "error": str(exc), "timestamp": time.time()}) ++ try: ++ await _send_event("error", code="internal_error", message=str(exc)) ++ await ws.close() ++ except Exception: ++ pass ++ return ws ++ finally: ++ try: ++ raw_path.unlink(missing_ok=True) ++ except Exception: ++ pass ++ ++ async def _handle_esp32_voice(self, request: "web.Request") -> "web.Response": ++ """POST /api/esp32/voice — receive audio, run Hermes, return TTS audio.""" ++ auth_err = self._check_auth(request) ++ if auth_err: ++ return auth_err ++ ++ device_id = ( ++ request.headers.get("X-Device-ID") ++ or request.query.get("device_id") ++ or "default" ++ ).strip()[:80] or "default" ++ ++ try: ++ audio_bytes = await request.read() ++ except Exception as exc: ++ return web.json_response(_openai_error(f"Failed to read audio body: {exc}"), status=400) ++ ++ if not audio_bytes: ++ return web.json_response(_openai_error("Empty audio body", code="empty_audio"), status=400) ++ if len(audio_bytes) > ESP32_AUDIO_MAX_BYTES: ++ return web.json_response( ++ _openai_error( ++ f"Audio body is too large ({len(audio_bytes)} bytes); max is {ESP32_AUDIO_MAX_BYTES}", ++ code="audio_too_large", ++ ), ++ status=413, ++ ) ++ ++ suffix = self._audio_extension_for_content_type(request.headers.get("Content-Type", "")) ++ try: ++ from hermes_constants import get_hermes_dir ++ audio_dir = get_hermes_dir("cache/audio", "audio_cache") / "esp32" ++ except Exception: ++ audio_dir = Path.home() / ".hermes" / "cache" / "audio" / "esp32" ++ audio_dir.mkdir(parents=True, exist_ok=True) ++ safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" ++ audio_path = audio_dir / f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}{suffix}" ++ audio_path.write_bytes(audio_bytes) ++ ++ audit_ctx = self._request_audit_context(request) ++ audit_event: Dict[str, Any] = { ++ "timestamp": time.time(), ++ "device_id": device_id, ++ "source_ip": audit_ctx.get("remote") or audit_ctx.get("peer_ip") or "", ++ "peer_ip": audit_ctx.get("peer_ip") or "", ++ "forwarded_for": audit_ctx.get("forwarded_for") or "", ++ "real_ip": audit_ctx.get("real_ip") or "", ++ "user_agent": audit_ctx.get("user_agent") or "", ++ "reply_mode": self._esp32_reply_mode(request), ++ "screen_device": request.headers.get("X-Hermes-Screen-Device") or request.query.get("screen_device") or "", ++ "screen_url": request.headers.get("X-Hermes-Screen-Url") or request.headers.get("X-Screen-MCP-Url") or request.query.get("screen_url") or "", ++ "content_type": request.headers.get("Content-Type", ""), ++ "audio_bytes": len(audio_bytes), ++ "audio_path": str(audio_path), ++ "status": "received", ++ } ++ ++ def _append_audit(event: Dict[str, Any]) -> None: ++ try: ++ log_path = audio_dir / "voice_requests.jsonl" ++ with log_path.open("a", encoding="utf-8") as handle: ++ handle.write(json.dumps(event, default=str) + "\n") ++ except Exception: ++ logger.debug("[api_server] failed to append ESP32 voice audit log", exc_info=True) ++ ++ _append_audit(audit_event) ++ ++ if self._esp32_reply_mode(request) == "ack": ++ screen_url = self._resolve_esp32_screen_url(request, device_id) ++ ++ async def _ack_background() -> None: ++ try: ++ await self._send_screen_animation(screen_url, "ack") ++ await self._send_screen_animation(screen_url, "captioning") ++ from tools.transcription_tools import transcribe_audio ++ stt_result_bg = await asyncio.wait_for( ++ asyncio.to_thread(transcribe_audio, str(audio_path)), ++ timeout=45, ++ ) ++ if not stt_result_bg.get("success"): ++ failed_event = dict(audit_event) ++ failed_event.update({ ++ "status": "transcription_failed", ++ "error": stt_result_bg.get("error") or "Transcription failed", ++ "timestamp": time.time(), ++ }) ++ _append_audit(failed_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but transcription failed. Try again.", ++ clear=True, ++ ) ++ return ++ transcript_bg = str(stt_result_bg.get("transcript") or "").strip() ++ if not transcript_bg: ++ empty_event = dict(audit_event) ++ empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) ++ _append_audit(empty_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but got no words. Try again.", ++ clear=True, ++ ) ++ return ++ transcript_event = dict(audit_event) ++ transcript_event.update({"status": "transcribed", "transcript": transcript_bg, "timestamp": time.time()}) ++ _append_audit(transcript_event) ++ await self._run_esp32_voice_background( ++ transcript=transcript_bg, ++ device_id=device_id, ++ instructions=request.headers.get("X-Hermes-Instructions") or None, ++ audio_dir=audio_dir, ++ screen_url=screen_url, ++ ) ++ except asyncio.TimeoutError: ++ timeout_event = dict(audit_event) ++ timeout_event.update({"status": "transcription_timeout", "error": "Transcription exceeded 45s", "timestamp": time.time()}) ++ _append_audit(timeout_event) ++ await self._send_screen_text( ++ screen_url, ++ "I heard audio, but captioning timed out. Try again.", ++ clear=True, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 ack-mode background processing failed") ++ failed_event = dict(audit_event) ++ failed_event.update({"status": "background_exception", "error": str(exc), "timestamp": time.time()}) ++ _append_audit(failed_event) ++ ++ task = asyncio.create_task(_ack_background()) ++ try: ++ self._background_tasks.add(task) ++ except TypeError: ++ pass ++ if hasattr(task, "add_done_callback"): ++ task.add_done_callback(self._background_tasks.discard) ++ ack_path = await self._esp32_ack_wav(audio_dir=audio_dir, device_id=device_id) ++ headers = { ++ "X-Hermes-Conversation": f"esp32:{device_id}", ++ "X-Hermes-Reply-Mode": "ack", ++ "X-Hermes-Screen-Url": screen_url or "", ++ "Content-Type": "audio/wav", ++ } ++ return web.FileResponse(ack_path, headers=headers) ++ ++ try: ++ from tools.transcription_tools import transcribe_audio ++ stt_result = await asyncio.to_thread(transcribe_audio, str(audio_path)) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 transcription failed") ++ failed_event = dict(audit_event) ++ failed_event.update({"status": "transcription_exception", "error": str(exc), "timestamp": time.time()}) ++ _append_audit(failed_event) ++ return web.json_response(_openai_error(f"Transcription failed: {exc}"), status=500) ++ ++ if not stt_result.get("success"): ++ failed_event = dict(audit_event) ++ failed_event.update({ ++ "status": "transcription_failed", ++ "error": stt_result.get("error") or "Transcription failed", ++ "timestamp": time.time(), ++ }) ++ _append_audit(failed_event) ++ return web.json_response( ++ _openai_error(stt_result.get("error") or "Transcription failed", code="transcription_failed"), ++ status=502, ++ ) ++ ++ transcript = str(stt_result.get("transcript") or "").strip() ++ if not transcript: ++ empty_event = dict(audit_event) ++ empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) ++ _append_audit(empty_event) ++ return web.json_response(_openai_error("Transcription was empty", code="empty_transcript"), status=422) ++ ++ transcript_event = dict(audit_event) ++ transcript_event.update({"status": "transcribed", "transcript": transcript, "timestamp": time.time()}) ++ _append_audit(transcript_event) ++ ++ instructions = request.headers.get("X-Hermes-Instructions") or None ++ try: ++ response_data, final_text = await self._run_esp32_voice_turn( ++ transcript=transcript, ++ device_id=device_id, ++ instructions=instructions, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 agent turn failed") ++ return web.json_response(_openai_error(f"Agent turn failed: {exc}"), status=500) ++ ++ if not final_text.strip(): ++ final_text = self._extract_response_text(response_data) or "I heard you, but I do not have a spoken answer." ++ ++ try: ++ from tools.tts_tool import text_to_speech_tool ++ tts_raw = await asyncio.to_thread(text_to_speech_tool, final_text) ++ tts_result = json.loads(tts_raw) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 TTS failed") ++ return web.json_response( ++ _openai_error(f"TTS failed after response was generated: {exc}", code="tts_failed"), ++ status=500, ++ headers={ ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ }, ++ ) ++ ++ if not tts_result.get("success"): ++ return web.json_response( ++ _openai_error(tts_result.get("error") or "TTS failed", code="tts_failed"), ++ status=502, ++ headers={ ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ }, ++ ) ++ ++ tts_path = str(tts_result.get("file_path") or "") ++ if not tts_path or not Path(tts_path).exists(): ++ return web.json_response( ++ _openai_error("TTS succeeded but no audio file was produced", code="tts_file_missing"), ++ status=500, ++ ) ++ ++ try: ++ response_audio_path = self._convert_audio_to_esp32_wav( ++ tts_path, ++ output_dir=audio_dir, ++ device_id=device_id, ++ ) ++ except Exception as exc: ++ logger.exception("[api_server] ESP32 WAV normalization failed") ++ return web.json_response( ++ _openai_error(f"Could not convert TTS audio to ESP32 WAV: {exc}", code="wav_conversion_failed"), ++ status=500, ++ headers={ ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ }, ++ ) ++ ++ headers = { ++ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), ++ "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), ++ "X-Hermes-Response-Id": str(response_data.get("id") or ""), ++ "X-Hermes-Conversation": f"esp32:{device_id}", ++ "X-Hermes-TTS-Source-Type": self._audio_response_mime(tts_path), ++ "Content-Type": "audio/wav", ++ } ++ return web.FileResponse( ++ response_audio_path, ++ headers=headers, ++ ) ++ ++ # ------------------------------------------------------------------ ++ # Cron jobs API ++ # ------------------------------------------------------------------ ++ ++ _JOB_ID_RE = __import__("re").compile(r"[a-f0-9]{12}") ++ # Allowed fields for update — prevents clients injecting arbitrary keys ++ _UPDATE_ALLOWED_FIELDS = {"name", "schedule", "prompt", "deliver", "skills", "skill", "repeat", "enabled"} ++ _MAX_NAME_LENGTH = 200 ++ _MAX_PROMPT_LENGTH = 5000 ++ ++ @staticmethod ++ ++ + async def _handle_models(self, request: "web.Request") -> "web.Response": + """GET /v1/models — list hermes-agent and any configured model_routes aliases.""" + auth_err = self._check_auth(request) +@@ -5018,6 +6359,9 @@ class APIServerAdapter(BasePlatformAdapter): + # bootstrap shims use this key as a feature-detection hook; registering + # native routes first lets those shims no-op instead of shadowing the + # upstream session-control handlers. ++ # Local ESP32 push-to-talk endpoints: POST complete audio, or stream PCM over WebSocket. ++ self._app.router.add_post("/api/esp32/voice", self._handle_esp32_voice) ++ self._app.router.add_get("/api/esp32/voice/ws", self._handle_esp32_voice_ws) + self._app["api_server_adapter"] = self + + # Start background sweep to clean up orphaned (unconsumed) run streams diff --git a/scripts/restore_after_update.py b/scripts/restore_after_update.py new file mode 100644 index 0000000..fc3b10c --- /dev/null +++ b/scripts/restore_after_update.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +""" +Restore ESP32 voice gateway after `hermes update`. + +Usage: + ./scripts/restore_after_update.py + python3 scripts/restore_after_update.py --verify + python3 scripts/restore_after_update.py --force + +What it does: +- Checks if gateway/platforms/api_server.py has ESP32 routes +- If missing, applies patch/api_server_esp32_live_draft_with_llm.patch via git apply +- Verifies routes return 400 not 404 (400 means route exists but missing WS upgrade / empty body) +- Copies patch to ~/.hermes/ fallback location +- Prints next steps (gateway restart) + +Env: + HERMES_AGENT_DIR - override hermes-agent checkout path +""" + +import argparse +import subprocess +import sys +from pathlib import Path + +def find_agent_dir() -> Path | None: + import os + candidates = [] + env_dir = os.getenv("HERMES_AGENT_DIR") + if env_dir: + candidates.append(Path(env_dir)) + candidates.append(Path.home() / ".hermes" / "hermes-agent") + alt_path = Path(__file__).resolve().parents[2] / ".hermes" / "hermes-agent" + if alt_path.exists(): + candidates.append(alt_path) + try: + import hermes_cli # type: ignore + candidates.append(Path(hermes_cli.__file__).parents[1]) + except Exception: + pass + for c in candidates: + if c and (c / "gateway" / "platforms" / "api_server.py").exists(): + return c + return None + +def has_routes(api_file: Path) -> bool: + try: + txt = api_file.read_text() + return "ESP32_AUDIO_MAX_BYTES" in txt[:20000] and "/api/esp32/voice" in txt and "_handle_esp32_voice_ws" in txt + except Exception: + return False + +def find_patch(repo_root: Path) -> Path | None: + for p in [ + repo_root / "patch" / "api_server_esp32_live_draft_with_llm.patch", + repo_root / "hermes_plugin" / "patch" / "api_server_esp32_live_draft_with_llm.patch", + Path.home() / ".hermes" / "esp32_live_draft_with_llm_patch.patch", + Path.home() / ".hermes" / "esp32_voice_patch.patch", + ]: + if p.exists() and p.stat().st_size > 1000: + return p + # any in patch/ + for f in (repo_root / "patch").glob("*.patch"): + if f.stat().st_size > 1000: + return f + return None + +def apply_patch(patch_path: Path, agent_dir: Path, force: bool=False) -> bool: + api_file = agent_dir / "gateway" / "platforms" / "api_server.py" + print(f"[restore] patch={patch_path} agent_dir={agent_dir} api_file={api_file} force={force}") + if not (agent_dir / ".git").exists(): + print(f"[restore] {agent_dir} is not a git repo") + return False + if has_routes(api_file) and not force: + print("[restore] routes already present, skipping (use --force to reapply)") + return True + # Try apply + if force: + # reset file to HEAD first? we do checkout + try: + subprocess.run(["git", "checkout", "HEAD", "--", "gateway/platforms/api_server.py"], cwd=str(agent_dir), timeout=15) + except Exception: + pass + # Check + chk = subprocess.run(["git", "apply", "--check", str(patch_path)], cwd=str(agent_dir), capture_output=True, text=True, timeout=15) + if chk.returncode != 0: + print(f"[restore] git apply --check output:\n{chk.stdout[:1000]}\n{chk.stderr[:1000]}") + # Try 3way if not forced yet + if not force: + print("[restore] trying git apply --3way") + r = subprocess.run(["git", "apply", "--3way", str(patch_path)], cwd=str(agent_dir), capture_output=True, text=True, timeout=30) + print(r.stdout[-500:] if r.stdout else "", r.stderr[-1000:] if r.stderr else "") + if r.returncode == 0: + print("[restore] 3way apply ok") + return has_routes(api_file) + else: + r = subprocess.run(["git", "apply", str(patch_path)], cwd=str(agent_dir), capture_output=True, text=True, timeout=30) + print(r.stdout[-500:] if r.stdout else "", r.stderr[-1000:] if r.stderr else "") + return r.returncode == 0 and has_routes(api_file) + else: + r = subprocess.run(["git", "apply", str(patch_path)], cwd=str(agent_dir), capture_output=True, text=True, timeout=30) + if r.returncode != 0: + print(f"[restore] apply failed: {r.stdout} {r.stderr}") + return False + print(f"[restore] applied OK") + return has_routes(api_file) + +def verify_curly_brace_imports(agent_dir: Path) -> bool: + """Guard against missing top imports ipaddress/urlparse (bug that caused 500 loop 2026-07-14)""" + api_file = agent_dir / "gateway" / "platforms" / "api_server.py" + txt = api_file.read_text() + has_ip = "import ipaddress" in txt + has_url = "from urllib.parse import urlparse" in txt + print(f"[verify] top imports: ipaddress={has_ip} urlparse={has_url}") + if not has_ip or not has_url: + print("[verify] MISSING critical imports – this will cause 500 import loop (see skill pitfalls). Fix by ensuring patch includes them.") + return False + return True + +def compile_check(agent_dir: Path) -> bool: + api_file = agent_dir / "gateway" / "platforms" / "api_server.py" + r = subprocess.run([sys.executable, "-m", "py_compile", str(api_file)], capture_output=True, text=True, timeout=15) + if r.returncode != 0: + print(f"[compile] failed: {r.stderr}") + return False + print("[compile] OK") + return True + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--force", action="store_true", help="Force reapply even if routes present") + parser.add_argument("--verify", action="store_true", help="Only verify, don't apply") + args = parser.parse_args() + + repo_root = Path(__file__).resolve().parents[1] + agent_dir = find_agent_dir() + if not agent_dir: + print("Could not find hermes-agent dir (checked ~/.hermes/hermes-agent, HERMES_AGENT_DIR, pip location)") + sys.exit(2) + api_file = agent_dir / "gateway" / "platforms" / "api_server.py" + print(f"Agent dir: {agent_dir}") + print(f"API file: {api_file}") + print(f"Has routes: {has_routes(api_file)}") + + if args.verify: + ok = verify_curly_brace_imports(agent_dir) and compile_check(agent_dir) and has_routes(api_file) + sys.exit(0 if ok else 1) + + patch = find_patch(repo_root) + if not patch: + print(f"No patch found. Looked in {repo_root}/patch/ and ~/.hermes/*.patch") + sys.exit(2) + print(f"Using patch: {patch} ({patch.stat().st_size} bytes)") + + ok = apply_patch(patch, agent_dir, force=args.force) + if not ok: + print("[restore] FAILED to restore") + sys.exit(1) + + verify_curly_brace_imports(agent_dir) + compile_check(agent_dir) + + # Copy to ~/.hermes fallback + try: + dest = Path.home() / ".hermes" / "esp32_live_draft_with_llm_patch.patch" + if patch.resolve() != dest.resolve(): + import shutil + shutil.copy2(patch, dest) + print(f"[restore] copied patch to {dest} for fallback") + except Exception as e: + print(f"[restore] failed to copy fallback: {e}") + + print(""" +[restore] DONE. Next: + 1. hermes gateway restart (from a DIFFERENT shell, not from inside Hermes chat - see skill pitfalls) + Use: systemctl --user restart hermes-gateway hermes-gateway-kids --no-pager || hermes gateway restart + 2. If stuck in deactivating: + pkill -9 -f "hermes_cli.main gateway run"; systemctl --user reset-failed hermes-gateway*; systemctl --user start hermes-gateway + 3. Verify routes (should be 400 not 404): + API_KEY=$(grep -E API_SERVER_KEY .env -h ~/.hermes/.env ~/.hermes/profiles/*/ .env 2>/dev/null | head -1 | cut -d= -f2 | tr -d '\"'\\'') + curl -s -o /dev/null -w "%{http_code}\\n" -H "Authorization: Bearer $API_KEY" http://127.0.0.1:8642/api/esp32/voice/ws + curl -s -o /dev/null -w "%{http_code}\\n" -H "Authorization: Bearer $API_KEY" -X POST http://127.0.0.1:8642/api/esp32/voice --data-binary @/dev/null + # Both should print 400. 404 means routes still missing. +""") + +if __name__ == "__main__": + main() diff --git a/scripts/restore_after_update.sh b/scripts/restore_after_update.sh new file mode 100644 index 0000000..dab94ff --- /dev/null +++ b/scripts/restore_after_update.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +# One-liner called by cron/gateway after update to ensure routes present +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +python3 "$REPO_ROOT/scripts/restore_after_update.py" "$@"