# Original ESP32 extraction (for reference, not used at runtime) # From api_server.py ESP32_AUDIO_MAX_BYTES -> _handle_esp32_voice ESP32_AUDIO_MAX_BYTES = 8_000_000 # ~4 minutes of 16 kHz mono PCM WAV; enough for push-to-talk tests ESP32_WS_MAX_AUDIO_BYTES = 8_000_000 ESP32_WS_MIN_AUDIO_BYTES = 1_600 # 50 ms at 16 kHz mono s16le; rejects empty accidental taps ESP32_WS_TTS_CHUNK_BYTES = 16_384 # Live draft ingestion config (from whisper-translation/apple_speech engine) ESP32_LIVE_DRAFT_ENABLED = os.getenv("ESP32_LIVE_DRAFT_ENABLED", "1").lower() not in {"0", "false", "no", "off"} ESP32_LIVE_DRAFT_INTERVAL_MS = int(os.getenv("ESP32_LIVE_DRAFT_INTERVAL_MS", "800")) # how often to snapshot for draft ESP32_LIVE_DRAFT_MIN_BYTES = int(os.getenv("ESP32_LIVE_DRAFT_MIN_BYTES", "8000")) # min new bytes since last draft ESP32_LIVE_DRAFT_MIN_TOTAL_MS = int(os.getenv("ESP32_LIVE_DRAFT_MIN_TOTAL_MS", "500")) # don't draft until we have 500ms ESP32_MACMINI_MCP_URL = os.getenv("MACMINI_MCP_URL") or None # overridable, else from config.yaml mcp_servers.macmini ESP32_MACMINI_MCP_TOKEN = os.getenv("MACMINI_MCP_TOKEN") or None ESP32_LIVE_DRAFT_USE_MACMINI = os.getenv("ESP32_LIVE_DRAFT_USE_MACMINI", "1").lower() not in {"0", "false", "no", "off"} _ESP32_AUDIO_EXTENSIONS = { "audio/wav": ".wav", "audio/wave": ".wav", "audio/x-wav": ".wav", "audio/mpeg": ".mp3", "audio/mp3": ".mp3", "audio/ogg": ".ogg", "audio/opus": ".ogg", "audio/webm": ".webm", "audio/mp4": ".m4a", "audio/x-m4a": ".m4a", "application/octet-stream": ".wav", } _AUDIO_RESPONSE_MIME_BY_SUFFIX = { ".mp3": "audio/mpeg", ".wav": "audio/wav", ".ogg": "audio/ogg", ".opus": "audio/ogg", ".m4a": "audio/mp4", ".mp4": "audio/mp4", ".flac": "audio/flac", } ESP32_VOICE_ANIMATION_PATH = "hermes_voice_animations.py" ESP32_VOICE_ANIMATION_SCRIPT = "# Hermes ESP32 voice-state animations for MicroPython screen boards.\n# Uploaded by the API server; can also be run manually with:\n# exec(open('hermes_voice_animations.py').read()); run_state('ack')\nimport time\ntry:\n import board_config\nexcept Exception:\n board_config = None\n\n\ndef _display():\n return getattr(board_config, 'display_instance', None) if board_config else None\n\n\ndef _sleep(ms):\n try:\n time.sleep_ms(ms)\n except AttributeError:\n time.sleep(ms / 1000)\n\n\ndef _text(d, msg, x, y, scale=1, color=1):\n if hasattr(d, 'text_large') and scale > 1:\n try:\n d.text_large(msg, x, y, scale=scale, color=color)\n return\n except TypeError:\n pass\n d.text(msg, x, y, color)\n\n\ndef _center_text(d, msg, y, scale=1, color=1):\n width = getattr(d, 'width', 320)\n char_w = 8 * scale\n x = max(0, (width - len(msg) * char_w) // 2)\n _text(d, msg, x, y, scale=scale, color=color)\n\n\ndef _wrap(text, width_chars=34, max_lines=4):\n words = str(text or '').replace('\\n', ' ').split()\n lines, line = [], ''\n for w in words:\n candidate = (line + ' ' + w).strip()\n if len(candidate) > width_chars and line:\n lines.append(line)\n line = w\n if len(lines) >= max_lines:\n break\n else:\n line = candidate\n if line and len(lines) < max_lines:\n lines.append(line)\n return lines\n\n\ndef _spinner(d, title, subtitle='', frames=24, delay=70):\n width, height = getattr(d, 'width', 320), getattr(d, 'height', 240)\n cx, cy = width // 2, height // 2 - 8\n spokes = [(-22,0),(-16,-16),(0,-22),(16,-16),(22,0),(16,16),(0,22),(-16,16)]\n for frame in range(frames):\n d.clear(0)\n _center_text(d, title, 18, scale=2)\n for i, (dx, dy) in enumerate(spokes):\n color = 1 if (i + frame) % len(spokes) in (0, 1, 2, 3) else 0\n size = 4 if i == frame % len(spokes) else 2\n d.fill_rect(cx + dx - size // 2, cy + dy - size // 2, size, size, color)\n d.rect(8, height - 34, width - 16, 18, 1)\n d.fill_rect(10, height - 32, ((width - 20) * ((frame % 16) + 1)) // 16, 14, 1)\n if subtitle:\n _center_text(d, subtitle[:32], height - 56, scale=1)\n d.show()\n _sleep(delay)\n\n\ndef ack(frames=18):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'GOT IT', 18, scale=2)\n # expanding check pulse\n box = 48 + (i % 6) * 8\n x, y = width // 2 - box // 2, height // 2 - box // 2\n d.rect(x, y, box, box, 1)\n d.line(width//2 - 24, height//2, width//2 - 6, height//2 + 18, 1)\n d.line(width//2 - 6, height//2 + 18, width//2 + 30, height//2 - 22, 1)\n _center_text(d, 'Listening captured', height - 36, scale=1)\n d.show()\n _sleep(55)\n\n\ndef captioning(frames=36):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'CAPTIONING', 16, scale=2)\n for bar in range(9):\n h = 8 + ((i * 5 + bar * 13) % 42)\n x = width // 2 - 72 + bar * 18\n d.fill_rect(x, height // 2 - h // 2, 10, h, 1)\n dots = '.' * ((i // 5) % 4)\n _center_text(d, 'Transcribing' + dots, height - 42, scale=1)\n d.show()\n _sleep(65)\n\n\ndef waiting(text='', frames=44):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'THINKING', 12, scale=2)\n r = 18 + (i % 10) * 2\n cx, cy = width // 2, height // 2 - 10\n d.rect(cx - r, cy - r, r * 2, r * 2, 1)\n d.rect(cx - r//2, cy - r//2, r, r, 1)\n if text:\n d.text('Heard:', 10, height - 68, 1)\n for idx, line in enumerate(_wrap(text, 36, 3)):\n d.text(line[:38], 10, height - 54 + idx * 12, 1)\n else:\n _center_text(d, 'Waiting for reply...', height - 42, scale=1)\n d.show()\n _sleep(75)\n\n\ndef run_state(state='ack', text='', frames=None):\n state = str(state or 'ack').lower()\n if state in ('ack', 'listening', 'got_it'):\n ack(frames or 18)\n elif state in ('caption', 'captioning', 'transcribing'):\n captioning(frames or 36)\n elif state in ('wait', 'waiting', 'thinking', 'reply'):\n waiting(text, frames or 44)\n else:\n _spinner(_display(), 'HERMES', state[:28], frames or 20)\n" def _coerce_port(value: Any, default: int = DEFAULT_PORT) -> int: """Parse a listen port without letting malformed env/config values crash startup.""" try: return int(value) except (TypeError, ValueError): return default _TRUE_REQUEST_BOOL_STRINGS = frozenset({"1", "true", "yes", "on"}) _FALSE_REQUEST_BOOL_STRINGS = frozenset({"0", "false", "no", "off"}) def _coerce_request_bool(value: Any, default: bool = False) -> bool: """Normalize boolean-like API payload values. External clients should send real JSON booleans, but some OpenAI-compatible frontends and middleware serialize flags like ``stream`` as strings. Using Python truthiness on those values misroutes requests because ``"false"`` is still truthy. Treat only explicit bool-ish scalars as booleans; everything else falls back to the caller's default. """ if isinstance(value, bool): return value if value is None: return default if isinstance(value, str): normalized = value.strip().lower() if normalized in _TRUE_REQUEST_BOOL_STRINGS: return True if normalized in _FALSE_REQUEST_BOOL_STRINGS: return False return default if isinstance(value, (int, float)): return bool(value) return default def _normalize_chat_content( content: Any, *, _max_depth: int = 10, _depth: int = 0, ) -> str: """Normalize OpenAI chat message content into a plain text string. Some clients (Open WebUI, LobeChat, etc.) send content as an array of typed parts instead of a plain string:: [{"type": "text", "text": "hello"}, {"type": "input_text", "text": "..."}] This function flattens those into a single string so the agent pipeline (which expects strings) doesn't choke. Defensive limits prevent abuse: recursion depth, list size, and output length are all bounded. """ if _depth > _max_depth: return "" if content is None: return "" if isinstance(content, str): return content[:MAX_NORMALIZED_TEXT_LENGTH] if len(content) > MAX_NORMALIZED_TEXT_LENGTH else content if isinstance(content, list): parts: List[str] = [] total_len = 0 items = content[:MAX_CONTENT_LIST_SIZE] if len(content) > MAX_CONTENT_LIST_SIZE else content for item in items: if isinstance(item, str): if item: part = item[:MAX_NORMALIZED_TEXT_LENGTH] parts.append(part) total_len += len(part) elif isinstance(item, dict): item_type = str(item.get("type") or "").strip().lower() if item_type in {"text", "input_text", "output_text"}: text = item.get("text", "") if text: try: part = str(text)[:MAX_NORMALIZED_TEXT_LENGTH] parts.append(part) total_len += len(part) except Exception: pass # Silently skip image_url / other non-text parts elif isinstance(item, list): nested = _normalize_chat_content(item, _max_depth=_max_depth, _depth=_depth + 1) if nested: parts.append(nested) total_len += len(nested) # Check accumulated size if total_len >= MAX_NORMALIZED_TEXT_LENGTH: break result = "\n".join(parts) return result[:MAX_NORMALIZED_TEXT_LENGTH] if len(result) > MAX_NORMALIZED_TEXT_LENGTH else result # Fallback for unexpected types (int, float, bool, etc.) try: result = str(content) return result[:MAX_NORMALIZED_TEXT_LENGTH] if len(result) > MAX_NORMALIZED_TEXT_LENGTH else result except Exception: return "" # Content part type aliases used by the OpenAI Chat Completions and Responses # APIs. We accept both spellings on input and emit a single canonical internal # shape (``{"type": "text", ...}`` / ``{"type": "image_url", ...}``) that the # rest of the agent pipeline already understands. _TEXT_PART_TYPES = frozenset({"text", "input_text", "output_text"}) _IMAGE_PART_TYPES = frozenset({"image_url", "input_image"}) _FILE_PART_TYPES = frozenset({"file", "input_file"}) def _normalize_multimodal_content(content: Any) -> Any: """Validate and normalize multimodal content for the API server. Returns a plain string when the content is text-only, or a list of ``{"type": "text"|"image_url", ...}`` parts when images are present. The output shape is the native OpenAI Chat Completions vision format, which the agent pipeline accepts verbatim (OpenAI-wire providers) or converts (``_preprocess_anthropic_content`` for Anthropic). Raises ``ValueError`` with an OpenAI-style code on invalid input: * ``unsupported_content_type`` — file/input_file/file_id parts, or non-image ``data:`` URLs. * ``invalid_image_url`` — missing URL or unsupported scheme. * ``invalid_content_part`` — malformed text/image objects. Callers translate the ValueError into a 400 response. """ # Scalar passthrough mirrors ``_normalize_chat_content``. if content is None: return "" if isinstance(content, str): return content[:MAX_NORMALIZED_TEXT_LENGTH] if len(content) > MAX_NORMALIZED_TEXT_LENGTH else content if not isinstance(content, list): # Mirror the legacy text-normalizer's fallback so callers that # pre-existed image support still get a string back. return _normalize_chat_content(content) items = content[:MAX_CONTENT_LIST_SIZE] if len(content) > MAX_CONTENT_LIST_SIZE else content normalized_parts: List[Dict[str, Any]] = [] text_accum_len = 0 for part in items: if isinstance(part, str): if part: trimmed = part[:MAX_NORMALIZED_TEXT_LENGTH] normalized_parts.append({"type": "text", "text": trimmed}) text_accum_len += len(trimmed) continue if not isinstance(part, dict): # Ignore unknown scalars for forward compatibility with future # Responses API additions (e.g. ``refusal``). The same policy # the text normalizer applies. continue raw_type = part.get("type") part_type = str(raw_type or "").strip().lower() if part_type in _TEXT_PART_TYPES: text = part.get("text") if text is None: continue if not isinstance(text, str): text = str(text) if text: trimmed = text[:MAX_NORMALIZED_TEXT_LENGTH] normalized_parts.append({"type": "text", "text": trimmed}) text_accum_len += len(trimmed) continue if part_type in _IMAGE_PART_TYPES: detail = part.get("detail") image_ref = part.get("image_url") # OpenAI Responses sends ``input_image`` with a top-level # ``image_url`` string; Chat Completions sends ``image_url`` as # ``{"url": "...", "detail": "..."}``. Support both. if isinstance(image_ref, dict): url_value = image_ref.get("url") detail = image_ref.get("detail", detail) else: url_value = image_ref if not isinstance(url_value, str) or not url_value.strip(): raise ValueError("invalid_image_url:Image parts must include a non-empty image URL.") url_value = url_value.strip() lowered = url_value.lower() if lowered.startswith("data:"): if not lowered.startswith("data:image/") or "," not in url_value: raise ValueError( "unsupported_content_type:Only image data URLs are supported. " "Non-image data payloads are not supported." ) elif not (lowered.startswith("http://") or lowered.startswith("https://")): raise ValueError( "invalid_image_url:Image inputs must use http(s) URLs or data:image/... URLs." ) image_part: Dict[str, Any] = {"type": "image_url", "image_url": {"url": url_value}} if detail is not None: if not isinstance(detail, str) or not detail.strip(): raise ValueError("invalid_content_part:Image detail must be a non-empty string when provided.") image_part["image_url"]["detail"] = detail.strip() normalized_parts.append(image_part) continue if part_type in _FILE_PART_TYPES: raise ValueError( "unsupported_content_type:Inline image inputs are supported, " "but uploaded files and document inputs are not supported on this endpoint." ) # Unknown part type — reject explicitly so clients get a clear error # instead of a silently dropped turn. raise ValueError( f"unsupported_content_type:Unsupported content part type {raw_type!r}. " "Only text and image_url/input_image parts are supported." ) if not normalized_parts: return "" # Text-only: collapse to a plain string so downstream logging/trajectory # code sees the native shape and prompt caching on text-only turns is # unaffected. if all(p.get("type") == "text" for p in normalized_parts): return "\n".join(p["text"] for p in normalized_parts if p.get("text")) return normalized_parts def _content_has_visible_payload(content: Any) -> bool: """True when content has any text or image attachment. Used to reject empty turns.""" if isinstance(content, str): return bool(content.strip()) if isinstance(content, list): for part in content: if isinstance(part, dict): ptype = str(part.get("type") or "").strip().lower() if ptype in _TEXT_PART_TYPES and str(part.get("text") or "").strip(): return True if ptype in _IMAGE_PART_TYPES: return True return False def _multimodal_validation_error(exc: ValueError, *, param: str) -> "web.Response": """Translate a ``_normalize_multimodal_content`` ValueError into a 400 response.""" raw = str(exc) code, _, message = raw.partition(":") if not message: code, message = "invalid_content_part", raw return web.json_response( _openai_error(message, code=code, param=param), status=400, ) def _session_chat_user_message(body: Dict[str, Any], *, param: str = "message") -> tuple[Any, Optional["web.Response"]]: """Parse and normalize session chat ``message`` / ``input`` like chat completions.""" user_message = body.get("message") or body.get("input") if not _content_has_visible_payload(user_message): return None, web.json_response( _openai_error("Missing 'message' field", code="missing_message"), status=400, ) try: return _normalize_multimodal_content(user_message), None except ValueError as exc: return None, _multimodal_validation_error(exc, param=param) def check_api_server_requirements() -> bool: """Check if API server dependencies are available.""" return AIOHTTP_AVAILABLE class ResponseStore: """ SQLite-backed LRU store for Responses API state. Each stored response includes the full internal conversation history (with tool calls and results) so it can be reconstructed on subsequent requests via previous_response_id. Persists across gateway restarts. Falls back to in-memory SQLite if the on-disk path is unavailable. """ def __init__(self, max_size: int = MAX_STORED_RESPONSES, db_path: str = None): self._max_size = max_size if db_path is None: try: from hermes_cli.config import get_hermes_home db_path = str(get_hermes_home() / "response_store.db") except Exception: db_path = ":memory:" self._db_path: Optional[str] = db_path if db_path != ":memory:" else None try: self._conn = sqlite3.connect(db_path, check_same_thread=False) except Exception: self._conn = sqlite3.connect(":memory:", check_same_thread=False) self._db_path = None # Use shared WAL-fallback helper so response_store.db degrades # gracefully on NFS/SMB/FUSE-mounted HERMES_HOME (same filesystem # issue addressed for state.db/kanban.db — see # hermes_state._WAL_INCOMPAT_MARKERS). from hermes_state import apply_wal_with_fallback apply_wal_with_fallback(self._conn, db_label="response_store.db") self._conn.execute( """CREATE TABLE IF NOT EXISTS responses ( response_id TEXT PRIMARY KEY, data TEXT NOT NULL, accessed_at REAL NOT NULL )""" ) self._conn.execute( """CREATE TABLE IF NOT EXISTS conversations ( name TEXT PRIMARY KEY, response_id TEXT NOT NULL )""" ) self._conn.commit() # response_store.db contains conversation history (tool payloads, # prompts, results). Tighten to owner-only after creation so other # local users on a shared box can't read it. Run once at __init__ # rather than after every commit — chmod-on-every-write is wasted # syscalls on a hot path. self._tighten_file_permissions() def _tighten_file_permissions(self) -> None: """Force owner-only permissions on the DB and SQLite sidecars.""" if not self._db_path: return for candidate in ( Path(self._db_path), Path(f"{self._db_path}-wal"), Path(f"{self._db_path}-shm"), ): try: if candidate.exists(): candidate.chmod(0o600) except OSError: logger.debug( "Failed to restrict response store permissions for %s", candidate, exc_info=True, ) def get(self, response_id: str) -> Optional[Dict[str, Any]]: """Retrieve a stored response by ID (updates access time for LRU).""" row = self._conn.execute( "SELECT data FROM responses WHERE response_id = ?", (response_id,) ).fetchone() if row is None: return None self._conn.execute( "UPDATE responses SET accessed_at = ? WHERE response_id = ?", (time.time(), response_id), ) self._conn.commit() try: return json.loads(row[0]) except (json.JSONDecodeError, TypeError): logger.warning( "Corrupted JSON in response store for id=%s, evicting entry", response_id, ) self._conn.execute( "DELETE FROM responses WHERE response_id = ?", (response_id,), ) self._conn.commit() return None def put(self, response_id: str, data: Dict[str, Any]) -> None: """Store a response, evicting the oldest if at capacity.""" self._conn.execute( "INSERT OR REPLACE INTO responses (response_id, data, accessed_at) VALUES (?, ?, ?)", (response_id, json.dumps(data, default=str), time.time()), ) # Evict oldest entries beyond max_size count = self._conn.execute("SELECT COUNT(*) FROM responses").fetchone()[0] if count > self._max_size: # Collect IDs that will be evicted evict_ids = [ row[0] for row in self._conn.execute( "SELECT response_id FROM responses ORDER BY accessed_at ASC LIMIT ?", (count - self._max_size,), ).fetchall() ] if evict_ids: placeholders = ",".join("?" for _ in evict_ids) # Clear conversation mappings pointing to evicted responses self._conn.execute( f"DELETE FROM conversations WHERE response_id IN ({placeholders})", evict_ids, ) # Delete evicted responses self._conn.execute( f"DELETE FROM responses WHERE response_id IN ({placeholders})", evict_ids, ) self._conn.commit() def delete(self, response_id: str) -> bool: """Remove a response from the store. Returns True if found and deleted.""" # Clear conversation mappings pointing to this response self._conn.execute( "DELETE FROM conversations WHERE response_id = ?", (response_id,) ) cursor = self._conn.execute( "DELETE FROM responses WHERE response_id = ?", (response_id,) ) self._conn.commit() return cursor.rowcount > 0 def get_conversation(self, name: str) -> Optional[str]: """Get the latest response_id for a conversation name.""" row = self._conn.execute( "SELECT response_id FROM conversations WHERE name = ?", (name,) ).fetchone() return row[0] if row else None def set_conversation(self, name: str, response_id: str) -> None: """Map a conversation name to its latest response_id.""" self._conn.execute( "INSERT OR REPLACE INTO conversations (name, response_id) VALUES (?, ?)", (name, response_id), ) self._conn.commit() def close(self) -> None: """Close the database connection.""" try: self._conn.close() except Exception: pass def __len__(self) -> int: row = self._conn.execute("SELECT COUNT(*) FROM responses").fetchone() return row[0] if row else 0 # --------------------------------------------------------------------------- # CORS middleware # --------------------------------------------------------------------------- _CORS_HEADERS = { "Access-Control-Allow-Methods": "GET, POST, DELETE, OPTIONS", "Access-Control-Allow-Headers": "Authorization, Content-Type, Idempotency-Key", } if AIOHTTP_AVAILABLE: @web.middleware async def cors_middleware(request, handler): """Add CORS headers for explicitly allowed origins; handle OPTIONS preflight.""" adapter = request.app.get("api_server_adapter") origin = request.headers.get("Origin", "") cors_headers = None if adapter is not None: if not adapter._origin_allowed(origin): return web.Response(status=403) cors_headers = adapter._cors_headers_for_origin(origin) if request.method == "OPTIONS": if cors_headers is None: return web.Response(status=403) return web.Response(status=200, headers=cors_headers) response = await handler(request) if cors_headers is not None: response.headers.update(cors_headers) return response else: cors_middleware = None # type: ignore[assignment] _MEDIA_IMG_EXT = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp"} _MEDIA_MIME = { ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".gif": "image/gif", ".webp": "image/webp", ".bmp": "image/bmp", } _MEDIA_DATA_URL_MAX_BYTES = 5 * 1024 * 1024 # skip images larger than 5MB def _resolve_media_to_data_urls(text: str) -> str: """Replace ``MEDIA:`` image tags with inline base64 data URLs. Remote OpenAI-compatible frontends can't read local file paths, so ``MEDIA:`` tags referencing images on the server are useless to them. Inline small local images as markdown data URLs; non-image or unreadable paths are left untouched. Uses the same anchored ``MEDIA_TAG_CLEANUP_RE`` matcher and ``validate_media_delivery_path`` safety check every other platform adapter's media delivery already goes through (gateway/platforms/base.py) — an absolute-path anchor plus a known-extension requirement, and a resolved-path check against the credential/system-path denylist. The prior pattern here matched any bare token after ``MEDIA:`` (including a relative/traversal path like ``../../etc/passwd.png``) and read the file directly with no denylist, so any image-suffixed, readable file the process could see was base64-exfiltrated to the API caller if its path merely appeared in the model's own final reply text. """ if not text or "MEDIA:" not in text: return text import base64 def _to_data_url(path_str: str) -> Optional[str]: # validate_media_delivery_path() strips wrapping quotes/backticks # and trailing punctuation internally, same as MEDIA_TAG_CLEANUP_RE's # other callers (extract_media / _strip_media_tag_directives) rely on. safe_path = validate_media_delivery_path(path_str) if not safe_path: return None p = Path(safe_path) suffix = p.suffix.lower() if suffix not in _MEDIA_IMG_EXT: return None try: if p.stat().st_size > _MEDIA_DATA_URL_MAX_BYTES: return None b64 = base64.b64encode(p.read_bytes()).decode() except OSError: return None return f"![image](data:{_MEDIA_MIME[suffix]};base64,{b64})" def _repl(m: "re.Match[str]") -> str: return _to_data_url(m.group("path")) or m.group(0) try: return MEDIA_TAG_CLEANUP_RE.sub(_repl, text) except Exception: return text def _redact_api_error_text(value: Any, *, limit: int | None = None) -> str: """Redact API-bound error text before it crosses the HTTP boundary.""" redacted = redact_sensitive_text(str(value), force=True) if limit is not None: return redacted[:limit] return redacted def _openai_error(message: str, err_type: str = "invalid_request_error", param: str = None, code: str = None) -> Dict[str, Any]: """OpenAI-style error envelope.""" return { "error": { "message": _redact_api_error_text(message), "type": err_type, "param": param, "code": code, } } _api_agent_request_reservation: ContextVar[Optional[dict[str, bool]]] = ContextVar( "api_agent_request_reservation", default=None ) def _admit_api_agent_request(handler): """Reserve an authenticated API turn before its handler first awaits. Gateway shutdown and aiohttp requests share an event loop. Keeping the drain check and reservation in one non-awaiting block prevents a request admitted immediately before shutdown from becoming invisible while it is still parsing its body or resolving session state. The mutable reservation is intentionally shared with child tasks so agent/task bookkeeping releases this one slot exactly once. """ @wraps(handler) async def _wrapped(self, request, *args, **kwargs): auth_err = self._check_auth(request) if auth_err: return auth_err draining = self._draining_response() if draining is not None: return draining reservation = {"active": True} token = _api_agent_request_reservation.set(reservation) self._pending_agent_requests += 1 try: return await handler(self, request, *args, **kwargs) finally: if reservation["active"]: reservation["active"] = False self._pending_agent_requests = max(0, self._pending_agent_requests - 1) _api_agent_request_reservation.reset(token) return _wrapped def _release_pending_api_work(adapter, reservation: dict[str, bool]) -> None: """Release a pending-work reservation exactly once.""" if reservation["active"]: reservation["active"] = False adapter._pending_agent_requests = max(0, adapter._pending_agent_requests - 1) @contextmanager def _reserve_pending_api_work(adapter): """Keep externally-triggered background work visible across awaits. A handler can detach the reservation to an asyncio task; its done callback then owns release so shutdown cannot miss the handoff to background work. """ reservation = {"active": True, "detached": False} adapter._pending_agent_requests += 1 try: yield reservation finally: if not reservation["detached"]: _release_pending_api_work(adapter, reservation) if AIOHTTP_AVAILABLE: @web.middleware async def body_limit_middleware(request, handler): """Reject overly large request bodies early based on Content-Length.""" if request.method in {"POST", "PUT", "PATCH"}: cl = request.headers.get("Content-Length") if cl is not None: try: if int(cl) > MAX_REQUEST_BYTES: return web.json_response(_openai_error("Request body too large.", code="body_too_large"), status=413) except ValueError: return web.json_response(_openai_error("Invalid Content-Length header.", code="invalid_content_length"), status=400) try: return await handler(request) except web.HTTPRequestEntityTooLarge: # aiohttp's client_max_size tripped mid-read (chunked bodies carry # no Content-Length) — return a proper 413 instead of letting the # handler's broad JSON except turn it into 400 "Invalid JSON". return web.json_response( _openai_error("Request body too large.", code="body_too_large"), status=413, ) else: body_limit_middleware = None # type: ignore[assignment] _SECURITY_HEADERS = { "Content-Security-Policy": "default-src 'none'; frame-ancestors 'none'", "Permissions-Policy": "camera=(), microphone=(), geolocation=()", "Strict-Transport-Security": "max-age=31536000; includeSubDomains", "X-Content-Type-Options": "nosniff", "X-Frame-Options": "DENY", "X-XSS-Protection": "0", "Referrer-Policy": "no-referrer", } if AIOHTTP_AVAILABLE: @web.middleware async def security_headers_middleware(request, handler): """Add security headers to all responses (including errors).""" response = await handler(request) for k, v in _SECURITY_HEADERS.items(): response.headers.setdefault(k, v) return response else: security_headers_middleware = None # type: ignore[assignment] class _IdempotencyCache: """In-memory idempotency cache with TTL and basic LRU semantics.""" def __init__(self, max_items: int = 1000, ttl_seconds: int = 300): from collections import OrderedDict self._store = OrderedDict() self._inflight: Dict[tuple[str, str], "asyncio.Task[Any]"] = {} self._ttl = ttl_seconds self._max = max_items def _purge(self): now = time.time() expired = [k for k, v in self._store.items() if now - v["ts"] > self._ttl] for k in expired: self._store.pop(k, None) while len(self._store) > self._max: self._store.popitem(last=False) async def get_or_set(self, key: str, fingerprint: str, compute_coro): self._purge() item = self._store.get(key) if item and item["fp"] == fingerprint: return item["resp"] inflight_key = (key, fingerprint) task = self._inflight.get(inflight_key) if task is None: async def _compute_and_store(): resp = await compute_coro() import time as _t self._store[key] = {"resp": resp, "fp": fingerprint, "ts": _t.time()} self._purge() return resp task = asyncio.create_task(_compute_and_store()) self._inflight[inflight_key] = task def _clear_inflight(done_task: "asyncio.Task[Any]") -> None: if self._inflight.get(inflight_key) is done_task: self._inflight.pop(inflight_key, None) task.add_done_callback(_clear_inflight) return await asyncio.shield(task) _idem_cache = _IdempotencyCache() def _make_request_fingerprint(body: Dict[str, Any], keys: List[str]) -> str: from hashlib import sha256 subset = {k: body.get(k) for k in keys} return sha256(repr(subset).encode("utf-8")).hexdigest() def _derive_chat_session_id( system_prompt: Optional[str], first_user_message: str, ) -> str: """Derive a stable session ID from the conversation's first user message. OpenAI-compatible frontends (Open WebUI, LibreChat, etc.) send the full conversation history with every request. The system prompt and first user message are constant across all turns of the same conversation, so hashing them produces a deterministic session ID that lets the API server reuse the same Hermes session (and therefore the same Docker container sandbox directory) across turns. """ seed = f"{system_prompt or ''}\n{first_user_message}" digest = hashlib.sha256(seed.encode("utf-8")).hexdigest()[:16] return f"api-{digest}" _CRON_AVAILABLE = False try: from cron.jobs import ( list_jobs as _cron_list, get_job as _cron_get, create_job as _cron_create, update_job as _cron_update, remove_job as _cron_remove, pause_job as _cron_pause, resume_job as _cron_resume, trigger_job as _cron_trigger, ) _CRON_AVAILABLE = True except ImportError: _cron_list = None _cron_get = None _cron_create = None _cron_update = None _cron_remove = None _cron_pause = None _cron_resume = None _cron_trigger = None def _notify_cron_provider_jobs_changed() -> None: """Tell the active cron scheduler provider the job set changed after a REST mutation (no-op for the built-in). Best-effort — never breaks the handler.""" try: from cron.scheduler import _notify_provider_jobs_changed _notify_provider_jobs_changed() except Exception: pass # Defense-in-depth: mirror the agent-facing cronjob tool, which scans the # user-supplied prompt for exfiltration/injection payloads at create/update # time (tools/cronjob_tools.py). The REST cron endpoints are authenticated # (every handler runs _check_auth, and connect() refuses to start without # API_SERVER_KEY), so this is not the trust boundary — it's parity with the # tool path so a malicious prompt is rejected the same way regardless of # which surface created the job. Imported defensively: a missing scanner # must not disable the cron REST API. try: from tools.cronjob_tools import _scan_cron_prompt as _scan_cron_prompt except Exception: # pragma: no cover - scanner is optional hardening _scan_cron_prompt = None class APIServerAdapter(BasePlatformAdapter): """ OpenAI-compatible HTTP API server adapter. Runs an aiohttp web server that accepts OpenAI-format requests and routes them through hermes-agent's AIAgent. """ # Stateless request/response: every route (the OpenAI-spec # /v1/chat/completions and /v1/responses, and the proprietary /v1/runs SSE # stream) tears down its channel when the turn ends. There is no persistent # outbound channel to push a background completion to a client that already # received its response, and ``send()`` is a no-op stub. So async-delivery # tools (terminal notify_on_complete / watch_patterns, delegate_task # background=True) must NOT promise delivery on this path — see # ``async_delivery_supported()``. supports_async_delivery: bool = False def __init__(self, config: PlatformConfig): super().__init__(config, Platform.API_SERVER) extra = config.extra or {} self._host: str = extra.get("host", os.getenv("API_SERVER_HOST", DEFAULT_HOST)) raw_port = extra.get("port") if raw_port is None: raw_port = os.getenv("API_SERVER_PORT", str(DEFAULT_PORT)) self._port: int = _coerce_port(raw_port, DEFAULT_PORT) self._api_key: str = extra.get("key", os.getenv("API_SERVER_KEY", "")) self._cors_origins: tuple[str, ...] = self._parse_cors_origins( extra.get("cors_origins", os.getenv("API_SERVER_CORS_ORIGINS", "")), ) self._model_name: str = self._resolve_model_name( extra.get("model_name", os.getenv("API_SERVER_MODEL_NAME", "")), ) # model_routes: maps incoming ``model`` field values to specific # provider/model configs so one API server instance can serve # multiple clients on different backends. # # Config format (platforms.api_server.extra in the gateway config): # model_routes: # minimax-m2: # alias the client sends as the "model" field # model: "minimax/minimax-m1" # provider: "openrouter" # optional — resolved via the provider # # credential chain when set # api_key: "sk-…" # optional — per-route UPSTREAM provider # # key override (NOT caller auth; never logged) # base_url: "https://…" # optional — per-route base URL override self._model_routes: Dict[str, Dict[str, Any]] = self._parse_model_routes( extra.get("model_routes"), ) self._app: Optional["web.Application"] = None self._runner: Optional["web.AppRunner"] = None self._site: Optional["web.TCPSite"] = None self._response_store = ResponseStore() # Active run streams: run_id -> asyncio.Queue of SSE event dicts self._run_streams: Dict[str, "asyncio.Queue[Optional[Dict]]"] = {} # Creation timestamps for orphaned-run TTL sweep self._run_streams_created: Dict[str, float] = {} # Runs with a connected SSE consumer; their queue is actively draining. self._run_stream_subscribers: set[str] = set() # Active run agent/task references for stop support self._active_run_agents: Dict[str, Any] = {} self._active_run_tasks: Dict[str, "asyncio.Task"] = {} # Stop is cooperative: the executor thread may outlive the HTTP request. self._stopping_run_ids: set[str] = set() # Pollable run status for dashboards and external control-plane UIs. self._run_statuses: Dict[str, Dict[str, Any]] = {} # Active approval session key for each run_id. The approval core # resolves requests by session key, while API clients address the # in-flight run by run_id. self._run_approval_sessions: Dict[str, str] = {} self._session_db: Optional[Any] = None # Lazy-init SessionDB for session continuity # Concurrency cap shared across all agent-serving endpoints # (/v1/chat/completions, /v1/responses, /v1/runs). Read from # config.yaml gateway.api_server.max_concurrent_runs; 0 disables # the cap. Bounds CPU / memory / upstream-LLM-quota exhaustion # from a request flood (#7483). self._max_concurrent_runs: int = self._resolve_max_concurrent_runs() # Number of in-flight runs on the non-streaming chat/responses paths # (the /v1/runs path tracks its own in-flight set via # _active_run_tasks). self._inflight_agent_runs: int = 0 # Requests admitted before their handler reaches agent bookkeeping. # Shutdown counts this reservation so the request cannot slip through # the drain between its first await and _run_agent()/task registration. self._pending_agent_requests: int = 0 def active_agent_work_count(self) -> int: """Return all live agent work owned by this API adapter. ``/v1/runs`` registers an asyncio task before it constructs and stores its agent, so ``_active_run_agents`` has a real queued-before-agent gap. Reuse the task-based accounting used by the concurrent-run limit: it covers that gap and excludes completed tasks retained until cleanup. """ try: return ( int(getattr(self, "_pending_agent_requests", 0)) + int(self._inflight_agent_runs) + sum(not task.done() for task in self._active_run_tasks.values()) ) except Exception: return 0 @staticmethod def _gateway_is_draining() -> bool: """Whether the owning gateway currently refuses new agent turns.""" try: from gateway.run import _gateway_runner_ref runner = _gateway_runner_ref() return bool( runner and ( getattr(runner, "_draining", False) or getattr(runner, "_external_drain_active", False) ) ) except Exception: return False def _draining_response(self) -> Optional["web.Response"]: """Return a retryable response while the gateway drains existing work.""" if not self._gateway_is_draining(): return None return web.json_response( _openai_error( "Gateway is draining existing work; retry shortly.", code="gateway_draining", ), status=503, headers={"Retry-After": "1"}, ) def _activate_admitted_request(self) -> None: """Transfer this request's drain reservation to agent bookkeeping.""" reservation = _api_agent_request_reservation.get() if reservation and reservation["active"]: reservation["active"] = False self._pending_agent_requests = max(0, self._pending_agent_requests - 1) def _readiness_work_counts(self) -> tuple[int, int, int]: """Return bounded work counts from each subsystem's public state.""" active_api_runs = sum( 1 for status in self._run_statuses.values() if status.get("status") in {"queued", "running", "waiting_for_approval"} ) process_depth = 0 active_delegations = 0 try: from tools.process_registry import process_registry process_depth = process_registry.completion_queue.qsize() except Exception: pass try: from tools.async_delegation import active_count active_delegations = active_count() except Exception: pass return active_api_runs, process_depth, active_delegations @staticmethod def _parse_cors_origins(value: Any) -> tuple[str, ...]: """Normalize configured CORS origins into a stable tuple.""" if not value: return () if isinstance(value, str): items = value.split(",") elif isinstance(value, (list, tuple, set)): items = value else: items = [str(value)] return tuple(str(item).strip() for item in items if str(item).strip()) @staticmethod def _resolve_max_concurrent_runs() -> int: """Read the concurrent-run cap from config.yaml (0 disables). gateway.api_server.max_concurrent_runs. Falls back to the historical default of 10 when unset or malformed. Negative values are clamped to 0 (disabled). """ default = 10 try: from hermes_cli.config import cfg_get, load_config raw = cfg_get( load_config(), "gateway", "api_server", "max_concurrent_runs", default=default, ) value = int(raw) except Exception: return default return max(0, value) @staticmethod def _resolve_model_name(explicit: str) -> str: """Derive the advertised model name for /v1/models. Priority: 1. Explicit override (config extra or API_SERVER_MODEL_NAME env var) 2. Active profile name (so each profile advertises a distinct model) 3. Fallback: "hermes-agent" """ if explicit and explicit.strip(): return explicit.strip() try: from hermes_cli.profiles import get_active_profile_name profile = get_active_profile_name() if profile and profile not in {"default", "custom"}: return profile except Exception: pass return "hermes-agent" def _cors_headers_for_origin(self, origin: str) -> Optional[Dict[str, str]]: """Return CORS headers for an allowed browser origin.""" if not origin or not self._cors_origins: return None if "*" in self._cors_origins: headers = dict(_CORS_HEADERS) headers["Access-Control-Allow-Origin"] = "*" headers["Access-Control-Max-Age"] = "600" return headers if origin not in self._cors_origins: return None headers = dict(_CORS_HEADERS) headers["Access-Control-Allow-Origin"] = origin headers["Vary"] = "Origin" headers["Access-Control-Max-Age"] = "600" return headers def _origin_allowed(self, origin: str) -> bool: """Allow non-browser clients and explicitly configured browser origins.""" if not origin: return True if not self._cors_origins: return False return "*" in self._cors_origins or origin in self._cors_origins @staticmethod def _clean_log_value(value: Any, *, max_len: int = 200) -> str: """Sanitize request metadata before it reaches security logs.""" if value is None: return "" text = str(value).replace("\r", " ").replace("\n", " ").strip() return text[:max_len] def _request_audit_context(self, request: "web.Request") -> Dict[str, str]: """Return non-secret source metadata for security/audit warnings.""" peer_ip = "" try: peer = request.transport.get_extra_info("peername") if request.transport else None if isinstance(peer, (tuple, list)) and peer: peer_ip = str(peer[0]) except Exception: peer_ip = "" return { "remote": self._clean_log_value(getattr(request, "remote", "") or peer_ip), "peer_ip": self._clean_log_value(peer_ip), "forwarded_for": self._clean_log_value(request.headers.get("X-Forwarded-For", "")), "real_ip": self._clean_log_value(request.headers.get("X-Real-IP", "")), "method": self._clean_log_value(request.method, max_len=16), "path": self._clean_log_value(request.path_qs, max_len=500), "user_agent": self._clean_log_value(request.headers.get("User-Agent", ""), max_len=300), } def _request_audit_log_suffix(self, request: "web.Request") -> str: ctx = self._request_audit_context(request) fields = [f"{key}={value!r}" for key, value in ctx.items() if value] return " ".join(fields) if fields else "source='unknown'" def _cron_origin_from_request(self, request: "web.Request") -> Dict[str, str]: """Persist safe API source metadata on cron jobs created over HTTP.""" ctx = self._request_audit_context(request) origin = { "platform": "api_server", "chat_id": "api", } if ctx.get("remote"): origin["source_ip"] = ctx["remote"] if ctx.get("peer_ip"): origin["peer_ip"] = ctx["peer_ip"] if ctx.get("forwarded_for"): origin["forwarded_for"] = ctx["forwarded_for"] if ctx.get("real_ip"): origin["real_ip"] = ctx["real_ip"] if ctx.get("user_agent"): origin["user_agent"] = ctx["user_agent"] return origin # ------------------------------------------------------------------ # Auth helper # ------------------------------------------------------------------ def _check_auth(self, request: "web.Request") -> Optional["web.Response"]: """ Validate Bearer token from Authorization header. Returns None if auth is OK, or a 401 web.Response on failure. connect() refuses to start the API server without API_SERVER_KEY, so the no-key branch only exists for tests or unsupported manual wiring. """ if not self._api_key: return None auth_header = request.headers.get("Authorization", "") if auth_header.startswith("Bearer "): token = auth_header[7:].strip() if hmac.compare_digest(token, self._api_key): return None # Auth OK logger.warning( "API server rejected invalid API key: %s", self._request_audit_log_suffix(request), ) return web.json_response( {"error": {"message": "Invalid API key", "type": "invalid_request_error", "code": "invalid_api_key"}}, status=401, ) # ------------------------------------------------------------------ # Session header helpers # ------------------------------------------------------------------ # Soft length cap for session identifiers. Headers are bounded in # aggregate by aiohttp (``client_max_size`` / default 8 KiB per # header), but we impose a tighter limit on the session headers so a # caller can't burn memory by passing a multi-kilobyte "session key". # 256 chars is well above any realistic stable channel identifier # (e.g. ``agent:main:webui:dm:user-42``) while staying small enough # that the sanitized form is safe to pass into Honcho / state.db. _MAX_SESSION_HEADER_LEN = 256 def _parse_session_key_header( self, request: "web.Request" ) -> tuple[Optional[str], Optional["web.Response"]]: """Extract and validate the ``X-Hermes-Session-Key`` header. The session key is a stable per-channel identifier that scopes long-term memory (e.g. Honcho sessions) across transcripts. It is independent of ``X-Hermes-Session-Id``: callers may send either, both, or neither. Returns ``(session_key, None)`` on success (with an empty/absent header yielding ``None`` for the key), or ``(None, error_response)`` on validation failure. Security: like session continuation, accepting a caller-supplied memory scope requires API-key authentication so that an unauthenticated client on a local-only server can't inject itself into another user's long-term memory scope by guessing a key. """ raw = request.headers.get("X-Hermes-Session-Key", "").strip() if not raw: return None, None if not self._api_key: logger.warning( "X-Hermes-Session-Key rejected: no API key configured. " "Set API_SERVER_KEY to enable long-term memory scoping." ) return None, web.json_response( _openai_error( "X-Hermes-Session-Key requires API key authentication. " "Configure API_SERVER_KEY to enable this feature." ), status=403, ) # Reject control characters that could enable header injection on # the echo path. if re.search(r'[\r\n\x00]', raw): return None, web.json_response( {"error": {"message": "Invalid session key", "type": "invalid_request_error"}}, status=400, ) if len(raw) > self._MAX_SESSION_HEADER_LEN: return None, web.json_response( {"error": {"message": "Session key too long", "type": "invalid_request_error"}}, status=400, ) return raw, None # ------------------------------------------------------------------ # Session DB helper # ------------------------------------------------------------------ def _ensure_session_db(self): """Lazily initialise and return the shared SessionDB instance. Sessions are persisted to ``state.db`` so that ``hermes sessions list`` shows API-server conversations alongside CLI and gateway ones. """ if self._session_db is None: try: from hermes_state import SessionDB self._session_db = SessionDB() except Exception as e: logger.debug("SessionDB unavailable for API server: %s", e) return self._session_db # ------------------------------------------------------------------ # Agent creation helper # ------------------------------------------------------------------ @staticmethod def _parse_model_routes(raw: Any) -> Dict[str, Dict[str, Any]]: """Validate and normalize the ``model_routes`` config block. Accepts a mapping of ``alias -> {model, provider?, api_key?, base_url?}``. Invalid shapes are dropped (never raised) so a config typo can't take the whole API server down. Route values are coerced to strings. Security: per-route ``api_key`` values are UPSTREAM provider credentials (used to call the routed model's backend), not caller authentication — callers still authenticate with the global API_SERVER_KEY bearer token via ``_check_auth``. Route api_keys must never be logged; only alias names and non-secret fields may appear in logs. """ if not isinstance(raw, dict): if raw: logger.warning( "api_server model_routes ignored: expected a mapping, got %s", type(raw).__name__, ) return {} allowed_keys = ("model", "provider", "api_key", "base_url") routes: Dict[str, Dict[str, Any]] = {} for alias, cfg in raw.items(): alias_str = str(alias).strip() if not alias_str or not isinstance(cfg, dict): logger.warning( "api_server model_routes: dropping invalid route entry %r", alias_str or alias ) continue route = { key: str(cfg[key]).strip() for key in allowed_keys if cfg.get(key) is not None and str(cfg[key]).strip() } if not route.get("model"): logger.warning( "api_server model_routes: route %r has no 'model'; dropping", alias_str ) continue routes[alias_str] = route return routes def _resolve_route(self, model_alias: Any) -> Optional[Dict[str, Any]]: """Return the model_routes entry for *model_alias*, or None.""" if not self._model_routes or not isinstance(model_alias, str): return None return self._model_routes.get(model_alias) def _session_model_override_for(self, session_key: Optional[str]) -> Optional[Dict[str, Any]]: """Return the gateway's session ``/model`` override for *session_key*, if any. The gateway tracks per-session ``/model`` switches in ``GatewayRunner._session_model_overrides``. API-server requests that share such a session key must keep honouring the explicit session override even when the request's ``model`` field matches a configured route — a user-issued ``/model`` always wins over static config. """ if not session_key: return None try: from gateway.run import _gateway_runner_ref runner = _gateway_runner_ref() if runner is None: return None override = runner._session_model_overrides.get(session_key) return dict(override) if isinstance(override, dict) else None except Exception: return None def _create_agent( self, ephemeral_system_prompt: Optional[str] = None, session_id: Optional[str] = None, stream_delta_callback=None, tool_progress_callback=None, tool_start_callback=None, tool_complete_callback=None, gateway_session_key: Optional[str] = None, route: Optional[Dict[str, Any]] = None, ) -> Any: """ Create an AIAgent instance using the gateway's runtime config. Uses _resolve_runtime_agent_kwargs() to pick up model, api_key, base_url, etc. from config.yaml / env vars. Toolsets are resolved from config.yaml platform_toolsets.api_server (same as all other gateway platforms), falling back to the hermes-api-server default. ``gateway_session_key`` is a stable per-channel identifier supplied by the client (via ``X-Hermes-Session-Key``). Unlike ``session_id`` which scopes the short-term transcript and rotates on /new, this key is meant to persist across transcripts so long-term memory providers (e.g. Honcho) can scope their per-chat state correctly — matching the semantics of the native gateway's ``session_key``. ``route`` is an optional ``model_routes`` entry (per-client model routing). When set — and no session ``/model`` override exists for this session — its model/provider/api_key/base_url override the global defaults for this agent instance only. """ from run_agent import AIAgent from gateway.run import ( _current_max_iterations, _resolve_runtime_agent_kwargs, _resolve_gateway_model, _load_gateway_config, GatewayRunner, ) from hermes_cli.tools_config import _get_platform_tools runtime_kwargs = _resolve_runtime_agent_kwargs() reasoning_config = GatewayRunner._load_reasoning_config() model = _resolve_gateway_model() # When the primary provider's auth fails (expired token / 429 quota # cap), _resolve_runtime_agent_kwargs() falls through to the fallback # provider chain, whose runtime dict carries its own ``model`` key. # Pop it and let it override the config model, mirroring the native # gateway path (_resolve_session_agent_runtime in run.py). Otherwise # the explicit ``model=model`` below collides with the ``**runtime_kwargs`` # spread → "got multiple values for keyword argument 'model'", 500ing # every /v1/chat/completions request while a fallback is active. runtime_model = runtime_kwargs.pop("model", None) if runtime_model: model = runtime_model # Per-client model routing (model_routes config). The route was # resolved from the request's ``model`` field by the HTTP handler. # Precedence (highest first): session ``/model`` override → model_routes # route → global config — an explicit user-issued ``/model`` on the # session always beats static per-client route config. session_override = self._session_model_override_for( gateway_session_key or session_id ) if route and not session_override: if route.get("provider"): # Resolve real credentials for the routed provider (mirrors # the channel_overrides path in gateway/run.py) so a route # without an explicit api_key/base_url still gets the right # provider auth instead of the default provider's key. try: from gateway.run import _resolve_runtime_agent_kwargs_for_provider provider_kwargs = _resolve_runtime_agent_kwargs_for_provider( route["provider"] ) provider_kwargs.pop("model", None) runtime_kwargs.update(provider_kwargs) except Exception: # Fall back to just switching the provider name; explicit # per-route api_key/base_url below can still complete auth. runtime_kwargs["provider"] = route["provider"] if route.get("model"): model = route["model"] # Per-route secrets are upstream provider credentials. Never log # them (compare _check_auth: caller auth stays the global bearer # key checked with hmac.compare_digest). if route.get("api_key"): runtime_kwargs["api_key"] = route["api_key"] if route.get("base_url"): runtime_kwargs["base_url"] = route["base_url"] logger.debug( "api_server model route applied: model=%s provider=%s", model, runtime_kwargs.get("provider"), ) elif route and session_override: logger.debug( "api_server model route skipped: session /model override wins for %s", gateway_session_key or session_id, ) user_config = _load_gateway_config() enabled_toolsets = sorted(_get_platform_tools(user_config, "api_server")) max_iterations = _current_max_iterations() # Load fallback provider chain so the API server platform has the # same fallback behaviour as Telegram/Discord/Slack (fixes #4954). fallback_model = GatewayRunner._load_fallback_model() agent = AIAgent( model=model, **runtime_kwargs, max_iterations=max_iterations, quiet_mode=True, verbose_logging=False, ephemeral_system_prompt=ephemeral_system_prompt or None, enabled_toolsets=enabled_toolsets, session_id=session_id, platform="api_server", stream_delta_callback=stream_delta_callback, tool_progress_callback=tool_progress_callback, tool_start_callback=tool_start_callback, tool_complete_callback=tool_complete_callback, session_db=self._ensure_session_db(), fallback_model=fallback_model, reasoning_config=reasoning_config, gateway_session_key=gateway_session_key, ) return agent # ------------------------------------------------------------------ # HTTP Handlers # ------------------------------------------------------------------ async def _handle_health(self, request: "web.Request") -> "web.Response": """GET /health — simple health check.""" return web.json_response( {"status": "ok", "platform": "hermes-agent", "version": _hermes_version()} ) async def _handle_health_detailed(self, request: "web.Request") -> "web.Response": """GET /health/detailed — rich status for cross-container dashboard probing. Returns gateway state, connected platforms, PID, and uptime so the dashboard can display full status without needing a shared PID file or /proc access. Requires the same Bearer auth as other API routes. """ auth_err = self._check_auth(request) if auth_err: return auth_err from gateway.status import ( derive_gateway_busy, derive_gateway_drainable, parse_active_agents, read_runtime_status, ) runtime = read_runtime_status() or {} gw_state = runtime.get("gateway_state") gw_active = parse_active_agents(runtime.get("active_agents", 0)) # This endpoint is served BY the gateway process, so it is by definition # alive — gateway_running is True. Derive busy/drainable from the same # shared contract /api/status uses so the two surfaces never disagree. active_api_runs, process_depth, active_delegations = self._readiness_work_counts() from gateway.run import _resolve_gateway_model readiness = collect_runtime_readiness( configured_model=_resolve_gateway_model(), runtime_status=runtime, active_api_runs=active_api_runs, process_completion_queue_depth=process_depth, active_delegations=active_delegations, ) return web.json_response({ "status": readiness["status"], "readiness": readiness, "platform": "hermes-agent", "version": _hermes_version(), "gateway_state": gw_state, "platforms": runtime.get("platforms", {}), "active_agents": gw_active, "gateway_busy": derive_gateway_busy( gateway_running=True, gateway_state=gw_state, active_agents=gw_active, ), "gateway_drainable": derive_gateway_drainable( gateway_running=True, gateway_state=gw_state, ), "exit_reason": runtime.get("exit_reason"), "updated_at": runtime.get("updated_at"), "pid": os.getpid(), }) @staticmethod def _audio_extension_for_content_type(content_type: str) -> str: """Map an inbound audio Content-Type to a safe cache-file suffix.""" media_type = (content_type or "").split(";", 1)[0].strip().lower() return _ESP32_AUDIO_EXTENSIONS.get(media_type, ".wav") @staticmethod def _audio_response_mime(path: str) -> str: """Return an HTTP Content-Type for a generated audio file path.""" suffix = Path(path).suffix.lower() return _AUDIO_RESPONSE_MIME_BY_SUFFIX.get(suffix, "application/octet-stream") @staticmethod def _convert_audio_to_esp32_wav(input_path: str, *, output_dir: Path, device_id: str) -> Path: """Normalize generated TTS audio to ESP32-friendly WAV. Hermes TTS providers commonly emit MP3 (Edge) or OGG. The ESP32 test client expects a small, predictable RIFF/WAVE file, so decode with ffmpeg to raw PCM and rewrite the WAV container with Python's ``wave`` module. That produces a minimal ``fmt`` + ``data`` WAV: mono, 16 kHz, signed 16-bit little-endian PCM. """ import subprocess import wave safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" output_dir.mkdir(parents=True, exist_ok=True) raw_path = output_dir / f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}.s16le" wav_path = raw_path.with_suffix(".wav") cmd = [ "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-i", input_path, "-ac", "1", "-ar", "16000", "-f", "s16le", str(raw_path), ] try: subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) raw = raw_path.read_bytes() with wave.open(str(wav_path), "wb") as wav: wav.setnchannels(1) wav.setsampwidth(2) wav.setframerate(16000) wav.writeframes(raw) return wav_path finally: try: raw_path.unlink(missing_ok=True) except Exception: pass @staticmethod def _safe_audio_header(value: Any, max_length: int) -> str: """Make diagnostic text safe for HTTP response headers.""" text = str(value or "")[:max_length] return text.replace("\r", " ").replace("\n", " ").replace("\x00", " ") @staticmethod def _esp32_reply_mode(request: "web.Request") -> str: """Return 'sync' or 'ack' for the ESP32 voice response contract.""" value = ( request.headers.get("X-Hermes-Reply-Mode") or request.query.get("reply_mode") or request.query.get("mode") or "sync" ) value = str(value).strip().lower() if value in {"ack", "async", "background", "quick_ack", "quick-ack"}: return "ack" return "sync" @staticmethod def _looks_like_http_url(value: str) -> bool: try: parsed = urlparse(value) return parsed.scheme in {"http", "https"} and bool(parsed.netloc) except Exception: return False @staticmethod def _load_local_device_url(device_name: str) -> Optional[str]: """Best-effort local MCP screen lookup from ~/.hermes/local_devices.yaml.""" if not device_name: return None try: import yaml registry = Path.home() / ".hermes" / "local_devices.yaml" data = yaml.safe_load(registry.read_text()) if registry.exists() else {} aliases = data.get("aliases") if isinstance(data, dict) else None devices = data.get("devices") if isinstance(data, dict) else None names = [str(device_name).strip(), str(device_name).strip().replace("-", "_")] seen: set[str] = set() while names: name = names.pop(0) if not name or name in seen: continue seen.add(name) if isinstance(aliases, dict): alias = aliases.get(name) if isinstance(alias, str) and alias: if alias.startswith("http"): return alias names.append(alias) info = devices.get(name) if isinstance(devices, dict) else None if isinstance(info, dict): return str(info.get("fallback_url") or info.get("url") or "") or None except Exception: return None return None @staticmethod def _load_local_device_url_by_ip(source_ip: str) -> Optional[str]: """Best-effort local MCP screen lookup by observed device IP. Some MCP-capable devices, notably the iPhone companion app, do not run their MCP endpoint on port 80. When the voice gateway sees traffic from one of those devices, prefer the registry URL over synthesizing http:///api/mcp. """ value = str(source_ip or "").strip() if not value: return None if value.startswith("::ffff:"): value = value.removeprefix("::ffff:") try: import yaml registry = Path.home() / ".hermes" / "local_devices.yaml" data = yaml.safe_load(registry.read_text()) if registry.exists() else {} devices = data.get("devices") if isinstance(data, dict) else None if not isinstance(devices, dict): return None for info in devices.values(): if not isinstance(info, dict): continue candidates = { str(info.get("reserved_ip") or "").strip(), str(info.get("host") or "").strip(), } for url_key in ("fallback_url", "url"): raw_url = str(info.get(url_key) or "").strip() if raw_url: parsed = urlparse(raw_url) if parsed.hostname: candidates.add(parsed.hostname) if value in candidates: return str(info.get("fallback_url") or info.get("url") or "") or None except Exception: return None return None @staticmethod def _esp32_screen_url_from_source_ip(source_ip: str) -> Optional[str]: """Build the MCP endpoint for the ESP32 that made this request. ESP32 boards can move between DHCP leases. For voice ACK follow-ups the initiating board is the safest target, so prefer its observed peer IP instead of a possibly stale local-device registry entry. """ value = str(source_ip or "").strip() if not value: return None if value.startswith("::ffff:"): value = value.removeprefix("::ffff:") # Do not route to loopback/proxy/invalid addresses; registry fallback is # better in those cases. try: ip = ipaddress.ip_address(value) except ValueError: return None if ip.is_loopback or ip.is_unspecified or ip.is_multicast: return None host = f"[{value}]" if ip.version == 6 else value return f"http://{host}/api/mcp" def _resolve_esp32_screen_url(self, request: "web.Request", device_id: str) -> Optional[str]: """Resolve the screen MCP endpoint associated with this voice turn.""" explicit = ( request.headers.get("X-Hermes-Screen-Url") or request.headers.get("X-Screen-MCP-Url") or request.query.get("screen_url") ) if explicit and self._looks_like_http_url(explicit.strip()): return explicit.strip() if str(device_id or "").strip().lower().startswith("iphone"): iphone_url = self._load_local_device_url(str(device_id).strip()) if iphone_url and self._looks_like_http_url(iphone_url): return iphone_url audit_ctx = self._request_audit_context(request) for source_ip in (audit_ctx.get("peer_ip"), audit_ctx.get("remote")): registry_url = self._load_local_device_url_by_ip(source_ip or "") if registry_url and self._looks_like_http_url(registry_url): return registry_url audit_ctx = self._request_audit_context(request) for source_ip in (audit_ctx.get("peer_ip"), audit_ctx.get("remote")): source_url = self._esp32_screen_url_from_source_ip(source_ip or "") if source_url: return source_url screen_device = ( request.headers.get("X-Hermes-Screen-Device") or request.query.get("screen_device") or device_id ) candidates = [screen_device, "little32", "esp32_screen"] for candidate in candidates: url = self._load_local_device_url(str(candidate).strip()) if url and self._looks_like_http_url(url): return url return None async def _call_screen_mcp( self, screen_url: Optional[str], tool_name: str, arguments: Dict[str, Any], *, timeout: float = 8.0, ) -> bool: """Call a simple HTTP JSON-RPC MCP screen tool directly.""" if not screen_url: return False try: from aiohttp import ClientSession, ClientTimeout payload = { "jsonrpc": "2.0", "id": uuid.uuid4().hex[:12], "method": "tools/call", "params": {"name": tool_name, "arguments": arguments}, } async with ClientSession(timeout=ClientTimeout(total=timeout)) as session: async with session.post(screen_url, json=payload) as resp: body = await resp.text() if resp.status >= 400: logger.warning( "[api_server] screen MCP %s failed: HTTP %s %s", tool_name, resp.status, body[:300], ) return False try: data = json.loads(body) if body else {} if isinstance(data, dict) and data.get("error"): logger.warning("[api_server] screen MCP %s returned error: %s", tool_name, data.get("error")) return False except Exception: pass return True except Exception as exc: logger.warning("[api_server] screen MCP %s failed: %s", tool_name, exc) return False @staticmethod def _screen_stream_target(screen_url: Optional[str]) -> Optional[tuple[str, int]]: """Return the raw UDP video-stream target for MCP screens that expose one.""" if not screen_url: return None try: parsed = urlparse(screen_url) host = parsed.hostname if not host: return None return (host, 8082) except Exception: return None @staticmethod def _voice_animation_udp_frames(state: str, text: str = "", frames: int = 18) -> List[bytes]: """Render small Hermes voice-state animations as RLCD-compatible frames.""" from PIL import Image, ImageDraw import math width, height = 400, 300 def to_rlcd(img: "Image.Image") -> bytes: img_1bit = img.convert("1", dither=Image.Dither.FLOYDSTEINBERG) px = img_1bit.load() buf = bytearray(15000) for y in range(height): for x in range(width): if px[x, y]: inv_y = 299 - y bx = x // 2 by = inv_y // 4 idx = bx * 75 + by lx = x % 2 ly = inv_y % 4 bit = 7 - (ly * 2 + lx) buf[idx] |= (1 << bit) return bytes(buf) label_by_state = { "ack": "Got it", "captioning": "Listening", "waiting": "Thinking", "speaking": "Replying", "success": "Done", "error": "Try again", } label = label_by_state.get(str(state), str(state or "Hermes").title()) snippet = " ".join(str(text or "").split())[:42] rendered: List[bytes] = [] for frame in range(max(1, int(frames))): img = Image.new("1", (width, height), 0) draw = ImageDraw.Draw(img) cx, cy = width // 2, height // 2 phase = frame * 0.35 if state == "captioning": draw.rectangle((188, 38, 212, 75), outline=1) draw.arc((175, 55, 225, 105), 20, 160, fill=1) for x in range(42, width - 42, 5): y = cy + int(32 * math.sin(x * 0.045 + phase * 2)) y2 = cy + int(14 * math.sin(x * 0.085 - phase * 3)) draw.line((x, y, min(width - 1, x + 4), y2), fill=1) elif state in {"waiting", "speaking"}: for radius in (34, 64, 94): pulse = radius + int(8 * math.sin(phase + radius)) draw.ellipse((cx - pulse, cy - pulse, cx + pulse, cy + pulse), outline=1) for i in range(7): x = 93 + i * 34 bar = int(18 + 42 * (0.5 + 0.5 * math.sin(phase * 2 + i))) draw.rectangle((x, 228 - bar, x + 17, 228), fill=1) elif state == "error": draw.line((150, 88, 250, 188), fill=1) draw.line((250, 88, 150, 188), fill=1) draw.rectangle((145, 82, 255, 194), outline=1) else: r = 42 + int(16 * math.sin(phase)) draw.ellipse((cx - r, cy - r, cx + r, cy + r), outline=1) draw.ellipse((cx - 8, cy - 8, cx + 8, cy + 8), fill=1) draw.text((max(8, (width - len(label) * 6) // 2), 24), label, fill=1) if snippet: draw.text((max(8, (width - len(snippet) * 6) // 2), 260), snippet, fill=1) rendered.append(to_rlcd(img)) return rendered async def _send_stream_animation( self, screen_url: Optional[str], state: str, *, text: str = "", frames: Optional[int] = None, ) -> bool: """Send a server-rendered voice animation over the screen streaming socket.""" target = self._screen_stream_target(screen_url) if not target: return False frame_count = frames if frames is not None else (12 if state == "ack" else 18) try: rendered = await asyncio.to_thread(self._voice_animation_udp_frames, state, text, frame_count) sock = _socket.socket(_socket.AF_INET, _socket.SOCK_DGRAM) try: for frame_id, frame in enumerate(rendered): fid = frame_id % 256 for chunk_idx in range(15): start = chunk_idx * 1000 sock.sendto(bytes([fid, chunk_idx]) + frame[start : start + 1000], target) await asyncio.sleep(0.045) finally: sock.close() return True except Exception as exc: logger.warning("[api_server] screen stream animation failed: %s", exc) return False async def _send_screen_animation( self, screen_url: Optional[str], state: str, *, text: str = "", frames: Optional[int] = None, ) -> bool: """Upload and run the ESP32 voice-state animation script on the initiating screen.""" if not screen_url: return False if await self._send_stream_animation(screen_url, state, text=text, frames=frames): return True if screen_url not in self._esp32_voice_animation_uploaded: uploaded = await self._call_screen_mcp( screen_url, "write_file", {"path": ESP32_VOICE_ANIMATION_PATH, "content": ESP32_VOICE_ANIMATION_SCRIPT}, ) if not uploaded: return False self._esp32_voice_animation_uploaded.add(screen_url) code = ( f"exec(open({ESP32_VOICE_ANIMATION_PATH!r}).read()); " f"run_state({str(state)!r}, text={str(text or '')[:240]!r}, frames={frames!r})" ) return await self._call_screen_mcp(screen_url, "execute_python", {"code": code}) async def _send_screen_text(self, screen_url: Optional[str], text: str, *, clear: bool = False) -> None: """Best-effort text update to the initiating MCP screen.""" if not screen_url: return if clear: await self._call_screen_mcp(screen_url, "clear_screen", {"color": 0}) # Most Reyna screens accept x/y/text. Extra wrapping is avoided here so # firmware-specific clients can keep their own font/layout behavior. await self._call_screen_mcp(screen_url, "draw_text", {"x": 8, "y": 8, "text": text[:900]}) @staticmethod def _extract_first_image_reference(text: str) -> Optional[str]: """Return the first local path, MEDIA path, data URL, or HTTP image URL in text.""" if not text: return None patterns = [ r"MEDIA:([^\s)]+)", r"!\[[^\]]*\]\(([^)]+)\)", r"(?Phttps?://[^\s)]+\.(?:png|jpe?g|webp|gif)(?:\?[^\s)]*)?)", r"(?P/[^\s)]+\.(?:png|jpe?g|webp|gif|bmp))", r"(?Pdata:image/[^\s)]+)", ] for pattern in patterns: for match in re.finditer(pattern, text, flags=re.IGNORECASE): ref = match.groupdict().get("url") or match.groupdict().get("path") or match.groupdict().get("data") or match.group(1) if ref: return ref.strip().strip('"\'') return None async def _send_screen_image_response(self, screen_url: Optional[str], text: str) -> bool: """Best-effort render of the first generated/linked image onto the ESP32 screen.""" if not screen_url: return False image_ref = self._extract_first_image_reference(text) if not image_ref: return False try: import base64 import io from PIL import Image, ImageOps raw: bytes lowered = image_ref.lower() if lowered.startswith("data:image/"): _, _, encoded = image_ref.partition(",") raw = base64.b64decode(encoded) elif self._looks_like_http_url(image_ref): from aiohttp import ClientSession, ClientTimeout async with ClientSession(timeout=ClientTimeout(total=20)) as session: async with session.get(image_ref) as resp: if resp.status >= 400: logger.warning("[api_server] screen image fetch failed: HTTP %s", resp.status) return False raw = await resp.read() else: path = Path(image_ref).expanduser() if not path.exists() or not path.is_file(): return False raw = await asyncio.to_thread(path.read_bytes) if not raw: return False def render_pbm() -> str: with Image.open(io.BytesIO(raw)) as img: img = ImageOps.exif_transpose(img).convert("RGB") canvas = Image.new("RGB", (320, 240), "white") img.thumbnail((320, 240), Image.Resampling.LANCZOS) x = (320 - img.width) // 2 y = (240 - img.height) // 2 canvas.paste(img, (x, y)) mono = canvas.convert("1", dither=Image.Dither.FLOYDSTEINBERG) out = io.BytesIO() mono.save(out, format="PPM") # mode=1 writes raw PBM (P4) return base64.b64encode(out.getvalue()).decode("ascii") pbm_base64 = await asyncio.to_thread(render_pbm) return await self._call_screen_mcp( screen_url, "draw_image", {"pbm_base64": pbm_base64, "x": 0, "y": 0, "dither": False}, timeout=15, ) except Exception: logger.exception("[api_server] failed to render ESP32 screen image response") return False async def _send_screen_audio_response( self, screen_url: Optional[str], text: str, *, audio_dir: Path, device_id: str, volume: int = 80, ) -> bool: """Best-effort spoken follow-up for ACK-mode ESP32 voice turns.""" if not screen_url or not text.strip(): return False try: import base64 from tools.tts_tool import text_to_speech_tool # Keep board playback short and payloads small. The ESP32 HTTP MCP # endpoint resets on large base64 WAV bodies; MP3 is much smaller # and the firmware exposes play_mp3_base64 for exactly this path. spoken_text = self._short_voice_summary(text, 420) tts_raw = await asyncio.to_thread(text_to_speech_tool, spoken_text) tts_result = json.loads(tts_raw) src = str(tts_result.get("file_path") or "") if not tts_result.get("success") or not src or not Path(src).exists(): logger.warning( "[api_server] screen audio TTS failed: %s", tts_result.get("error") or "missing TTS output", ) return False src_path = Path(src) # Detect if this is iPhone (has no play_mp3_base64, only play_audio_base64) is_iphone = "iphone" in str(device_id).lower() or ":8080" in str(screen_url or "") if not is_iphone and src_path.suffix.lower() == ".mp3": mp3_base64 = base64.b64encode(src_path.read_bytes()).decode("ascii") if await self._call_screen_mcp( screen_url, "play_mp3_base64", {"mp3_base64": mp3_base64, "volume": volume}, timeout=75, ): return True wav_path = await asyncio.to_thread( self._convert_audio_to_esp32_wav, src, output_dir=audio_dir, device_id=f"{device_id}_response", ) wav_bytes = wav_path.read_bytes() # iPhone can handle larger WAVs than ESP32 (tested 180k+), increase limit for iPhone max_wav = 600_000 if is_iphone else 220_000 if len(wav_bytes) > max_wav: # For iPhone, truncate to first 6 seconds to fit if is_iphone: import wave, io try: with wave.open(str(wav_path), "rb") as wf: fr = wf.getframerate() nframes = int(fr * 6.0) wf.rewind() frames = wf.readframes(nframes) bio = io.BytesIO() with wave.open(bio, "wb") as out: out.setnchannels(wf.getnchannels()) out.setsampwidth(wf.getsampwidth()) out.setframerate(fr) out.writeframes(frames) wav_bytes = bio.getvalue() except Exception: pass else: logger.warning( "[api_server] screen audio WAV too large for MCP payload: %s bytes", len(wav_bytes), ) return False wav_base64 = base64.b64encode(wav_bytes).decode("ascii") return await self._call_screen_mcp( screen_url, "play_audio_base64", {"wav_base64": wav_base64, "volume": volume}, timeout=45, ) except Exception: logger.exception("[api_server] failed to play ESP32 screen audio response") return False @staticmethod def _short_voice_summary(text: str, limit: int = 220) -> str: """Local fallback summary for voice/screen snippets without another LLM call.""" clean = " ".join(str(text or "").split()) if len(clean) <= limit: return clean return clean[: max(0, limit - 1)].rstrip() + "…" async def _esp32_ack_wav(self, *, audio_dir: Path, device_id: str) -> Path: """Return a reusable quiet ACK WAV; the visual screen animation is the acknowledgement.""" ack_path = audio_dir / "quick_ack.wav" if ack_path.exists(): return ack_path # ACK-mode clients expect a valid audio/wav response immediately, but # Reyna ESP32 screens now show the acknowledgement visually. Keep the # HTTP contract while avoiding the old spoken "Got it" message. import wave sample_rate = 16000 duration_s = 0.08 frames = b"\x00\x00" * int(sample_rate * duration_s) with wave.open(str(ack_path), "wb") as wav: wav.setnchannels(1) wav.setsampwidth(2) wav.setframerate(sample_rate) wav.writeframes(frames) return ack_path def _append_esp32_voice_audit(self, audio_dir: Path, event: Dict[str, Any]) -> None: """Append a best-effort ESP32 voice request audit event.""" try: log_path = audio_dir / "voice_requests.jsonl" with log_path.open("a", encoding="utf-8") as handle: handle.write(json.dumps(event, default=str) + "\n") except Exception: logger.debug("[api_server] failed to append ESP32 voice audit log", exc_info=True) async def _run_esp32_voice_audio_background( self, *, audio_path: Path, audit_event: Dict[str, Any], device_id: str, instructions: Optional[str], audio_dir: Path, screen_url: Optional[str], ) -> None: """Transcribe uploaded ESP32 audio and continue the voice turn after an immediate ACK.""" try: await self._send_screen_animation(screen_url, "captioning") from tools.transcription_tools import transcribe_audio stt_result = await asyncio.to_thread(transcribe_audio, str(audio_path)) if not stt_result.get("success"): failed_event = dict(audit_event) failed_event.update({ "status": "transcription_failed", "error": stt_result.get("error") or "Transcription failed", "timestamp": time.time(), }) self._append_esp32_voice_audit(audio_dir, failed_event) await self._send_screen_text( screen_url, "I heard audio, but transcription failed. Try again.", clear=True, ) return transcript = str(stt_result.get("transcript") or "").strip() if not transcript: empty_event = dict(audit_event) empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) self._append_esp32_voice_audit(audio_dir, empty_event) await self._send_screen_text( screen_url, "I heard audio, but got no words. Try again.", clear=True, ) return transcript_event = dict(audit_event) transcript_event.update({"status": "transcribed", "transcript": transcript, "timestamp": time.time()}) self._append_esp32_voice_audit(audio_dir, transcript_event) await self._run_esp32_voice_background( transcript=transcript, device_id=device_id, instructions=instructions, audio_dir=audio_dir, screen_url=screen_url, ) except Exception as exc: logger.exception("[api_server] ESP32 ACK background voice processing failed") failed_event = dict(audit_event) failed_event.update({"status": "background_exception", "error": str(exc), "timestamp": time.time()}) self._append_esp32_voice_audit(audio_dir, failed_event) await self._send_screen_text( screen_url, "I got the audio, but background processing failed.", clear=True, ) async def _run_esp32_voice_background( self, *, transcript: str, device_id: str, instructions: Optional[str], audio_dir: Path, screen_url: Optional[str], ) -> None: """Run the slow ESP32 voice turn after the device already heard an ACK.""" try: animation_ok = await self._send_screen_animation(screen_url, "waiting", text=transcript) if not animation_ok: await self._send_screen_text( screen_url, f"Heard: {self._short_voice_summary(transcript, 180)}\n\nWorking…", clear=True, ) screen_hint = ( "\n\nThis request came from an ESP32 voice/screen device. " "Keep the spoken follow-up short. The gateway will display the full answer on the initiating screen when possible." ) merged_instructions = (instructions or "") + screen_hint response_data, final_text = await self._run_esp32_voice_turn( transcript=transcript, device_id=device_id, instructions=merged_instructions, ) if not final_text.strip(): final_text = self._extract_response_text(response_data) or "I finished, but I do not have a text answer." display_text = final_text if len(display_text) > 1200: display_text = self._short_voice_summary(display_text, 900) + "\n\nFull answer is long; WhatsApp delivery is the next wiring step." await self._send_screen_text(screen_url, display_text, clear=True) await self._send_screen_image_response(screen_url, final_text) await self._send_screen_audio_response( screen_url, final_text, audio_dir=audio_dir, device_id=device_id, ) except Exception: logger.exception("[api_server] ESP32 background voice turn failed") await self._send_screen_text(screen_url, "Hermes hit an error while preparing the full answer.", clear=True) @staticmethod def _extract_response_text(response_data: Dict[str, Any]) -> str: """Extract assistant text from an OpenAI Responses-shaped payload.""" chunks: List[str] = [] for item in response_data.get("output") or []: if not isinstance(item, dict) or item.get("type") != "message": continue for part in item.get("content") or []: if isinstance(part, dict) and part.get("type") in {"output_text", "text"}: text = part.get("text") if isinstance(text, str) and text: chunks.append(text) return "\n".join(chunks).strip() async def _run_esp32_voice_turn( self, *, transcript: str, device_id: str, instructions: Optional[str] = None, ) -> tuple[Dict[str, Any], str]: """Run one persistent Responses-style turn for an ESP32 voice device.""" conversation = f"esp32:{device_id}" previous_response_id = self._response_store.get_conversation(conversation) conversation_history: List[Dict[str, Any]] = [] stored_session_id = None stored_instructions = None if previous_response_id: stored = self._response_store.get(previous_response_id) if stored is not None: conversation_history = list(stored.get("conversation_history", [])) stored_session_id = stored.get("session_id") stored_instructions = stored.get("instructions") if instructions is None: instructions = stored_instructions session_id = stored_session_id or str(uuid.uuid4()) user_message = ( f'[Voice input from ESP32 device "{device_id}"]\n' f'{transcript}' ) result, usage = await self._run_agent( user_message=user_message, conversation_history=conversation_history, ephemeral_system_prompt=instructions, session_id=session_id, gateway_session_key=conversation, ) final_response = result.get("final_response", "") or result.get("error", "") or "(No response generated)" response_id = f"resp_{uuid.uuid4().hex[:28]}" created_at = int(time.time()) full_history = self._build_response_conversation_history( conversation_history, user_message, result, final_response, ) output_start_index = self._response_messages_turn_start_index( conversation_history, user_message, result, ) output_items = self._extract_output_items(result, start_index=output_start_index) response_data = { "id": response_id, "object": "response", "status": "completed", "created_at": created_at, "model": self._model_name, "output": output_items, "usage": { "input_tokens": usage.get("input_tokens", 0), "output_tokens": usage.get("output_tokens", 0), "total_tokens": usage.get("total_tokens", 0), }, } self._response_store.put(response_id, { "response": response_data, "conversation_history": full_history, "instructions": instructions, "session_id": session_id, }) self._response_store.set_conversation(conversation, response_id) return response_data, final_response @staticmethod def _esp32_audio_cache_dir() -> Path: """Return the cache directory for ESP32 voice artifacts.""" try: from hermes_constants import get_hermes_dir return get_hermes_dir("cache/audio", "audio_cache") / "esp32" except Exception: return Path.home() / ".hermes" / "cache" / "audio" / "esp32" @staticmethod def _write_pcm_s16le_wav(raw_path: Path, wav_path: Path, *, sample_rate: int, channels: int = 1) -> None: """Wrap a raw PCM s16le file in a minimal RIFF/WAVE container.""" import wave with wave.open(str(wav_path), "wb") as wav: wav.setnchannels(channels) wav.setsampwidth(2) wav.setframerate(sample_rate) with raw_path.open("rb") as raw_handle: while True: chunk = raw_handle.read(64 * 1024) if not chunk: break wav.writeframes(chunk) # --- Live draft helpers (from your whisper-translation/apple_speech pipe protocol) --- def _load_macmini_mcp_config(self) -> dict: try: if ESP32_MACMINI_MCP_URL and ESP32_MACMINI_MCP_TOKEN: return {"url": ESP32_MACMINI_MCP_URL, "token": ESP32_MACMINI_MCP_TOKEN} cfg_path = Path.home() / ".hermes" / "config.yaml" if cfg_path.exists(): import yaml cfg = yaml.safe_load(cfg_path.read_text()) or {} m = (cfg.get("mcp_servers") or {}).get("macmini") or {} url = m.get("url") or "" token = "" headers = m.get("headers") or {} auth = headers.get("Authorization") or "" if auth.startswith("Bearer "): token = auth[7:] if url: return {"url": url, "token": token} except Exception: pass return {} @staticmethod def _s16le_to_wav_bytes(pcm_bytes: bytes, sample_rate: int = 16000, channels: int = 1) -> bytes: import io, wave bio = io.BytesIO() with wave.open(bio, "wb") as wf: wf.setnchannels(channels) wf.setsampwidth(2) wf.setframerate(sample_rate) wf.writeframes(pcm_bytes) return bio.getvalue() async def _transcribe_draft_via_macmini(self, wav_bytes: bytes, locale: str = "en-US") -> str | None: if not ESP32_LIVE_DRAFT_USE_MACMINI: return None cfg = self._load_macmini_mcp_config() url = cfg.get("url") token = cfg.get("token") if not url: return None try: import base64 b64 = base64.b64encode(wav_bytes).decode("ascii") # First try new live pipe tool for tool_name in ["speech_live_transcribe", "speech_transcribe_file"]: if tool_name == "speech_transcribe_file": # file tool expects filePath on Mac, not base64 - can't use unless we upload, skip for draft continue payload = { "jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": tool_name, "arguments": {"audioBase64": b64, "locale": locale} if tool_name=="speech_live_transcribe" else {"filePath": "/tmp/dummy.wav", "locale": locale}}, } headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"} if token: headers["Authorization"] = f"Bearer {token}" import aiohttp async with aiohttp.ClientSession() as session: async with session.post(url, json=payload, headers=headers, timeout=aiohttp.ClientTimeout(total=8)) as resp: txt = await resp.text() if resp.status != 200: logger.debug(f"[api_server] macmini {tool_name} HTTP {resp.status}: {txt[:500]}") continue for line in txt.splitlines(): if not line.startswith("data: "): continue try: data = json.loads(line[6:]) if data.get("result", {}).get("isError"): # log error to help debugging err_txt = (data.get("result", {}).get("content") or [{}])[0].get("text","")[:500] if "not found" in err_txt.lower(): logger.warning(f"[api_server] macmini tool {tool_name} not found — need git pull on macmini (have {err_txt[:100]})") else: logger.debug(f"[api_server] macmini {tool_name} error: {err_txt}") continue content = (data.get("result", {}).get("content") or [{}])[0].get("text", "") or "" if not content: continue inner = json.loads(content) t = inner.get("text") or "" if t: return t.strip() except Exception as ex: logger.debug(f"[api_server] macmini parse err: {ex}") continue except Exception as e: logger.debug(f"[api_server] macmini live draft exception: {e}") return None async def _transcribe_draft_local(self, wav_bytes: bytes) -> str | None: try: import tempfile fd, tmp_path = tempfile.mkstemp(suffix=".wav") os.close(fd) Path(tmp_path).write_bytes(wav_bytes) def _do(): try: from faster_whisper import WhisperModel model = None try: from tools.transcription_tools import _local_model if _local_model is not None: model = _local_model except Exception: pass if model is None: model = WhisperModel("tiny", device="cpu", compute_type="int8", num_workers=1) segs, _ = model.transcribe(tmp_path, beam_size=1, best_of=1, language="en", condition_on_previous_text=False, vad_filter=True) return " ".join(s.text.strip() for s in segs).strip() except Exception as ex: logger.debug(f"[api_server] local draft err: {ex}") return None finally: try: Path(tmp_path).unlink(missing_ok=True) except Exception: pass return await asyncio.to_thread(_do) except Exception: return None async def _apple_llm_quick_reply_via_macmini(self, draft: str, context: str = "", instructions: str = "") -> str | None: """Call Mac mini apple_llm_quick_reply for instant smart ACK from draft.""" if not ESP32_LIVE_DRAFT_USE_MACMINI: return None cfg = self._load_macmini_mcp_config() url = cfg.get("url") token = cfg.get("token") if not url: return None try: payload = { "jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "apple_llm_quick_reply", "arguments": {"draft": draft, "context": context or "", "instructions": instructions or ""}}, } headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"} if token: headers["Authorization"] = f"Bearer {token}" import aiohttp async with aiohttp.ClientSession() as session: async with session.post(url, json=payload, headers=headers, timeout=aiohttp.ClientTimeout(total=4)) as resp: txt = await resp.text() if resp.status != 200: logger.debug(f"[api_server] macmini quick_reply HTTP {resp.status}: {txt[:500]}") return None for line in txt.splitlines(): if not line.startswith("data: "): continue try: data = json.loads(line[6:]) if data.get("result", {}).get("isError"): err_txt = (data.get("result", {}).get("content") or [{}])[0].get("text","")[:500] if "not found" in err_txt.lower(): logger.warning(f"[api_server] macmini apple_llm_quick_reply not found — need git pull + npm restart on mac_mini (192.168.68.102). Have: {err_txt[:120]}") else: logger.debug(f"[api_server] quick_reply error: {err_txt}") continue content = (data.get("result", {}).get("content") or [{}])[0].get("text", "") or "" if not content: continue inner = json.loads(content) t = inner.get("text") or "" if t: return t.strip() except Exception as ex: logger.debug(f"[api_server] quick_reply parse err: {ex}") continue except Exception as e: logger.debug(f"[api_server] macmini quick_reply exception: {e}") return None async def _run_quick_reply_then_full(self, draft_text: str, final_transcript: str, device_id: str, screen_url, instructions: str | None, audio_dir): """New flow: when voice stops, we already have draft. Emit instant smart reply via Apple LLM, then full Hermes turn.""" # This is kept for future speculative pre-start; for now we just log — actual quick reply emitted in WS handler before background start pass async def _speak_instant_via_macmini(self, text: str, voice: str = "Alex", use_voicebox: bool = False) -> str | None: """Synthesize instant reply text via Mac mini voice for immediate audio while Hermes thinks. - Default: use Mac say Alex boy voice via speech_synthesize_base64 (fast <500ms, reliable) - If use_voicebox=True, use Voicebox Qwen Aiden boy voice via voicebox_quick_reply (higher quality, slower 2-4s when queue not stuck) Returns base64 wav if success, else None. """ cfg = self._load_macmini_mcp_config() url = cfg.get("url") token = cfg.get("token") if not url: return None import aiohttp, json headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"} if token: headers["Authorization"] = f"Bearer {token}" tool_name = "voicebox_quick_reply" if use_voicebox else "speech_synthesize_base64" args = {"text": text[:300]} if use_voicebox: args["profile"] = "Aiden" else: args["voice"] = voice try: payload = { "jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": tool_name, "arguments": args}, } async with aiohttp.ClientSession() as session: async with session.post(url, json=payload, headers=headers, timeout=aiohttp.ClientTimeout(total=10 if not use_voicebox else 20)) as resp: txt = await resp.text() if resp.status != 200: logger.debug(f"[api_server] macmini {tool_name} HTTP {resp.status}: {txt[:500]}") return None for line in txt.splitlines(): if not line.startswith("data: "): continue try: data = json.loads(line[6:]) if data.get("result", {}).get("isError"): err_txt = (data.get("result", {}).get("content") or [{}])[0].get("text","")[:500] logger.debug(f"[api_server] macmini {tool_name} error: {err_txt}") continue content = (data.get("result", {}).get("content") or [{}])[0].get("text", "") or "" if not content: continue inner = json.loads(content) b64 = inner.get("wavBase64") or inner.get("wav_base64") if b64: return b64 except Exception as ex: logger.debug(f"[api_server] {tool_name} parse err: {ex}") continue except Exception as e: logger.debug(f"[api_server] {tool_name} exception: {e}") return None async def _handle_esp32_voice_ws(self, request: "web.Request") -> "web.WebSocketResponse": """GET /api/esp32/voice/ws — ESP32 PCM streaming with LIVE draft captioning. LIVE DRAFT MODE (from whisper-translation/engine_apple_transcribe.py): - While ingesting binary s16le, every ~800ms snapshots current PCM -> in-mem WAV -> tries Mac mini speech_live_transcribe (ApplePipeTranscriber --pipe volatile, <200ms drafts) fallback to local tiny for fast feedback. - Emits WS events: draft + interim_transcript for UI. Final via macmini or local base. """ auth_err = self._check_auth(request) if auth_err: return auth_err # type: ignore[return-value] ws = web.WebSocketResponse(heartbeat=None, max_msg_size=512 * 1024) await ws.prepare(request) async def _send_event(event: str, **payload): if not ws.closed: await ws.send_str(json.dumps({"event": event, **payload})) device_id = ( request.headers.get("X-Device-ID") or request.query.get("device_id") or "default" ).strip()[:80] or "default" safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" audio_dir = self._esp32_audio_cache_dir() audio_dir.mkdir(parents=True, exist_ok=True) stamp = f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}" raw_path = audio_dir / f"{stamp}.s16le" wav_path = audio_dir / f"{stamp}.wav" sample_rate = 16000 channels = 1 sample_width = 2 total_audio_bytes = 0 started = False stopped = False first_audio_at: float | None = None started_at = time.time() # live draft state (directly from your whisper-translation ApplePipeTranscriber pattern) live_enabled = ESP32_LIVE_DRAFT_ENABLED pcm_buffer = bytearray() last_draft_at = 0.0 last_draft_bytes = 0 last_draft_text = "" draft_in_progress = False draft_count = 0 locale_req = request.headers.get("X-Hermes-Locale") or request.query.get("locale") or "en-US" audit_ctx = self._request_audit_context(request) audit_event = { "timestamp": started_at, "device_id": device_id, "source_ip": audit_ctx.get("remote") or audit_ctx.get("peer_ip") or "", "peer_ip": audit_ctx.get("peer_ip") or "", "forwarded_for": audit_ctx.get("forwarded_for") or "", "real_ip": audit_ctx.get("real_ip") or "", "user_agent": audit_ctx.get("user_agent") or "", "transport": "websocket", "raw_path": str(raw_path), "audio_path": str(wav_path), "status": "accepted", "live_draft_enabled": live_enabled, } self._append_esp32_voice_audit(audio_dir, audit_event) screen_url = self._resolve_esp32_screen_url(request, device_id) visual_listening_sent = False def _schedule_voice_state(state: str, *, text: str = "", frames: int | None = None) -> None: if not screen_url: return task = asyncio.create_task(self._send_screen_animation(screen_url, state, text=text, frames=frames)) try: self._background_tasks.add(task) except TypeError: pass if hasattr(task, "add_done_callback"): task.add_done_callback(self._background_tasks.discard) async def _do_live_draft(snapshot_bytes: bytes): nonlocal last_draft_text, draft_in_progress, draft_count if draft_in_progress: return if len(snapshot_bytes) < int(sample_rate * 0.5 * 2): # at least 500ms of s16le at 16k mono return draft_in_progress = True try: wav_b = self._s16le_to_wav_bytes(snapshot_bytes, sample_rate=sample_rate, channels=channels) txt = None if ESP32_LIVE_DRAFT_USE_MACMINI: txt = await self._transcribe_draft_via_macmini(wav_b, locale=locale_req) if not txt: txt = await self._transcribe_draft_local(wav_b) if not txt: return txt = txt.strip() if not txt or txt == last_draft_text or len(txt) < 2: return last_draft_text = txt draft_count += 1 await _send_event("draft", text=txt, is_final=False, isFinal=False, chunk=draft_count, bytes=len(snapshot_bytes)) await _send_event("interim_transcript", text=txt, is_final=False, draft_count=draft_count) if screen_url and len(txt) > 1: try: t = asyncio.create_task(self._send_screen_text(screen_url, txt, clear=False)) self._background_tasks.add(t) t.add_done_callback(self._background_tasks.discard) except Exception: pass finally: draft_in_progress = False await _send_event("ready", device_id=device_id, screen_url=screen_url or "", live_draft=live_enabled, live_model="apple-pipe-volatile") try: raw_handle = raw_path.open("wb") try: async for msg in ws: if msg.type == web.WSMsgType.TEXT: try: payload = json.loads(msg.data) except json.JSONDecodeError: await _send_event("error", code="bad_json", message="Text frames must be JSON") continue event = str(payload.get("event") or payload.get("type") or "").strip().lower() if event == "start": if started: await _send_event("error", code="already_started", message="Audio stream already started") continue fmt = str(payload.get("format") or "pcm_s16le").strip().lower() channels = int(payload.get("channels") or 1) sample_width = int(payload.get("sample_width") or 2) sample_rate = int(payload.get("sample_rate") or 16000) device_id = str(payload.get("device_id") or device_id).strip()[:80] or device_id if fmt not in {"pcm_s16le", "s16le", "pcm"} or channels != 1 or sample_width != 2: await _send_event("error", code="unsupported_audio_format", message="Use 16 kHz mono signed 16-bit little-endian PCM") await ws.close() return ws if sample_rate < 8000 or sample_rate > 48000: await _send_event("error", code="bad_sample_rate", message="sample_rate must be 8000-48000") await ws.close() return ws started = True if not visual_listening_sent: _schedule_voice_state("captioning", frames=8) visual_listening_sent = True await _send_event("listening", sample_rate=sample_rate, channels=channels, sample_width=sample_width) elif event in {"stop", "end"}: stopped = True break elif event in {"ping", "keepalive"}: await _send_event("pong") else: await _send_event("error", code="unknown_event", message=f"Unknown event: {event or ''}") elif msg.type == web.WSMsgType.BINARY: if not started: started = True if not visual_listening_sent: _schedule_voice_state("captioning", frames=8) visual_listening_sent = True await _send_event("listening", sample_rate=sample_rate, channels=channels, sample_width=sample_width) chunk = bytes(msg.data or b"") if not chunk: continue if first_audio_at is None: first_audio_at = time.time() total_audio_bytes += len(chunk) pcm_buffer.extend(chunk) if total_audio_bytes > ESP32_WS_MAX_AUDIO_BYTES: await _send_event("error", code="audio_too_large", message=f"Audio stream exceeded {ESP32_WS_MAX_AUDIO_BYTES} bytes") await ws.close() return ws raw_handle.write(chunk) # live draft trigger (every 800ms & 8KB new data) — direct captioning fluff stripped if live_enabled: now = time.time() elapsed_ms = (now - last_draft_at) * 1000 new_bytes = len(pcm_buffer) - last_draft_bytes total_ms = len(pcm_buffer) / 2 / sample_rate * 1000 if total_ms >= ESP32_LIVE_DRAFT_MIN_TOTAL_MS and elapsed_ms >= ESP32_LIVE_DRAFT_INTERVAL_MS and new_bytes >= ESP32_LIVE_DRAFT_MIN_BYTES: last_draft_at = now last_draft_bytes = len(pcm_buffer) # copy snapshot, fire-and-forget draft (but guard concurrent) snapshot = bytes(pcm_buffer) asyncio.create_task(_do_live_draft(snapshot)) elif msg.type == web.WSMsgType.ERROR: logger.warning("[api_server] ESP32 voice websocket error: %s", ws.exception()) break finally: raw_handle.close() if not stopped: await _send_event("error", code="missing_stop", message="Send a stop event after audio chunks") await ws.close() return ws if total_audio_bytes < ESP32_WS_MIN_AUDIO_BYTES: await _send_event("error", code="audio_too_short", message="Not enough audio received") await ws.close() return ws await asyncio.to_thread( self._write_pcm_s16le_wav, raw_path, wav_path, sample_rate=sample_rate, channels=channels, ) audit_event.update({ "status": "received", "audio_bytes": total_audio_bytes, "sample_rate": sample_rate, "first_audio_latency_ms": int(((first_audio_at or started_at) - started_at) * 1000), "stop_latency_ms": int((time.time() - started_at) * 1000), "draft_count": draft_count, "last_draft": last_draft_text, }) self._append_esp32_voice_audit(audio_dir, dict(audit_event)) _schedule_voice_state("captioning", frames=10) await _send_event("thinking", stage="transcribing", draft_count=draft_count) # Parallel instant: start quick_reply from draft ASAP, but prefer full transcript for context # Previously we used last_draft_text which was often just "Perfect. Perfect." leading to generic reply t_quick_start = time.time() instant_future = None draft_for_llm = last_draft_text if last_draft_text and len(last_draft_text.strip()) > 2 else None # Only start early if draft seems substantial (>30 chars and not just generic starter) - else wait for final transcript if draft_for_llm and ESP32_LIVE_DRAFT_USE_MACMINI and len(draft_for_llm.strip()) > 30: try: instant_future = asyncio.create_task( self._apple_llm_quick_reply_via_macmini( draft=draft_for_llm, context="", instructions=request.headers.get("X-Hermes-Instructions") or "" ) ) except Exception: instant_future = None # Final transcription: prefer Mac mini live pipe (ApplePipeTranscriber) transcript = None final_source = "none" t_transcribe_start = time.time() try: if ESP32_LIVE_DRAFT_USE_MACMINI and len(pcm_buffer) > 0: full_wav = self._s16le_to_wav_bytes(bytes(pcm_buffer), sample_rate=sample_rate, channels=channels) mac_txt = await self._transcribe_draft_via_macmini(full_wav, locale=locale_req) if mac_txt and len(mac_txt.strip()) >= 2: transcript = mac_txt.strip() final_source = "macmini-live-final" if instant_future is None and mac_txt: # Start instant from final if no draft was available earlier try: draft_for_llm = mac_txt[:200] instant_future = asyncio.create_task( self._apple_llm_quick_reply_via_macmini( draft=mac_txt[:200], context="", instructions=request.headers.get("X-Hermes-Instructions") or "" ) ) except Exception: pass except Exception as e: logger.debug(f"[api_server] macmini final transcribe failed: {e}") if not transcript: from tools.transcription_tools import transcribe_audio try: if not wav_path.exists(): await asyncio.to_thread( self._write_pcm_s16le_wav, raw_path, wav_path, sample_rate=sample_rate, channels=channels, ) except Exception: pass stt_result = await asyncio.to_thread(transcribe_audio, str(wav_path) if wav_path.exists() else str(raw_path)) if not stt_result.get("success"): message = stt_result.get("error") or "Transcription failed" self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "transcription_failed", "error": message, "timestamp": time.time()}) await _send_event("error", code="transcription_failed", message=message) await ws.close() return ws transcript = str(stt_result.get("transcript") or "").strip() final_source = "local-faster-whisper" if not transcript: self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "empty_transcript", "timestamp": time.time()}) await _send_event("error", code="empty_transcript", message="Transcription was empty") await ws.close() return ws transcribe_ms = int((time.time() - t_transcribe_start) * 1000) self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "transcribed", "transcript": transcript, "timestamp": time.time(), "draft_count": draft_count, "final_source": final_source, "transcribe_ms": transcribe_ms}) await _send_event("transcript", text=transcript, is_final=True, isFinal=True, draft_count=draft_count, source=final_source, transcribe_ms=transcribe_ms) await _send_event("final", text=transcript, isFinal=True, source=final_source, transcribe_ms=transcribe_ms) summary = self._short_voice_summary(transcript, 180) # Now await instant that was started in parallel on stop instant_reply = None instant_ok = False quick_ms = 0 if draft_for_llm is None or len(draft_for_llm.strip()) < 20: draft_for_llm = transcript[:300] # For contextual reply, always feed final transcript if draft was short/generic draft_to_use_for_llm = transcript[:400] if (not draft_for_llm or len(transcript) > len(draft_for_llm) or len(draft_for_llm.strip()) < 30) else draft_for_llm try: if instant_future is not None: try: instant_reply = await asyncio.wait_for(instant_future, timeout=3.0) # If instant from short draft is generic, discard and regenerate from full transcript if instant_reply and len(draft_for_llm.strip()) < 30 and len(transcript) > len(draft_for_llm): # Prefer transcript-based reply for context instant_reply = None except asyncio.TimeoutError: instant_reply = None try: instant_future.cancel() except Exception: pass if not instant_reply and ESP32_LIVE_DRAFT_USE_MACMINI: # No draft, or future failed — try once more from final transcript (contextual) instant_reply = await self._apple_llm_quick_reply_via_macmini( draft=draft_to_use_for_llm[:400], context="", instructions=request.headers.get("X-Hermes-Instructions") or "" ) if instant_reply: quick_ms = int((time.time() - t_quick_start) * 1000) instant_ok = True await _send_event("instant_response", text=instant_reply, is_quick=True, quick_ms=quick_ms, draft=draft_for_llm[:200], source="apple-llm-3b-ane", final=False, persistent=True) await _send_event("quick_reply", text=instant_reply, ms=quick_ms, persistent=True) await _send_event("response_text", text=instant_reply, response_id="", final=False, is_instant=True, quick_ms=quick_ms, source="apple-llm-3b-ane") # Also speak instant reply via Mac mini voicebox/boy voice while Hermes generates full answer # This gives user audio feedback immediately instead of silence try: # First try fast Mac say Alex (boy voice) for speed <500ms, voicebox Aiden if available but slower t_speak_start = time.time() # Try Aiden voicebox first if you prefer quality, else Alex for speed. Config: try Alex first for speed. audio_b64 = await self._speak_instant_via_macmini(instant_reply, voice="Alex", use_voicebox=False) if not audio_b64: # Fallback try voicebox Aiden boy voice (Qwen 1.7B) if say failed or you prefer quality try: audio_b64 = await self._speak_instant_via_macmini(instant_reply, voice="Aiden", use_voicebox=True) except Exception: pass if audio_b64 and screen_url: speak_ms = int((time.time() - t_speak_start) * 1000) # Send audio to ESP32 screen via play_audio_base64 - this was your request: hear fast reply while Hermes thinks try: # Use play_audio_base64 (WAV 16k mono) - Mac say already 16k audio_task = asyncio.create_task( self._call_screen_mcp( screen_url, "play_audio_base64", {"wav_base64": audio_b64, "volume": 80}, timeout=20, ) ) self._background_tasks.add(audio_task) audio_task.add_done_callback(self._background_tasks.discard) except Exception as e: logger.debug(f"[api_server] instant audio play failed: {e}") await _send_event("instant_audio", text=instant_reply, quick_ms=quick_ms, speak_ms=speak_ms, voice="Alex/Aiden boy via Mac mini voicebox", source="macmini-say/voicebox") self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "instant_audio", "speak_ms": speak_ms, "timestamp": time.time()}) except Exception as e: logger.debug(f"[api_server] instant speak failed: {e}") if screen_url: try: task = asyncio.create_task(self._send_screen_text(screen_url, instant_reply, clear=True)) self._background_tasks.add(task) task.add_done_callback(self._background_tasks.discard) except Exception: pass self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "instant_reply", "instant_reply": instant_reply, "quick_ms": quick_ms, "timestamp": time.time()}) except Exception as e: logger.debug(f"[api_server] instant reply failed: {e}") if not instant_ok: ack_text = f"Heard: {summary}\n\nWorking on the full answer now." await _send_event("response_text", text=ack_text, response_id="", final=False, is_fallback=True) self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "queued_background", "transcript": transcript, "timestamp": time.time(), "instant_ok": instant_ok}) try: await _send_event("done", mode="background", screen_url=screen_url or "", draft_count=draft_count, source=final_source, instant_reply=instant_reply or "", instant_ok=instant_ok, quick_ms=quick_ms, transcribe_ms=transcribe_ms) if instant_ok: await _send_event("thinking", stage="full_answer", keep_instant=True, instant_text=instant_reply, persistent_instant=True) await ws.close() except Exception: pass background_task = asyncio.create_task( self._run_esp32_voice_background( transcript=transcript, device_id=device_id, instructions=request.headers.get("X-Hermes-Instructions") or None, audio_dir=audio_dir, screen_url=screen_url, ) ) try: self._background_tasks.add(background_task) except TypeError: pass if hasattr(background_task, "add_done_callback"): background_task.add_done_callback(self._background_tasks.discard) return ws except Exception as exc: logger.exception("[api_server] ESP32 voice websocket processing failed") self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "websocket_exception", "error": str(exc), "timestamp": time.time()}) try: await _send_event("error", code="internal_error", message=str(exc)) await ws.close() except Exception: pass return ws finally: try: raw_path.unlink(missing_ok=True) except Exception: pass async def _handle_esp32_voice(self, request: "web.Request") -> "web.Response": """POST /api/esp32/voice — receive audio, run Hermes, return TTS audio.""" auth_err = self._check_auth(request) if auth_err: return auth_err device_id = ( request.headers.get("X-Device-ID") or request.query.get("device_id") or "default" ).strip()[:80] or "default" try: audio_bytes = await request.read() except Exception as exc: return web.json_response(_openai_error(f"Failed to read audio body: {exc}"), status=400) if not audio_bytes: return web.json_response(_openai_error("Empty audio body", code="empty_audio"), status=400) if len(audio_bytes) > ESP32_AUDIO_MAX_BYTES: return web.json_response( _openai_error( f"Audio body is too large ({len(audio_bytes)} bytes); max is {ESP32_AUDIO_MAX_BYTES}", code="audio_too_large", ), status=413, ) suffix = self._audio_extension_for_content_type(request.headers.get("Content-Type", "")) try: from hermes_constants import get_hermes_dir audio_dir = get_hermes_dir("cache/audio", "audio_cache") / "esp32" except Exception: audio_dir = Path.home() / ".hermes" / "cache" / "audio" / "esp32" audio_dir.mkdir(parents=True, exist_ok=True) safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default" audio_path = audio_dir / f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}{suffix}" audio_path.write_bytes(audio_bytes) audit_ctx = self._request_audit_context(request) audit_event: Dict[str, Any] = { "timestamp": time.time(), "device_id": device_id, "source_ip": audit_ctx.get("remote") or audit_ctx.get("peer_ip") or "", "peer_ip": audit_ctx.get("peer_ip") or "", "forwarded_for": audit_ctx.get("forwarded_for") or "", "real_ip": audit_ctx.get("real_ip") or "", "user_agent": audit_ctx.get("user_agent") or "", "reply_mode": self._esp32_reply_mode(request), "screen_device": request.headers.get("X-Hermes-Screen-Device") or request.query.get("screen_device") or "", "screen_url": request.headers.get("X-Hermes-Screen-Url") or request.headers.get("X-Screen-MCP-Url") or request.query.get("screen_url") or "", "content_type": request.headers.get("Content-Type", ""), "audio_bytes": len(audio_bytes), "audio_path": str(audio_path), "status": "received", } def _append_audit(event: Dict[str, Any]) -> None: try: log_path = audio_dir / "voice_requests.jsonl" with log_path.open("a", encoding="utf-8") as handle: handle.write(json.dumps(event, default=str) + "\n") except Exception: logger.debug("[api_server] failed to append ESP32 voice audit log", exc_info=True) _append_audit(audit_event) if self._esp32_reply_mode(request) == "ack": screen_url = self._resolve_esp32_screen_url(request, device_id) async def _ack_background() -> None: try: await self._send_screen_animation(screen_url, "ack") await self._send_screen_animation(screen_url, "captioning") from tools.transcription_tools import transcribe_audio stt_result_bg = await asyncio.wait_for( asyncio.to_thread(transcribe_audio, str(audio_path)), timeout=45, ) if not stt_result_bg.get("success"): failed_event = dict(audit_event) failed_event.update({ "status": "transcription_failed", "error": stt_result_bg.get("error") or "Transcription failed", "timestamp": time.time(), }) _append_audit(failed_event) await self._send_screen_text( screen_url, "I heard audio, but transcription failed. Try again.", clear=True, ) return transcript_bg = str(stt_result_bg.get("transcript") or "").strip() if not transcript_bg: empty_event = dict(audit_event) empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) _append_audit(empty_event) await self._send_screen_text( screen_url, "I heard audio, but got no words. Try again.", clear=True, ) return transcript_event = dict(audit_event) transcript_event.update({"status": "transcribed", "transcript": transcript_bg, "timestamp": time.time()}) _append_audit(transcript_event) await self._run_esp32_voice_background( transcript=transcript_bg, device_id=device_id, instructions=request.headers.get("X-Hermes-Instructions") or None, audio_dir=audio_dir, screen_url=screen_url, ) except asyncio.TimeoutError: timeout_event = dict(audit_event) timeout_event.update({"status": "transcription_timeout", "error": "Transcription exceeded 45s", "timestamp": time.time()}) _append_audit(timeout_event) await self._send_screen_text( screen_url, "I heard audio, but captioning timed out. Try again.", clear=True, ) except Exception as exc: logger.exception("[api_server] ESP32 ack-mode background processing failed") failed_event = dict(audit_event) failed_event.update({"status": "background_exception", "error": str(exc), "timestamp": time.time()}) _append_audit(failed_event) task = asyncio.create_task(_ack_background()) try: self._background_tasks.add(task) except TypeError: pass if hasattr(task, "add_done_callback"): task.add_done_callback(self._background_tasks.discard) ack_path = await self._esp32_ack_wav(audio_dir=audio_dir, device_id=device_id) headers = { "X-Hermes-Conversation": f"esp32:{device_id}", "X-Hermes-Reply-Mode": "ack", "X-Hermes-Screen-Url": screen_url or "", "Content-Type": "audio/wav", } return web.FileResponse(ack_path, headers=headers) try: from tools.transcription_tools import transcribe_audio stt_result = await asyncio.to_thread(transcribe_audio, str(audio_path)) except Exception as exc: logger.exception("[api_server] ESP32 transcription failed") failed_event = dict(audit_event) failed_event.update({"status": "transcription_exception", "error": str(exc), "timestamp": time.time()}) _append_audit(failed_event) return web.json_response(_openai_error(f"Transcription failed: {exc}"), status=500) if not stt_result.get("success"): failed_event = dict(audit_event) failed_event.update({ "status": "transcription_failed", "error": stt_result.get("error") or "Transcription failed", "timestamp": time.time(), }) _append_audit(failed_event) return web.json_response( _openai_error(stt_result.get("error") or "Transcription failed", code="transcription_failed"), status=502, ) transcript = str(stt_result.get("transcript") or "").strip() if not transcript: empty_event = dict(audit_event) empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()}) _append_audit(empty_event) return web.json_response(_openai_error("Transcription was empty", code="empty_transcript"), status=422) transcript_event = dict(audit_event) transcript_event.update({"status": "transcribed", "transcript": transcript, "timestamp": time.time()}) _append_audit(transcript_event) instructions = request.headers.get("X-Hermes-Instructions") or None try: response_data, final_text = await self._run_esp32_voice_turn( transcript=transcript, device_id=device_id, instructions=instructions, ) except Exception as exc: logger.exception("[api_server] ESP32 agent turn failed") return web.json_response(_openai_error(f"Agent turn failed: {exc}"), status=500) if not final_text.strip(): final_text = self._extract_response_text(response_data) or "I heard you, but I do not have a spoken answer." try: from tools.tts_tool import text_to_speech_tool tts_raw = await asyncio.to_thread(text_to_speech_tool, final_text) tts_result = json.loads(tts_raw) except Exception as exc: logger.exception("[api_server] ESP32 TTS failed") return web.json_response( _openai_error(f"TTS failed after response was generated: {exc}", code="tts_failed"), status=500, headers={ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), }, ) if not tts_result.get("success"): return web.json_response( _openai_error(tts_result.get("error") or "TTS failed", code="tts_failed"), status=502, headers={ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), }, ) tts_path = str(tts_result.get("file_path") or "") if not tts_path or not Path(tts_path).exists(): return web.json_response( _openai_error("TTS succeeded but no audio file was produced", code="tts_file_missing"), status=500, ) try: response_audio_path = self._convert_audio_to_esp32_wav( tts_path, output_dir=audio_dir, device_id=device_id, ) except Exception as exc: logger.exception("[api_server] ESP32 WAV normalization failed") return web.json_response( _openai_error(f"Could not convert TTS audio to ESP32 WAV: {exc}", code="wav_conversion_failed"), status=500, headers={ "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), }, ) headers = { "X-Hermes-Transcript": self._safe_audio_header(transcript, 1000), "X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000), "X-Hermes-Response-Id": str(response_data.get("id") or ""), "X-Hermes-Conversation": f"esp32:{device_id}", "X-Hermes-TTS-Source-Type": self._audio_response_mime(tts_path), "Content-Type": "audio/wav", } return web.FileResponse( response_audio_path, headers=headers, )