Adolfo Reyna
6d593cd9ef
- Migrates custom voice gateway from in-tree api_server.py patch to durable plugin - Auto-restores routes after hermes update (POST /api/esp32/voice + WS /api/esp32/voice/ws) - Includes live draft ingestion via MacMiniMCP speech_live_transcribe (Apple pipe volatile) - Includes instant smart reply via apple_llm_quick_reply (3B ANE ~300ms, contextual not generic) - Guard for missing ipaddress/urlparse imports that caused 500 loop (2026-07-14) - Tools: esp32_voice_gateway_status, esp32_voice_gateway_restore - Scripts: restore_after_update.py with 400-not-404 verification - Installed as ~/.hermes/plugins/esp32-voice-gateway (enabled, survives hermes update) Verified: routes present, plugin enabled, has_routes=True
3289 lines
151 KiB
Python
3289 lines
151 KiB
Python
# Original ESP32 extraction (for reference, not used at runtime)
|
|
# From api_server.py ESP32_AUDIO_MAX_BYTES -> _handle_esp32_voice
|
|
ESP32_AUDIO_MAX_BYTES = 8_000_000 # ~4 minutes of 16 kHz mono PCM WAV; enough for push-to-talk tests
|
|
ESP32_WS_MAX_AUDIO_BYTES = 8_000_000
|
|
ESP32_WS_MIN_AUDIO_BYTES = 1_600 # 50 ms at 16 kHz mono s16le; rejects empty accidental taps
|
|
ESP32_WS_TTS_CHUNK_BYTES = 16_384
|
|
|
|
# Live draft ingestion config (from whisper-translation/apple_speech engine)
|
|
ESP32_LIVE_DRAFT_ENABLED = os.getenv("ESP32_LIVE_DRAFT_ENABLED", "1").lower() not in {"0", "false", "no", "off"}
|
|
ESP32_LIVE_DRAFT_INTERVAL_MS = int(os.getenv("ESP32_LIVE_DRAFT_INTERVAL_MS", "800")) # how often to snapshot for draft
|
|
ESP32_LIVE_DRAFT_MIN_BYTES = int(os.getenv("ESP32_LIVE_DRAFT_MIN_BYTES", "8000")) # min new bytes since last draft
|
|
ESP32_LIVE_DRAFT_MIN_TOTAL_MS = int(os.getenv("ESP32_LIVE_DRAFT_MIN_TOTAL_MS", "500")) # don't draft until we have 500ms
|
|
ESP32_MACMINI_MCP_URL = os.getenv("MACMINI_MCP_URL") or None # overridable, else from config.yaml mcp_servers.macmini
|
|
ESP32_MACMINI_MCP_TOKEN = os.getenv("MACMINI_MCP_TOKEN") or None
|
|
ESP32_LIVE_DRAFT_USE_MACMINI = os.getenv("ESP32_LIVE_DRAFT_USE_MACMINI", "1").lower() not in {"0", "false", "no", "off"}
|
|
|
|
_ESP32_AUDIO_EXTENSIONS = {
|
|
"audio/wav": ".wav",
|
|
"audio/wave": ".wav",
|
|
"audio/x-wav": ".wav",
|
|
"audio/mpeg": ".mp3",
|
|
"audio/mp3": ".mp3",
|
|
"audio/ogg": ".ogg",
|
|
"audio/opus": ".ogg",
|
|
"audio/webm": ".webm",
|
|
"audio/mp4": ".m4a",
|
|
"audio/x-m4a": ".m4a",
|
|
"application/octet-stream": ".wav",
|
|
}
|
|
|
|
_AUDIO_RESPONSE_MIME_BY_SUFFIX = {
|
|
".mp3": "audio/mpeg",
|
|
".wav": "audio/wav",
|
|
".ogg": "audio/ogg",
|
|
".opus": "audio/ogg",
|
|
".m4a": "audio/mp4",
|
|
".mp4": "audio/mp4",
|
|
".flac": "audio/flac",
|
|
}
|
|
|
|
ESP32_VOICE_ANIMATION_PATH = "hermes_voice_animations.py"
|
|
ESP32_VOICE_ANIMATION_SCRIPT = "# Hermes ESP32 voice-state animations for MicroPython screen boards.\n# Uploaded by the API server; can also be run manually with:\n# exec(open('hermes_voice_animations.py').read()); run_state('ack')\nimport time\ntry:\n import board_config\nexcept Exception:\n board_config = None\n\n\ndef _display():\n return getattr(board_config, 'display_instance', None) if board_config else None\n\n\ndef _sleep(ms):\n try:\n time.sleep_ms(ms)\n except AttributeError:\n time.sleep(ms / 1000)\n\n\ndef _text(d, msg, x, y, scale=1, color=1):\n if hasattr(d, 'text_large') and scale > 1:\n try:\n d.text_large(msg, x, y, scale=scale, color=color)\n return\n except TypeError:\n pass\n d.text(msg, x, y, color)\n\n\ndef _center_text(d, msg, y, scale=1, color=1):\n width = getattr(d, 'width', 320)\n char_w = 8 * scale\n x = max(0, (width - len(msg) * char_w) // 2)\n _text(d, msg, x, y, scale=scale, color=color)\n\n\ndef _wrap(text, width_chars=34, max_lines=4):\n words = str(text or '').replace('\\n', ' ').split()\n lines, line = [], ''\n for w in words:\n candidate = (line + ' ' + w).strip()\n if len(candidate) > width_chars and line:\n lines.append(line)\n line = w\n if len(lines) >= max_lines:\n break\n else:\n line = candidate\n if line and len(lines) < max_lines:\n lines.append(line)\n return lines\n\n\ndef _spinner(d, title, subtitle='', frames=24, delay=70):\n width, height = getattr(d, 'width', 320), getattr(d, 'height', 240)\n cx, cy = width // 2, height // 2 - 8\n spokes = [(-22,0),(-16,-16),(0,-22),(16,-16),(22,0),(16,16),(0,22),(-16,16)]\n for frame in range(frames):\n d.clear(0)\n _center_text(d, title, 18, scale=2)\n for i, (dx, dy) in enumerate(spokes):\n color = 1 if (i + frame) % len(spokes) in (0, 1, 2, 3) else 0\n size = 4 if i == frame % len(spokes) else 2\n d.fill_rect(cx + dx - size // 2, cy + dy - size // 2, size, size, color)\n d.rect(8, height - 34, width - 16, 18, 1)\n d.fill_rect(10, height - 32, ((width - 20) * ((frame % 16) + 1)) // 16, 14, 1)\n if subtitle:\n _center_text(d, subtitle[:32], height - 56, scale=1)\n d.show()\n _sleep(delay)\n\n\ndef ack(frames=18):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'GOT IT', 18, scale=2)\n # expanding check pulse\n box = 48 + (i % 6) * 8\n x, y = width // 2 - box // 2, height // 2 - box // 2\n d.rect(x, y, box, box, 1)\n d.line(width//2 - 24, height//2, width//2 - 6, height//2 + 18, 1)\n d.line(width//2 - 6, height//2 + 18, width//2 + 30, height//2 - 22, 1)\n _center_text(d, 'Listening captured', height - 36, scale=1)\n d.show()\n _sleep(55)\n\n\ndef captioning(frames=36):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'CAPTIONING', 16, scale=2)\n for bar in range(9):\n h = 8 + ((i * 5 + bar * 13) % 42)\n x = width // 2 - 72 + bar * 18\n d.fill_rect(x, height // 2 - h // 2, 10, h, 1)\n dots = '.' * ((i // 5) % 4)\n _center_text(d, 'Transcribing' + dots, height - 42, scale=1)\n d.show()\n _sleep(65)\n\n\ndef waiting(text='', frames=44):\n d = _display()\n if not d:\n return\n width, height = d.width, d.height\n for i in range(frames):\n d.clear(0)\n _center_text(d, 'THINKING', 12, scale=2)\n r = 18 + (i % 10) * 2\n cx, cy = width // 2, height // 2 - 10\n d.rect(cx - r, cy - r, r * 2, r * 2, 1)\n d.rect(cx - r//2, cy - r//2, r, r, 1)\n if text:\n d.text('Heard:', 10, height - 68, 1)\n for idx, line in enumerate(_wrap(text, 36, 3)):\n d.text(line[:38], 10, height - 54 + idx * 12, 1)\n else:\n _center_text(d, 'Waiting for reply...', height - 42, scale=1)\n d.show()\n _sleep(75)\n\n\ndef run_state(state='ack', text='', frames=None):\n state = str(state or 'ack').lower()\n if state in ('ack', 'listening', 'got_it'):\n ack(frames or 18)\n elif state in ('caption', 'captioning', 'transcribing'):\n captioning(frames or 36)\n elif state in ('wait', 'waiting', 'thinking', 'reply'):\n waiting(text, frames or 44)\n else:\n _spinner(_display(), 'HERMES', state[:28], frames or 20)\n"
|
|
|
|
|
|
def _coerce_port(value: Any, default: int = DEFAULT_PORT) -> int:
|
|
"""Parse a listen port without letting malformed env/config values crash startup."""
|
|
try:
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
return default
|
|
|
|
|
|
_TRUE_REQUEST_BOOL_STRINGS = frozenset({"1", "true", "yes", "on"})
|
|
_FALSE_REQUEST_BOOL_STRINGS = frozenset({"0", "false", "no", "off"})
|
|
|
|
|
|
def _coerce_request_bool(value: Any, default: bool = False) -> bool:
|
|
"""Normalize boolean-like API payload values.
|
|
|
|
External clients should send real JSON booleans, but some OpenAI-compatible
|
|
frontends and middleware serialize flags like ``stream`` as strings. Using
|
|
Python truthiness on those values misroutes requests because ``"false"`` is
|
|
still truthy. Treat only explicit bool-ish scalars as booleans; everything
|
|
else falls back to the caller's default.
|
|
"""
|
|
if isinstance(value, bool):
|
|
return value
|
|
if value is None:
|
|
return default
|
|
if isinstance(value, str):
|
|
normalized = value.strip().lower()
|
|
if normalized in _TRUE_REQUEST_BOOL_STRINGS:
|
|
return True
|
|
if normalized in _FALSE_REQUEST_BOOL_STRINGS:
|
|
return False
|
|
return default
|
|
if isinstance(value, (int, float)):
|
|
return bool(value)
|
|
return default
|
|
|
|
|
|
def _normalize_chat_content(
|
|
content: Any, *, _max_depth: int = 10, _depth: int = 0,
|
|
) -> str:
|
|
"""Normalize OpenAI chat message content into a plain text string.
|
|
|
|
Some clients (Open WebUI, LobeChat, etc.) send content as an array of
|
|
typed parts instead of a plain string::
|
|
|
|
[{"type": "text", "text": "hello"}, {"type": "input_text", "text": "..."}]
|
|
|
|
This function flattens those into a single string so the agent pipeline
|
|
(which expects strings) doesn't choke.
|
|
|
|
Defensive limits prevent abuse: recursion depth, list size, and output
|
|
length are all bounded.
|
|
"""
|
|
if _depth > _max_depth:
|
|
return ""
|
|
if content is None:
|
|
return ""
|
|
if isinstance(content, str):
|
|
return content[:MAX_NORMALIZED_TEXT_LENGTH] if len(content) > MAX_NORMALIZED_TEXT_LENGTH else content
|
|
|
|
if isinstance(content, list):
|
|
parts: List[str] = []
|
|
total_len = 0
|
|
items = content[:MAX_CONTENT_LIST_SIZE] if len(content) > MAX_CONTENT_LIST_SIZE else content
|
|
for item in items:
|
|
if isinstance(item, str):
|
|
if item:
|
|
part = item[:MAX_NORMALIZED_TEXT_LENGTH]
|
|
parts.append(part)
|
|
total_len += len(part)
|
|
elif isinstance(item, dict):
|
|
item_type = str(item.get("type") or "").strip().lower()
|
|
if item_type in {"text", "input_text", "output_text"}:
|
|
text = item.get("text", "")
|
|
if text:
|
|
try:
|
|
part = str(text)[:MAX_NORMALIZED_TEXT_LENGTH]
|
|
parts.append(part)
|
|
total_len += len(part)
|
|
except Exception:
|
|
pass
|
|
# Silently skip image_url / other non-text parts
|
|
elif isinstance(item, list):
|
|
nested = _normalize_chat_content(item, _max_depth=_max_depth, _depth=_depth + 1)
|
|
if nested:
|
|
parts.append(nested)
|
|
total_len += len(nested)
|
|
# Check accumulated size
|
|
if total_len >= MAX_NORMALIZED_TEXT_LENGTH:
|
|
break
|
|
result = "\n".join(parts)
|
|
return result[:MAX_NORMALIZED_TEXT_LENGTH] if len(result) > MAX_NORMALIZED_TEXT_LENGTH else result
|
|
|
|
# Fallback for unexpected types (int, float, bool, etc.)
|
|
try:
|
|
result = str(content)
|
|
return result[:MAX_NORMALIZED_TEXT_LENGTH] if len(result) > MAX_NORMALIZED_TEXT_LENGTH else result
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
# Content part type aliases used by the OpenAI Chat Completions and Responses
|
|
# APIs. We accept both spellings on input and emit a single canonical internal
|
|
# shape (``{"type": "text", ...}`` / ``{"type": "image_url", ...}``) that the
|
|
# rest of the agent pipeline already understands.
|
|
_TEXT_PART_TYPES = frozenset({"text", "input_text", "output_text"})
|
|
_IMAGE_PART_TYPES = frozenset({"image_url", "input_image"})
|
|
_FILE_PART_TYPES = frozenset({"file", "input_file"})
|
|
|
|
|
|
def _normalize_multimodal_content(content: Any) -> Any:
|
|
"""Validate and normalize multimodal content for the API server.
|
|
|
|
Returns a plain string when the content is text-only, or a list of
|
|
``{"type": "text"|"image_url", ...}`` parts when images are present.
|
|
The output shape is the native OpenAI Chat Completions vision format,
|
|
which the agent pipeline accepts verbatim (OpenAI-wire providers) or
|
|
converts (``_preprocess_anthropic_content`` for Anthropic).
|
|
|
|
Raises ``ValueError`` with an OpenAI-style code on invalid input:
|
|
* ``unsupported_content_type`` — file/input_file/file_id parts, or
|
|
non-image ``data:`` URLs.
|
|
* ``invalid_image_url`` — missing URL or unsupported scheme.
|
|
* ``invalid_content_part`` — malformed text/image objects.
|
|
|
|
Callers translate the ValueError into a 400 response.
|
|
"""
|
|
# Scalar passthrough mirrors ``_normalize_chat_content``.
|
|
if content is None:
|
|
return ""
|
|
if isinstance(content, str):
|
|
return content[:MAX_NORMALIZED_TEXT_LENGTH] if len(content) > MAX_NORMALIZED_TEXT_LENGTH else content
|
|
if not isinstance(content, list):
|
|
# Mirror the legacy text-normalizer's fallback so callers that
|
|
# pre-existed image support still get a string back.
|
|
return _normalize_chat_content(content)
|
|
|
|
items = content[:MAX_CONTENT_LIST_SIZE] if len(content) > MAX_CONTENT_LIST_SIZE else content
|
|
normalized_parts: List[Dict[str, Any]] = []
|
|
text_accum_len = 0
|
|
|
|
for part in items:
|
|
if isinstance(part, str):
|
|
if part:
|
|
trimmed = part[:MAX_NORMALIZED_TEXT_LENGTH]
|
|
normalized_parts.append({"type": "text", "text": trimmed})
|
|
text_accum_len += len(trimmed)
|
|
continue
|
|
|
|
if not isinstance(part, dict):
|
|
# Ignore unknown scalars for forward compatibility with future
|
|
# Responses API additions (e.g. ``refusal``). The same policy
|
|
# the text normalizer applies.
|
|
continue
|
|
|
|
raw_type = part.get("type")
|
|
part_type = str(raw_type or "").strip().lower()
|
|
|
|
if part_type in _TEXT_PART_TYPES:
|
|
text = part.get("text")
|
|
if text is None:
|
|
continue
|
|
if not isinstance(text, str):
|
|
text = str(text)
|
|
if text:
|
|
trimmed = text[:MAX_NORMALIZED_TEXT_LENGTH]
|
|
normalized_parts.append({"type": "text", "text": trimmed})
|
|
text_accum_len += len(trimmed)
|
|
continue
|
|
|
|
if part_type in _IMAGE_PART_TYPES:
|
|
detail = part.get("detail")
|
|
image_ref = part.get("image_url")
|
|
# OpenAI Responses sends ``input_image`` with a top-level
|
|
# ``image_url`` string; Chat Completions sends ``image_url`` as
|
|
# ``{"url": "...", "detail": "..."}``. Support both.
|
|
if isinstance(image_ref, dict):
|
|
url_value = image_ref.get("url")
|
|
detail = image_ref.get("detail", detail)
|
|
else:
|
|
url_value = image_ref
|
|
if not isinstance(url_value, str) or not url_value.strip():
|
|
raise ValueError("invalid_image_url:Image parts must include a non-empty image URL.")
|
|
url_value = url_value.strip()
|
|
lowered = url_value.lower()
|
|
if lowered.startswith("data:"):
|
|
if not lowered.startswith("data:image/") or "," not in url_value:
|
|
raise ValueError(
|
|
"unsupported_content_type:Only image data URLs are supported. "
|
|
"Non-image data payloads are not supported."
|
|
)
|
|
elif not (lowered.startswith("http://") or lowered.startswith("https://")):
|
|
raise ValueError(
|
|
"invalid_image_url:Image inputs must use http(s) URLs or data:image/... URLs."
|
|
)
|
|
image_part: Dict[str, Any] = {"type": "image_url", "image_url": {"url": url_value}}
|
|
if detail is not None:
|
|
if not isinstance(detail, str) or not detail.strip():
|
|
raise ValueError("invalid_content_part:Image detail must be a non-empty string when provided.")
|
|
image_part["image_url"]["detail"] = detail.strip()
|
|
normalized_parts.append(image_part)
|
|
continue
|
|
|
|
if part_type in _FILE_PART_TYPES:
|
|
raise ValueError(
|
|
"unsupported_content_type:Inline image inputs are supported, "
|
|
"but uploaded files and document inputs are not supported on this endpoint."
|
|
)
|
|
|
|
# Unknown part type — reject explicitly so clients get a clear error
|
|
# instead of a silently dropped turn.
|
|
raise ValueError(
|
|
f"unsupported_content_type:Unsupported content part type {raw_type!r}. "
|
|
"Only text and image_url/input_image parts are supported."
|
|
)
|
|
|
|
if not normalized_parts:
|
|
return ""
|
|
|
|
# Text-only: collapse to a plain string so downstream logging/trajectory
|
|
# code sees the native shape and prompt caching on text-only turns is
|
|
# unaffected.
|
|
if all(p.get("type") == "text" for p in normalized_parts):
|
|
return "\n".join(p["text"] for p in normalized_parts if p.get("text"))
|
|
|
|
return normalized_parts
|
|
|
|
|
|
def _content_has_visible_payload(content: Any) -> bool:
|
|
"""True when content has any text or image attachment. Used to reject empty turns."""
|
|
if isinstance(content, str):
|
|
return bool(content.strip())
|
|
if isinstance(content, list):
|
|
for part in content:
|
|
if isinstance(part, dict):
|
|
ptype = str(part.get("type") or "").strip().lower()
|
|
if ptype in _TEXT_PART_TYPES and str(part.get("text") or "").strip():
|
|
return True
|
|
if ptype in _IMAGE_PART_TYPES:
|
|
return True
|
|
return False
|
|
|
|
|
|
def _multimodal_validation_error(exc: ValueError, *, param: str) -> "web.Response":
|
|
"""Translate a ``_normalize_multimodal_content`` ValueError into a 400 response."""
|
|
raw = str(exc)
|
|
code, _, message = raw.partition(":")
|
|
if not message:
|
|
code, message = "invalid_content_part", raw
|
|
return web.json_response(
|
|
_openai_error(message, code=code, param=param),
|
|
status=400,
|
|
)
|
|
|
|
|
|
def _session_chat_user_message(body: Dict[str, Any], *, param: str = "message") -> tuple[Any, Optional["web.Response"]]:
|
|
"""Parse and normalize session chat ``message`` / ``input`` like chat completions."""
|
|
user_message = body.get("message") or body.get("input")
|
|
if not _content_has_visible_payload(user_message):
|
|
return None, web.json_response(
|
|
_openai_error("Missing 'message' field", code="missing_message"),
|
|
status=400,
|
|
)
|
|
try:
|
|
return _normalize_multimodal_content(user_message), None
|
|
except ValueError as exc:
|
|
return None, _multimodal_validation_error(exc, param=param)
|
|
|
|
|
|
def check_api_server_requirements() -> bool:
|
|
"""Check if API server dependencies are available."""
|
|
return AIOHTTP_AVAILABLE
|
|
|
|
|
|
class ResponseStore:
|
|
"""
|
|
SQLite-backed LRU store for Responses API state.
|
|
|
|
Each stored response includes the full internal conversation history
|
|
(with tool calls and results) so it can be reconstructed on subsequent
|
|
requests via previous_response_id.
|
|
|
|
Persists across gateway restarts. Falls back to in-memory SQLite
|
|
if the on-disk path is unavailable.
|
|
"""
|
|
|
|
def __init__(self, max_size: int = MAX_STORED_RESPONSES, db_path: str = None):
|
|
self._max_size = max_size
|
|
if db_path is None:
|
|
try:
|
|
from hermes_cli.config import get_hermes_home
|
|
db_path = str(get_hermes_home() / "response_store.db")
|
|
except Exception:
|
|
db_path = ":memory:"
|
|
self._db_path: Optional[str] = db_path if db_path != ":memory:" else None
|
|
try:
|
|
self._conn = sqlite3.connect(db_path, check_same_thread=False)
|
|
except Exception:
|
|
self._conn = sqlite3.connect(":memory:", check_same_thread=False)
|
|
self._db_path = None
|
|
# Use shared WAL-fallback helper so response_store.db degrades
|
|
# gracefully on NFS/SMB/FUSE-mounted HERMES_HOME (same filesystem
|
|
# issue addressed for state.db/kanban.db — see
|
|
# hermes_state._WAL_INCOMPAT_MARKERS).
|
|
from hermes_state import apply_wal_with_fallback
|
|
apply_wal_with_fallback(self._conn, db_label="response_store.db")
|
|
self._conn.execute(
|
|
"""CREATE TABLE IF NOT EXISTS responses (
|
|
response_id TEXT PRIMARY KEY,
|
|
data TEXT NOT NULL,
|
|
accessed_at REAL NOT NULL
|
|
)"""
|
|
)
|
|
self._conn.execute(
|
|
"""CREATE TABLE IF NOT EXISTS conversations (
|
|
name TEXT PRIMARY KEY,
|
|
response_id TEXT NOT NULL
|
|
)"""
|
|
)
|
|
self._conn.commit()
|
|
# response_store.db contains conversation history (tool payloads,
|
|
# prompts, results). Tighten to owner-only after creation so other
|
|
# local users on a shared box can't read it. Run once at __init__
|
|
# rather than after every commit — chmod-on-every-write is wasted
|
|
# syscalls on a hot path.
|
|
self._tighten_file_permissions()
|
|
|
|
def _tighten_file_permissions(self) -> None:
|
|
"""Force owner-only permissions on the DB and SQLite sidecars."""
|
|
if not self._db_path:
|
|
return
|
|
for candidate in (
|
|
Path(self._db_path),
|
|
Path(f"{self._db_path}-wal"),
|
|
Path(f"{self._db_path}-shm"),
|
|
):
|
|
try:
|
|
if candidate.exists():
|
|
candidate.chmod(0o600)
|
|
except OSError:
|
|
logger.debug(
|
|
"Failed to restrict response store permissions for %s",
|
|
candidate,
|
|
exc_info=True,
|
|
)
|
|
|
|
def get(self, response_id: str) -> Optional[Dict[str, Any]]:
|
|
"""Retrieve a stored response by ID (updates access time for LRU)."""
|
|
row = self._conn.execute(
|
|
"SELECT data FROM responses WHERE response_id = ?", (response_id,)
|
|
).fetchone()
|
|
if row is None:
|
|
return None
|
|
self._conn.execute(
|
|
"UPDATE responses SET accessed_at = ? WHERE response_id = ?",
|
|
(time.time(), response_id),
|
|
)
|
|
self._conn.commit()
|
|
try:
|
|
return json.loads(row[0])
|
|
except (json.JSONDecodeError, TypeError):
|
|
logger.warning(
|
|
"Corrupted JSON in response store for id=%s, evicting entry",
|
|
response_id,
|
|
)
|
|
self._conn.execute(
|
|
"DELETE FROM responses WHERE response_id = ?",
|
|
(response_id,),
|
|
)
|
|
self._conn.commit()
|
|
return None
|
|
|
|
def put(self, response_id: str, data: Dict[str, Any]) -> None:
|
|
"""Store a response, evicting the oldest if at capacity."""
|
|
self._conn.execute(
|
|
"INSERT OR REPLACE INTO responses (response_id, data, accessed_at) VALUES (?, ?, ?)",
|
|
(response_id, json.dumps(data, default=str), time.time()),
|
|
)
|
|
# Evict oldest entries beyond max_size
|
|
count = self._conn.execute("SELECT COUNT(*) FROM responses").fetchone()[0]
|
|
if count > self._max_size:
|
|
# Collect IDs that will be evicted
|
|
evict_ids = [
|
|
row[0]
|
|
for row in self._conn.execute(
|
|
"SELECT response_id FROM responses ORDER BY accessed_at ASC LIMIT ?",
|
|
(count - self._max_size,),
|
|
).fetchall()
|
|
]
|
|
if evict_ids:
|
|
placeholders = ",".join("?" for _ in evict_ids)
|
|
# Clear conversation mappings pointing to evicted responses
|
|
self._conn.execute(
|
|
f"DELETE FROM conversations WHERE response_id IN ({placeholders})",
|
|
evict_ids,
|
|
)
|
|
# Delete evicted responses
|
|
self._conn.execute(
|
|
f"DELETE FROM responses WHERE response_id IN ({placeholders})",
|
|
evict_ids,
|
|
)
|
|
self._conn.commit()
|
|
|
|
def delete(self, response_id: str) -> bool:
|
|
"""Remove a response from the store. Returns True if found and deleted."""
|
|
# Clear conversation mappings pointing to this response
|
|
self._conn.execute(
|
|
"DELETE FROM conversations WHERE response_id = ?", (response_id,)
|
|
)
|
|
cursor = self._conn.execute(
|
|
"DELETE FROM responses WHERE response_id = ?", (response_id,)
|
|
)
|
|
self._conn.commit()
|
|
return cursor.rowcount > 0
|
|
|
|
def get_conversation(self, name: str) -> Optional[str]:
|
|
"""Get the latest response_id for a conversation name."""
|
|
row = self._conn.execute(
|
|
"SELECT response_id FROM conversations WHERE name = ?", (name,)
|
|
).fetchone()
|
|
return row[0] if row else None
|
|
|
|
def set_conversation(self, name: str, response_id: str) -> None:
|
|
"""Map a conversation name to its latest response_id."""
|
|
self._conn.execute(
|
|
"INSERT OR REPLACE INTO conversations (name, response_id) VALUES (?, ?)",
|
|
(name, response_id),
|
|
)
|
|
self._conn.commit()
|
|
|
|
def close(self) -> None:
|
|
"""Close the database connection."""
|
|
try:
|
|
self._conn.close()
|
|
except Exception:
|
|
pass
|
|
|
|
def __len__(self) -> int:
|
|
row = self._conn.execute("SELECT COUNT(*) FROM responses").fetchone()
|
|
return row[0] if row else 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CORS middleware
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_CORS_HEADERS = {
|
|
"Access-Control-Allow-Methods": "GET, POST, DELETE, OPTIONS",
|
|
"Access-Control-Allow-Headers": "Authorization, Content-Type, Idempotency-Key",
|
|
}
|
|
|
|
|
|
if AIOHTTP_AVAILABLE:
|
|
@web.middleware
|
|
async def cors_middleware(request, handler):
|
|
"""Add CORS headers for explicitly allowed origins; handle OPTIONS preflight."""
|
|
adapter = request.app.get("api_server_adapter")
|
|
origin = request.headers.get("Origin", "")
|
|
cors_headers = None
|
|
if adapter is not None:
|
|
if not adapter._origin_allowed(origin):
|
|
return web.Response(status=403)
|
|
cors_headers = adapter._cors_headers_for_origin(origin)
|
|
|
|
if request.method == "OPTIONS":
|
|
if cors_headers is None:
|
|
return web.Response(status=403)
|
|
return web.Response(status=200, headers=cors_headers)
|
|
|
|
response = await handler(request)
|
|
if cors_headers is not None:
|
|
response.headers.update(cors_headers)
|
|
return response
|
|
else:
|
|
cors_middleware = None # type: ignore[assignment]
|
|
|
|
|
|
_MEDIA_IMG_EXT = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp"}
|
|
_MEDIA_MIME = {
|
|
".png": "image/png",
|
|
".jpg": "image/jpeg",
|
|
".jpeg": "image/jpeg",
|
|
".gif": "image/gif",
|
|
".webp": "image/webp",
|
|
".bmp": "image/bmp",
|
|
}
|
|
_MEDIA_DATA_URL_MAX_BYTES = 5 * 1024 * 1024 # skip images larger than 5MB
|
|
|
|
|
|
def _resolve_media_to_data_urls(text: str) -> str:
|
|
"""Replace ``MEDIA:<path>`` image tags with inline base64 data URLs.
|
|
|
|
Remote OpenAI-compatible frontends can't read local file paths, so
|
|
``MEDIA:`` tags referencing images on the server are useless to them.
|
|
Inline small local images as markdown data URLs; non-image or unreadable
|
|
paths are left untouched.
|
|
|
|
Uses the same anchored ``MEDIA_TAG_CLEANUP_RE`` matcher and
|
|
``validate_media_delivery_path`` safety check every other platform
|
|
adapter's media delivery already goes through (gateway/platforms/base.py)
|
|
— an absolute-path anchor plus a known-extension requirement, and a
|
|
resolved-path check against the credential/system-path denylist. The
|
|
prior pattern here matched any bare token after ``MEDIA:`` (including a
|
|
relative/traversal path like ``../../etc/passwd.png``) and read the file
|
|
directly with no denylist, so any image-suffixed, readable file the
|
|
process could see was base64-exfiltrated to the API caller if its path
|
|
merely appeared in the model's own final reply text.
|
|
"""
|
|
if not text or "MEDIA:" not in text:
|
|
return text
|
|
import base64
|
|
|
|
def _to_data_url(path_str: str) -> Optional[str]:
|
|
# validate_media_delivery_path() strips wrapping quotes/backticks
|
|
# and trailing punctuation internally, same as MEDIA_TAG_CLEANUP_RE's
|
|
# other callers (extract_media / _strip_media_tag_directives) rely on.
|
|
safe_path = validate_media_delivery_path(path_str)
|
|
if not safe_path:
|
|
return None
|
|
p = Path(safe_path)
|
|
suffix = p.suffix.lower()
|
|
if suffix not in _MEDIA_IMG_EXT:
|
|
return None
|
|
try:
|
|
if p.stat().st_size > _MEDIA_DATA_URL_MAX_BYTES:
|
|
return None
|
|
b64 = base64.b64encode(p.read_bytes()).decode()
|
|
except OSError:
|
|
return None
|
|
return f""
|
|
|
|
def _repl(m: "re.Match[str]") -> str:
|
|
return _to_data_url(m.group("path")) or m.group(0)
|
|
|
|
try:
|
|
return MEDIA_TAG_CLEANUP_RE.sub(_repl, text)
|
|
except Exception:
|
|
return text
|
|
|
|
|
|
def _redact_api_error_text(value: Any, *, limit: int | None = None) -> str:
|
|
"""Redact API-bound error text before it crosses the HTTP boundary."""
|
|
redacted = redact_sensitive_text(str(value), force=True)
|
|
if limit is not None:
|
|
return redacted[:limit]
|
|
return redacted
|
|
|
|
|
|
def _openai_error(message: str, err_type: str = "invalid_request_error", param: str = None, code: str = None) -> Dict[str, Any]:
|
|
"""OpenAI-style error envelope."""
|
|
return {
|
|
"error": {
|
|
"message": _redact_api_error_text(message),
|
|
"type": err_type,
|
|
"param": param,
|
|
"code": code,
|
|
}
|
|
}
|
|
|
|
|
|
_api_agent_request_reservation: ContextVar[Optional[dict[str, bool]]] = ContextVar(
|
|
"api_agent_request_reservation", default=None
|
|
)
|
|
|
|
|
|
def _admit_api_agent_request(handler):
|
|
"""Reserve an authenticated API turn before its handler first awaits.
|
|
|
|
Gateway shutdown and aiohttp requests share an event loop. Keeping the
|
|
drain check and reservation in one non-awaiting block prevents a request
|
|
admitted immediately before shutdown from becoming invisible while it is
|
|
still parsing its body or resolving session state. The mutable reservation
|
|
is intentionally shared with child tasks so agent/task bookkeeping releases
|
|
this one slot exactly once.
|
|
"""
|
|
@wraps(handler)
|
|
async def _wrapped(self, request, *args, **kwargs):
|
|
auth_err = self._check_auth(request)
|
|
if auth_err:
|
|
return auth_err
|
|
draining = self._draining_response()
|
|
if draining is not None:
|
|
return draining
|
|
reservation = {"active": True}
|
|
token = _api_agent_request_reservation.set(reservation)
|
|
self._pending_agent_requests += 1
|
|
try:
|
|
return await handler(self, request, *args, **kwargs)
|
|
finally:
|
|
if reservation["active"]:
|
|
reservation["active"] = False
|
|
self._pending_agent_requests = max(0, self._pending_agent_requests - 1)
|
|
_api_agent_request_reservation.reset(token)
|
|
|
|
return _wrapped
|
|
|
|
|
|
def _release_pending_api_work(adapter, reservation: dict[str, bool]) -> None:
|
|
"""Release a pending-work reservation exactly once."""
|
|
if reservation["active"]:
|
|
reservation["active"] = False
|
|
adapter._pending_agent_requests = max(0, adapter._pending_agent_requests - 1)
|
|
|
|
|
|
@contextmanager
|
|
def _reserve_pending_api_work(adapter):
|
|
"""Keep externally-triggered background work visible across awaits.
|
|
|
|
A handler can detach the reservation to an asyncio task; its done callback
|
|
then owns release so shutdown cannot miss the handoff to background work.
|
|
"""
|
|
reservation = {"active": True, "detached": False}
|
|
adapter._pending_agent_requests += 1
|
|
try:
|
|
yield reservation
|
|
finally:
|
|
if not reservation["detached"]:
|
|
_release_pending_api_work(adapter, reservation)
|
|
|
|
|
|
if AIOHTTP_AVAILABLE:
|
|
@web.middleware
|
|
async def body_limit_middleware(request, handler):
|
|
"""Reject overly large request bodies early based on Content-Length."""
|
|
if request.method in {"POST", "PUT", "PATCH"}:
|
|
cl = request.headers.get("Content-Length")
|
|
if cl is not None:
|
|
try:
|
|
if int(cl) > MAX_REQUEST_BYTES:
|
|
return web.json_response(_openai_error("Request body too large.", code="body_too_large"), status=413)
|
|
except ValueError:
|
|
return web.json_response(_openai_error("Invalid Content-Length header.", code="invalid_content_length"), status=400)
|
|
try:
|
|
return await handler(request)
|
|
except web.HTTPRequestEntityTooLarge:
|
|
# aiohttp's client_max_size tripped mid-read (chunked bodies carry
|
|
# no Content-Length) — return a proper 413 instead of letting the
|
|
# handler's broad JSON except turn it into 400 "Invalid JSON".
|
|
return web.json_response(
|
|
_openai_error("Request body too large.", code="body_too_large"),
|
|
status=413,
|
|
)
|
|
else:
|
|
body_limit_middleware = None # type: ignore[assignment]
|
|
|
|
_SECURITY_HEADERS = {
|
|
"Content-Security-Policy": "default-src 'none'; frame-ancestors 'none'",
|
|
"Permissions-Policy": "camera=(), microphone=(), geolocation=()",
|
|
"Strict-Transport-Security": "max-age=31536000; includeSubDomains",
|
|
"X-Content-Type-Options": "nosniff",
|
|
"X-Frame-Options": "DENY",
|
|
"X-XSS-Protection": "0",
|
|
"Referrer-Policy": "no-referrer",
|
|
}
|
|
|
|
|
|
if AIOHTTP_AVAILABLE:
|
|
@web.middleware
|
|
async def security_headers_middleware(request, handler):
|
|
"""Add security headers to all responses (including errors)."""
|
|
response = await handler(request)
|
|
for k, v in _SECURITY_HEADERS.items():
|
|
response.headers.setdefault(k, v)
|
|
return response
|
|
else:
|
|
security_headers_middleware = None # type: ignore[assignment]
|
|
|
|
|
|
class _IdempotencyCache:
|
|
"""In-memory idempotency cache with TTL and basic LRU semantics."""
|
|
def __init__(self, max_items: int = 1000, ttl_seconds: int = 300):
|
|
from collections import OrderedDict
|
|
self._store = OrderedDict()
|
|
self._inflight: Dict[tuple[str, str], "asyncio.Task[Any]"] = {}
|
|
self._ttl = ttl_seconds
|
|
self._max = max_items
|
|
|
|
def _purge(self):
|
|
now = time.time()
|
|
expired = [k for k, v in self._store.items() if now - v["ts"] > self._ttl]
|
|
for k in expired:
|
|
self._store.pop(k, None)
|
|
while len(self._store) > self._max:
|
|
self._store.popitem(last=False)
|
|
|
|
async def get_or_set(self, key: str, fingerprint: str, compute_coro):
|
|
self._purge()
|
|
item = self._store.get(key)
|
|
if item and item["fp"] == fingerprint:
|
|
return item["resp"]
|
|
|
|
inflight_key = (key, fingerprint)
|
|
task = self._inflight.get(inflight_key)
|
|
if task is None:
|
|
async def _compute_and_store():
|
|
resp = await compute_coro()
|
|
import time as _t
|
|
self._store[key] = {"resp": resp, "fp": fingerprint, "ts": _t.time()}
|
|
self._purge()
|
|
return resp
|
|
|
|
task = asyncio.create_task(_compute_and_store())
|
|
self._inflight[inflight_key] = task
|
|
|
|
def _clear_inflight(done_task: "asyncio.Task[Any]") -> None:
|
|
if self._inflight.get(inflight_key) is done_task:
|
|
self._inflight.pop(inflight_key, None)
|
|
|
|
task.add_done_callback(_clear_inflight)
|
|
|
|
return await asyncio.shield(task)
|
|
|
|
|
|
_idem_cache = _IdempotencyCache()
|
|
|
|
|
|
def _make_request_fingerprint(body: Dict[str, Any], keys: List[str]) -> str:
|
|
from hashlib import sha256
|
|
subset = {k: body.get(k) for k in keys}
|
|
return sha256(repr(subset).encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _derive_chat_session_id(
|
|
system_prompt: Optional[str],
|
|
first_user_message: str,
|
|
) -> str:
|
|
"""Derive a stable session ID from the conversation's first user message.
|
|
|
|
OpenAI-compatible frontends (Open WebUI, LibreChat, etc.) send the full
|
|
conversation history with every request. The system prompt and first user
|
|
message are constant across all turns of the same conversation, so hashing
|
|
them produces a deterministic session ID that lets the API server reuse
|
|
the same Hermes session (and therefore the same Docker container sandbox
|
|
directory) across turns.
|
|
"""
|
|
seed = f"{system_prompt or ''}\n{first_user_message}"
|
|
digest = hashlib.sha256(seed.encode("utf-8")).hexdigest()[:16]
|
|
return f"api-{digest}"
|
|
|
|
|
|
_CRON_AVAILABLE = False
|
|
try:
|
|
from cron.jobs import (
|
|
list_jobs as _cron_list,
|
|
get_job as _cron_get,
|
|
create_job as _cron_create,
|
|
update_job as _cron_update,
|
|
remove_job as _cron_remove,
|
|
pause_job as _cron_pause,
|
|
resume_job as _cron_resume,
|
|
trigger_job as _cron_trigger,
|
|
)
|
|
_CRON_AVAILABLE = True
|
|
except ImportError:
|
|
_cron_list = None
|
|
_cron_get = None
|
|
_cron_create = None
|
|
_cron_update = None
|
|
_cron_remove = None
|
|
_cron_pause = None
|
|
_cron_resume = None
|
|
_cron_trigger = None
|
|
|
|
|
|
def _notify_cron_provider_jobs_changed() -> None:
|
|
"""Tell the active cron scheduler provider the job set changed after a REST
|
|
mutation (no-op for the built-in). Best-effort — never breaks the handler."""
|
|
try:
|
|
from cron.scheduler import _notify_provider_jobs_changed
|
|
_notify_provider_jobs_changed()
|
|
except Exception:
|
|
pass
|
|
|
|
# Defense-in-depth: mirror the agent-facing cronjob tool, which scans the
|
|
# user-supplied prompt for exfiltration/injection payloads at create/update
|
|
# time (tools/cronjob_tools.py). The REST cron endpoints are authenticated
|
|
# (every handler runs _check_auth, and connect() refuses to start without
|
|
# API_SERVER_KEY), so this is not the trust boundary — it's parity with the
|
|
# tool path so a malicious prompt is rejected the same way regardless of
|
|
# which surface created the job. Imported defensively: a missing scanner
|
|
# must not disable the cron REST API.
|
|
try:
|
|
from tools.cronjob_tools import _scan_cron_prompt as _scan_cron_prompt
|
|
except Exception: # pragma: no cover - scanner is optional hardening
|
|
_scan_cron_prompt = None
|
|
|
|
|
|
class APIServerAdapter(BasePlatformAdapter):
|
|
"""
|
|
OpenAI-compatible HTTP API server adapter.
|
|
|
|
Runs an aiohttp web server that accepts OpenAI-format requests
|
|
and routes them through hermes-agent's AIAgent.
|
|
"""
|
|
|
|
# Stateless request/response: every route (the OpenAI-spec
|
|
# /v1/chat/completions and /v1/responses, and the proprietary /v1/runs SSE
|
|
# stream) tears down its channel when the turn ends. There is no persistent
|
|
# outbound channel to push a background completion to a client that already
|
|
# received its response, and ``send()`` is a no-op stub. So async-delivery
|
|
# tools (terminal notify_on_complete / watch_patterns, delegate_task
|
|
# background=True) must NOT promise delivery on this path — see
|
|
# ``async_delivery_supported()``.
|
|
supports_async_delivery: bool = False
|
|
|
|
def __init__(self, config: PlatformConfig):
|
|
super().__init__(config, Platform.API_SERVER)
|
|
extra = config.extra or {}
|
|
self._host: str = extra.get("host", os.getenv("API_SERVER_HOST", DEFAULT_HOST))
|
|
raw_port = extra.get("port")
|
|
if raw_port is None:
|
|
raw_port = os.getenv("API_SERVER_PORT", str(DEFAULT_PORT))
|
|
self._port: int = _coerce_port(raw_port, DEFAULT_PORT)
|
|
self._api_key: str = extra.get("key", os.getenv("API_SERVER_KEY", ""))
|
|
self._cors_origins: tuple[str, ...] = self._parse_cors_origins(
|
|
extra.get("cors_origins", os.getenv("API_SERVER_CORS_ORIGINS", "")),
|
|
)
|
|
self._model_name: str = self._resolve_model_name(
|
|
extra.get("model_name", os.getenv("API_SERVER_MODEL_NAME", "")),
|
|
)
|
|
# model_routes: maps incoming ``model`` field values to specific
|
|
# provider/model configs so one API server instance can serve
|
|
# multiple clients on different backends.
|
|
#
|
|
# Config format (platforms.api_server.extra in the gateway config):
|
|
# model_routes:
|
|
# minimax-m2: # alias the client sends as the "model" field
|
|
# model: "minimax/minimax-m1"
|
|
# provider: "openrouter" # optional — resolved via the provider
|
|
# # credential chain when set
|
|
# api_key: "sk-…" # optional — per-route UPSTREAM provider
|
|
# # key override (NOT caller auth; never logged)
|
|
# base_url: "https://…" # optional — per-route base URL override
|
|
self._model_routes: Dict[str, Dict[str, Any]] = self._parse_model_routes(
|
|
extra.get("model_routes"),
|
|
)
|
|
self._app: Optional["web.Application"] = None
|
|
self._runner: Optional["web.AppRunner"] = None
|
|
self._site: Optional["web.TCPSite"] = None
|
|
self._response_store = ResponseStore()
|
|
# Active run streams: run_id -> asyncio.Queue of SSE event dicts
|
|
self._run_streams: Dict[str, "asyncio.Queue[Optional[Dict]]"] = {}
|
|
# Creation timestamps for orphaned-run TTL sweep
|
|
self._run_streams_created: Dict[str, float] = {}
|
|
# Runs with a connected SSE consumer; their queue is actively draining.
|
|
self._run_stream_subscribers: set[str] = set()
|
|
# Active run agent/task references for stop support
|
|
self._active_run_agents: Dict[str, Any] = {}
|
|
self._active_run_tasks: Dict[str, "asyncio.Task"] = {}
|
|
# Stop is cooperative: the executor thread may outlive the HTTP request.
|
|
self._stopping_run_ids: set[str] = set()
|
|
# Pollable run status for dashboards and external control-plane UIs.
|
|
self._run_statuses: Dict[str, Dict[str, Any]] = {}
|
|
# Active approval session key for each run_id. The approval core
|
|
# resolves requests by session key, while API clients address the
|
|
# in-flight run by run_id.
|
|
self._run_approval_sessions: Dict[str, str] = {}
|
|
self._session_db: Optional[Any] = None # Lazy-init SessionDB for session continuity
|
|
# Concurrency cap shared across all agent-serving endpoints
|
|
# (/v1/chat/completions, /v1/responses, /v1/runs). Read from
|
|
# config.yaml gateway.api_server.max_concurrent_runs; 0 disables
|
|
# the cap. Bounds CPU / memory / upstream-LLM-quota exhaustion
|
|
# from a request flood (#7483).
|
|
self._max_concurrent_runs: int = self._resolve_max_concurrent_runs()
|
|
# Number of in-flight runs on the non-streaming chat/responses paths
|
|
# (the /v1/runs path tracks its own in-flight set via
|
|
# _active_run_tasks).
|
|
self._inflight_agent_runs: int = 0
|
|
# Requests admitted before their handler reaches agent bookkeeping.
|
|
# Shutdown counts this reservation so the request cannot slip through
|
|
# the drain between its first await and _run_agent()/task registration.
|
|
self._pending_agent_requests: int = 0
|
|
|
|
def active_agent_work_count(self) -> int:
|
|
"""Return all live agent work owned by this API adapter.
|
|
|
|
``/v1/runs`` registers an asyncio task before it constructs and stores
|
|
its agent, so ``_active_run_agents`` has a real queued-before-agent gap.
|
|
Reuse the task-based accounting used by the concurrent-run limit: it
|
|
covers that gap and excludes completed tasks retained until cleanup.
|
|
"""
|
|
try:
|
|
return (
|
|
int(getattr(self, "_pending_agent_requests", 0))
|
|
+ int(self._inflight_agent_runs)
|
|
+ sum(not task.done() for task in self._active_run_tasks.values())
|
|
)
|
|
except Exception:
|
|
return 0
|
|
|
|
@staticmethod
|
|
def _gateway_is_draining() -> bool:
|
|
"""Whether the owning gateway currently refuses new agent turns."""
|
|
try:
|
|
from gateway.run import _gateway_runner_ref
|
|
|
|
runner = _gateway_runner_ref()
|
|
return bool(
|
|
runner
|
|
and (
|
|
getattr(runner, "_draining", False)
|
|
or getattr(runner, "_external_drain_active", False)
|
|
)
|
|
)
|
|
except Exception:
|
|
return False
|
|
|
|
def _draining_response(self) -> Optional["web.Response"]:
|
|
"""Return a retryable response while the gateway drains existing work."""
|
|
if not self._gateway_is_draining():
|
|
return None
|
|
return web.json_response(
|
|
_openai_error(
|
|
"Gateway is draining existing work; retry shortly.",
|
|
code="gateway_draining",
|
|
),
|
|
status=503,
|
|
headers={"Retry-After": "1"},
|
|
)
|
|
|
|
def _activate_admitted_request(self) -> None:
|
|
"""Transfer this request's drain reservation to agent bookkeeping."""
|
|
reservation = _api_agent_request_reservation.get()
|
|
if reservation and reservation["active"]:
|
|
reservation["active"] = False
|
|
self._pending_agent_requests = max(0, self._pending_agent_requests - 1)
|
|
|
|
def _readiness_work_counts(self) -> tuple[int, int, int]:
|
|
"""Return bounded work counts from each subsystem's public state."""
|
|
active_api_runs = sum(
|
|
1
|
|
for status in self._run_statuses.values()
|
|
if status.get("status") in {"queued", "running", "waiting_for_approval"}
|
|
)
|
|
process_depth = 0
|
|
active_delegations = 0
|
|
try:
|
|
from tools.process_registry import process_registry
|
|
|
|
process_depth = process_registry.completion_queue.qsize()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
from tools.async_delegation import active_count
|
|
|
|
active_delegations = active_count()
|
|
except Exception:
|
|
pass
|
|
return active_api_runs, process_depth, active_delegations
|
|
|
|
@staticmethod
|
|
def _parse_cors_origins(value: Any) -> tuple[str, ...]:
|
|
"""Normalize configured CORS origins into a stable tuple."""
|
|
if not value:
|
|
return ()
|
|
|
|
if isinstance(value, str):
|
|
items = value.split(",")
|
|
elif isinstance(value, (list, tuple, set)):
|
|
items = value
|
|
else:
|
|
items = [str(value)]
|
|
|
|
return tuple(str(item).strip() for item in items if str(item).strip())
|
|
|
|
@staticmethod
|
|
def _resolve_max_concurrent_runs() -> int:
|
|
"""Read the concurrent-run cap from config.yaml (0 disables).
|
|
|
|
gateway.api_server.max_concurrent_runs. Falls back to the historical
|
|
default of 10 when unset or malformed. Negative values are clamped
|
|
to 0 (disabled).
|
|
"""
|
|
default = 10
|
|
try:
|
|
from hermes_cli.config import cfg_get, load_config
|
|
|
|
raw = cfg_get(
|
|
load_config(),
|
|
"gateway",
|
|
"api_server",
|
|
"max_concurrent_runs",
|
|
default=default,
|
|
)
|
|
value = int(raw)
|
|
except Exception:
|
|
return default
|
|
return max(0, value)
|
|
|
|
@staticmethod
|
|
def _resolve_model_name(explicit: str) -> str:
|
|
"""Derive the advertised model name for /v1/models.
|
|
|
|
Priority:
|
|
1. Explicit override (config extra or API_SERVER_MODEL_NAME env var)
|
|
2. Active profile name (so each profile advertises a distinct model)
|
|
3. Fallback: "hermes-agent"
|
|
"""
|
|
if explicit and explicit.strip():
|
|
return explicit.strip()
|
|
try:
|
|
from hermes_cli.profiles import get_active_profile_name
|
|
profile = get_active_profile_name()
|
|
if profile and profile not in {"default", "custom"}:
|
|
return profile
|
|
except Exception:
|
|
pass
|
|
return "hermes-agent"
|
|
|
|
def _cors_headers_for_origin(self, origin: str) -> Optional[Dict[str, str]]:
|
|
"""Return CORS headers for an allowed browser origin."""
|
|
if not origin or not self._cors_origins:
|
|
return None
|
|
|
|
if "*" in self._cors_origins:
|
|
headers = dict(_CORS_HEADERS)
|
|
headers["Access-Control-Allow-Origin"] = "*"
|
|
headers["Access-Control-Max-Age"] = "600"
|
|
return headers
|
|
|
|
if origin not in self._cors_origins:
|
|
return None
|
|
|
|
headers = dict(_CORS_HEADERS)
|
|
headers["Access-Control-Allow-Origin"] = origin
|
|
headers["Vary"] = "Origin"
|
|
headers["Access-Control-Max-Age"] = "600"
|
|
return headers
|
|
|
|
def _origin_allowed(self, origin: str) -> bool:
|
|
"""Allow non-browser clients and explicitly configured browser origins."""
|
|
if not origin:
|
|
return True
|
|
|
|
if not self._cors_origins:
|
|
return False
|
|
|
|
return "*" in self._cors_origins or origin in self._cors_origins
|
|
|
|
@staticmethod
|
|
def _clean_log_value(value: Any, *, max_len: int = 200) -> str:
|
|
"""Sanitize request metadata before it reaches security logs."""
|
|
if value is None:
|
|
return ""
|
|
text = str(value).replace("\r", " ").replace("\n", " ").strip()
|
|
return text[:max_len]
|
|
|
|
def _request_audit_context(self, request: "web.Request") -> Dict[str, str]:
|
|
"""Return non-secret source metadata for security/audit warnings."""
|
|
peer_ip = ""
|
|
try:
|
|
peer = request.transport.get_extra_info("peername") if request.transport else None
|
|
if isinstance(peer, (tuple, list)) and peer:
|
|
peer_ip = str(peer[0])
|
|
except Exception:
|
|
peer_ip = ""
|
|
|
|
return {
|
|
"remote": self._clean_log_value(getattr(request, "remote", "") or peer_ip),
|
|
"peer_ip": self._clean_log_value(peer_ip),
|
|
"forwarded_for": self._clean_log_value(request.headers.get("X-Forwarded-For", "")),
|
|
"real_ip": self._clean_log_value(request.headers.get("X-Real-IP", "")),
|
|
"method": self._clean_log_value(request.method, max_len=16),
|
|
"path": self._clean_log_value(request.path_qs, max_len=500),
|
|
"user_agent": self._clean_log_value(request.headers.get("User-Agent", ""), max_len=300),
|
|
}
|
|
|
|
def _request_audit_log_suffix(self, request: "web.Request") -> str:
|
|
ctx = self._request_audit_context(request)
|
|
fields = [f"{key}={value!r}" for key, value in ctx.items() if value]
|
|
return " ".join(fields) if fields else "source='unknown'"
|
|
|
|
def _cron_origin_from_request(self, request: "web.Request") -> Dict[str, str]:
|
|
"""Persist safe API source metadata on cron jobs created over HTTP."""
|
|
ctx = self._request_audit_context(request)
|
|
origin = {
|
|
"platform": "api_server",
|
|
"chat_id": "api",
|
|
}
|
|
if ctx.get("remote"):
|
|
origin["source_ip"] = ctx["remote"]
|
|
if ctx.get("peer_ip"):
|
|
origin["peer_ip"] = ctx["peer_ip"]
|
|
if ctx.get("forwarded_for"):
|
|
origin["forwarded_for"] = ctx["forwarded_for"]
|
|
if ctx.get("real_ip"):
|
|
origin["real_ip"] = ctx["real_ip"]
|
|
if ctx.get("user_agent"):
|
|
origin["user_agent"] = ctx["user_agent"]
|
|
return origin
|
|
|
|
# ------------------------------------------------------------------
|
|
# Auth helper
|
|
# ------------------------------------------------------------------
|
|
|
|
def _check_auth(self, request: "web.Request") -> Optional["web.Response"]:
|
|
"""
|
|
Validate Bearer token from Authorization header.
|
|
|
|
Returns None if auth is OK, or a 401 web.Response on failure.
|
|
connect() refuses to start the API server without API_SERVER_KEY, so
|
|
the no-key branch only exists for tests or unsupported manual wiring.
|
|
"""
|
|
if not self._api_key:
|
|
return None
|
|
|
|
auth_header = request.headers.get("Authorization", "")
|
|
if auth_header.startswith("Bearer "):
|
|
token = auth_header[7:].strip()
|
|
if hmac.compare_digest(token, self._api_key):
|
|
return None # Auth OK
|
|
|
|
logger.warning(
|
|
"API server rejected invalid API key: %s",
|
|
self._request_audit_log_suffix(request),
|
|
)
|
|
return web.json_response(
|
|
{"error": {"message": "Invalid API key", "type": "invalid_request_error", "code": "invalid_api_key"}},
|
|
status=401,
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Session header helpers
|
|
# ------------------------------------------------------------------
|
|
|
|
# Soft length cap for session identifiers. Headers are bounded in
|
|
# aggregate by aiohttp (``client_max_size`` / default 8 KiB per
|
|
# header), but we impose a tighter limit on the session headers so a
|
|
# caller can't burn memory by passing a multi-kilobyte "session key".
|
|
# 256 chars is well above any realistic stable channel identifier
|
|
# (e.g. ``agent:main:webui:dm:user-42``) while staying small enough
|
|
# that the sanitized form is safe to pass into Honcho / state.db.
|
|
_MAX_SESSION_HEADER_LEN = 256
|
|
|
|
def _parse_session_key_header(
|
|
self, request: "web.Request"
|
|
) -> tuple[Optional[str], Optional["web.Response"]]:
|
|
"""Extract and validate the ``X-Hermes-Session-Key`` header.
|
|
|
|
The session key is a stable per-channel identifier that scopes
|
|
long-term memory (e.g. Honcho sessions) across transcripts. It
|
|
is independent of ``X-Hermes-Session-Id``: callers may send
|
|
either, both, or neither.
|
|
|
|
Returns ``(session_key, None)`` on success (with an empty/absent
|
|
header yielding ``None`` for the key), or ``(None, error_response)``
|
|
on validation failure.
|
|
|
|
Security: like session continuation, accepting a caller-supplied
|
|
memory scope requires API-key authentication so that an
|
|
unauthenticated client on a local-only server can't inject itself
|
|
into another user's long-term memory scope by guessing a key.
|
|
"""
|
|
raw = request.headers.get("X-Hermes-Session-Key", "").strip()
|
|
if not raw:
|
|
return None, None
|
|
|
|
if not self._api_key:
|
|
logger.warning(
|
|
"X-Hermes-Session-Key rejected: no API key configured. "
|
|
"Set API_SERVER_KEY to enable long-term memory scoping."
|
|
)
|
|
return None, web.json_response(
|
|
_openai_error(
|
|
"X-Hermes-Session-Key requires API key authentication. "
|
|
"Configure API_SERVER_KEY to enable this feature."
|
|
),
|
|
status=403,
|
|
)
|
|
|
|
# Reject control characters that could enable header injection on
|
|
# the echo path.
|
|
if re.search(r'[\r\n\x00]', raw):
|
|
return None, web.json_response(
|
|
{"error": {"message": "Invalid session key", "type": "invalid_request_error"}},
|
|
status=400,
|
|
)
|
|
|
|
if len(raw) > self._MAX_SESSION_HEADER_LEN:
|
|
return None, web.json_response(
|
|
{"error": {"message": "Session key too long", "type": "invalid_request_error"}},
|
|
status=400,
|
|
)
|
|
|
|
return raw, None
|
|
|
|
# ------------------------------------------------------------------
|
|
# Session DB helper
|
|
# ------------------------------------------------------------------
|
|
|
|
def _ensure_session_db(self):
|
|
"""Lazily initialise and return the shared SessionDB instance.
|
|
|
|
Sessions are persisted to ``state.db`` so that ``hermes sessions list``
|
|
shows API-server conversations alongside CLI and gateway ones.
|
|
"""
|
|
if self._session_db is None:
|
|
try:
|
|
from hermes_state import SessionDB
|
|
self._session_db = SessionDB()
|
|
except Exception as e:
|
|
logger.debug("SessionDB unavailable for API server: %s", e)
|
|
return self._session_db
|
|
|
|
# ------------------------------------------------------------------
|
|
# Agent creation helper
|
|
# ------------------------------------------------------------------
|
|
|
|
@staticmethod
|
|
def _parse_model_routes(raw: Any) -> Dict[str, Dict[str, Any]]:
|
|
"""Validate and normalize the ``model_routes`` config block.
|
|
|
|
Accepts a mapping of ``alias -> {model, provider?, api_key?, base_url?}``.
|
|
Invalid shapes are dropped (never raised) so a config typo can't take
|
|
the whole API server down. Route values are coerced to strings.
|
|
|
|
Security: per-route ``api_key`` values are UPSTREAM provider
|
|
credentials (used to call the routed model's backend), not caller
|
|
authentication — callers still authenticate with the global
|
|
API_SERVER_KEY bearer token via ``_check_auth``. Route api_keys must
|
|
never be logged; only alias names and non-secret fields may appear in
|
|
logs.
|
|
"""
|
|
if not isinstance(raw, dict):
|
|
if raw:
|
|
logger.warning(
|
|
"api_server model_routes ignored: expected a mapping, got %s",
|
|
type(raw).__name__,
|
|
)
|
|
return {}
|
|
|
|
allowed_keys = ("model", "provider", "api_key", "base_url")
|
|
routes: Dict[str, Dict[str, Any]] = {}
|
|
for alias, cfg in raw.items():
|
|
alias_str = str(alias).strip()
|
|
if not alias_str or not isinstance(cfg, dict):
|
|
logger.warning(
|
|
"api_server model_routes: dropping invalid route entry %r", alias_str or alias
|
|
)
|
|
continue
|
|
route = {
|
|
key: str(cfg[key]).strip()
|
|
for key in allowed_keys
|
|
if cfg.get(key) is not None and str(cfg[key]).strip()
|
|
}
|
|
if not route.get("model"):
|
|
logger.warning(
|
|
"api_server model_routes: route %r has no 'model'; dropping", alias_str
|
|
)
|
|
continue
|
|
routes[alias_str] = route
|
|
return routes
|
|
|
|
def _resolve_route(self, model_alias: Any) -> Optional[Dict[str, Any]]:
|
|
"""Return the model_routes entry for *model_alias*, or None."""
|
|
if not self._model_routes or not isinstance(model_alias, str):
|
|
return None
|
|
return self._model_routes.get(model_alias)
|
|
|
|
def _session_model_override_for(self, session_key: Optional[str]) -> Optional[Dict[str, Any]]:
|
|
"""Return the gateway's session ``/model`` override for *session_key*, if any.
|
|
|
|
The gateway tracks per-session ``/model`` switches in
|
|
``GatewayRunner._session_model_overrides``. API-server requests that
|
|
share such a session key must keep honouring the explicit session
|
|
override even when the request's ``model`` field matches a configured
|
|
route — a user-issued ``/model`` always wins over static config.
|
|
"""
|
|
if not session_key:
|
|
return None
|
|
try:
|
|
from gateway.run import _gateway_runner_ref
|
|
runner = _gateway_runner_ref()
|
|
if runner is None:
|
|
return None
|
|
override = runner._session_model_overrides.get(session_key)
|
|
return dict(override) if isinstance(override, dict) else None
|
|
except Exception:
|
|
return None
|
|
|
|
def _create_agent(
|
|
self,
|
|
ephemeral_system_prompt: Optional[str] = None,
|
|
session_id: Optional[str] = None,
|
|
stream_delta_callback=None,
|
|
tool_progress_callback=None,
|
|
tool_start_callback=None,
|
|
tool_complete_callback=None,
|
|
gateway_session_key: Optional[str] = None,
|
|
route: Optional[Dict[str, Any]] = None,
|
|
) -> Any:
|
|
"""
|
|
Create an AIAgent instance using the gateway's runtime config.
|
|
|
|
Uses _resolve_runtime_agent_kwargs() to pick up model, api_key,
|
|
base_url, etc. from config.yaml / env vars. Toolsets are resolved
|
|
from config.yaml platform_toolsets.api_server (same as all other
|
|
gateway platforms), falling back to the hermes-api-server default.
|
|
|
|
``gateway_session_key`` is a stable per-channel identifier supplied
|
|
by the client (via ``X-Hermes-Session-Key``). Unlike ``session_id``
|
|
which scopes the short-term transcript and rotates on /new, this
|
|
key is meant to persist across transcripts so long-term memory
|
|
providers (e.g. Honcho) can scope their per-chat state correctly
|
|
— matching the semantics of the native gateway's ``session_key``.
|
|
|
|
``route`` is an optional ``model_routes`` entry (per-client model
|
|
routing). When set — and no session ``/model`` override exists for
|
|
this session — its model/provider/api_key/base_url override the
|
|
global defaults for this agent instance only.
|
|
"""
|
|
from run_agent import AIAgent
|
|
from gateway.run import (
|
|
_current_max_iterations,
|
|
_resolve_runtime_agent_kwargs,
|
|
_resolve_gateway_model,
|
|
_load_gateway_config,
|
|
GatewayRunner,
|
|
)
|
|
from hermes_cli.tools_config import _get_platform_tools
|
|
|
|
runtime_kwargs = _resolve_runtime_agent_kwargs()
|
|
reasoning_config = GatewayRunner._load_reasoning_config()
|
|
model = _resolve_gateway_model()
|
|
|
|
# When the primary provider's auth fails (expired token / 429 quota
|
|
# cap), _resolve_runtime_agent_kwargs() falls through to the fallback
|
|
# provider chain, whose runtime dict carries its own ``model`` key.
|
|
# Pop it and let it override the config model, mirroring the native
|
|
# gateway path (_resolve_session_agent_runtime in run.py). Otherwise
|
|
# the explicit ``model=model`` below collides with the ``**runtime_kwargs``
|
|
# spread → "got multiple values for keyword argument 'model'", 500ing
|
|
# every /v1/chat/completions request while a fallback is active.
|
|
runtime_model = runtime_kwargs.pop("model", None)
|
|
if runtime_model:
|
|
model = runtime_model
|
|
|
|
# Per-client model routing (model_routes config). The route was
|
|
# resolved from the request's ``model`` field by the HTTP handler.
|
|
# Precedence (highest first): session ``/model`` override → model_routes
|
|
# route → global config — an explicit user-issued ``/model`` on the
|
|
# session always beats static per-client route config.
|
|
session_override = self._session_model_override_for(
|
|
gateway_session_key or session_id
|
|
)
|
|
if route and not session_override:
|
|
if route.get("provider"):
|
|
# Resolve real credentials for the routed provider (mirrors
|
|
# the channel_overrides path in gateway/run.py) so a route
|
|
# without an explicit api_key/base_url still gets the right
|
|
# provider auth instead of the default provider's key.
|
|
try:
|
|
from gateway.run import _resolve_runtime_agent_kwargs_for_provider
|
|
provider_kwargs = _resolve_runtime_agent_kwargs_for_provider(
|
|
route["provider"]
|
|
)
|
|
provider_kwargs.pop("model", None)
|
|
runtime_kwargs.update(provider_kwargs)
|
|
except Exception:
|
|
# Fall back to just switching the provider name; explicit
|
|
# per-route api_key/base_url below can still complete auth.
|
|
runtime_kwargs["provider"] = route["provider"]
|
|
if route.get("model"):
|
|
model = route["model"]
|
|
# Per-route secrets are upstream provider credentials. Never log
|
|
# them (compare _check_auth: caller auth stays the global bearer
|
|
# key checked with hmac.compare_digest).
|
|
if route.get("api_key"):
|
|
runtime_kwargs["api_key"] = route["api_key"]
|
|
if route.get("base_url"):
|
|
runtime_kwargs["base_url"] = route["base_url"]
|
|
logger.debug(
|
|
"api_server model route applied: model=%s provider=%s",
|
|
model,
|
|
runtime_kwargs.get("provider"),
|
|
)
|
|
elif route and session_override:
|
|
logger.debug(
|
|
"api_server model route skipped: session /model override wins for %s",
|
|
gateway_session_key or session_id,
|
|
)
|
|
|
|
user_config = _load_gateway_config()
|
|
enabled_toolsets = sorted(_get_platform_tools(user_config, "api_server"))
|
|
|
|
max_iterations = _current_max_iterations()
|
|
|
|
# Load fallback provider chain so the API server platform has the
|
|
# same fallback behaviour as Telegram/Discord/Slack (fixes #4954).
|
|
fallback_model = GatewayRunner._load_fallback_model()
|
|
|
|
agent = AIAgent(
|
|
model=model,
|
|
**runtime_kwargs,
|
|
max_iterations=max_iterations,
|
|
quiet_mode=True,
|
|
verbose_logging=False,
|
|
ephemeral_system_prompt=ephemeral_system_prompt or None,
|
|
enabled_toolsets=enabled_toolsets,
|
|
session_id=session_id,
|
|
platform="api_server",
|
|
stream_delta_callback=stream_delta_callback,
|
|
tool_progress_callback=tool_progress_callback,
|
|
tool_start_callback=tool_start_callback,
|
|
tool_complete_callback=tool_complete_callback,
|
|
session_db=self._ensure_session_db(),
|
|
fallback_model=fallback_model,
|
|
reasoning_config=reasoning_config,
|
|
gateway_session_key=gateway_session_key,
|
|
)
|
|
return agent
|
|
|
|
# ------------------------------------------------------------------
|
|
# HTTP Handlers
|
|
# ------------------------------------------------------------------
|
|
|
|
async def _handle_health(self, request: "web.Request") -> "web.Response":
|
|
"""GET /health — simple health check."""
|
|
return web.json_response(
|
|
{"status": "ok", "platform": "hermes-agent", "version": _hermes_version()}
|
|
)
|
|
|
|
async def _handle_health_detailed(self, request: "web.Request") -> "web.Response":
|
|
"""GET /health/detailed — rich status for cross-container dashboard probing.
|
|
|
|
Returns gateway state, connected platforms, PID, and uptime so the
|
|
dashboard can display full status without needing a shared PID file or
|
|
/proc access. Requires the same Bearer auth as other API routes.
|
|
"""
|
|
auth_err = self._check_auth(request)
|
|
if auth_err:
|
|
return auth_err
|
|
|
|
from gateway.status import (
|
|
derive_gateway_busy,
|
|
derive_gateway_drainable,
|
|
parse_active_agents,
|
|
read_runtime_status,
|
|
)
|
|
|
|
runtime = read_runtime_status() or {}
|
|
gw_state = runtime.get("gateway_state")
|
|
gw_active = parse_active_agents(runtime.get("active_agents", 0))
|
|
# This endpoint is served BY the gateway process, so it is by definition
|
|
# alive — gateway_running is True. Derive busy/drainable from the same
|
|
# shared contract /api/status uses so the two surfaces never disagree.
|
|
active_api_runs, process_depth, active_delegations = self._readiness_work_counts()
|
|
from gateway.run import _resolve_gateway_model
|
|
|
|
readiness = collect_runtime_readiness(
|
|
configured_model=_resolve_gateway_model(),
|
|
runtime_status=runtime,
|
|
active_api_runs=active_api_runs,
|
|
process_completion_queue_depth=process_depth,
|
|
active_delegations=active_delegations,
|
|
)
|
|
return web.json_response({
|
|
"status": readiness["status"],
|
|
"readiness": readiness,
|
|
"platform": "hermes-agent",
|
|
"version": _hermes_version(),
|
|
"gateway_state": gw_state,
|
|
"platforms": runtime.get("platforms", {}),
|
|
"active_agents": gw_active,
|
|
"gateway_busy": derive_gateway_busy(
|
|
gateway_running=True,
|
|
gateway_state=gw_state,
|
|
active_agents=gw_active,
|
|
),
|
|
"gateway_drainable": derive_gateway_drainable(
|
|
gateway_running=True,
|
|
gateway_state=gw_state,
|
|
),
|
|
"exit_reason": runtime.get("exit_reason"),
|
|
"updated_at": runtime.get("updated_at"),
|
|
"pid": os.getpid(),
|
|
})
|
|
|
|
@staticmethod
|
|
def _audio_extension_for_content_type(content_type: str) -> str:
|
|
"""Map an inbound audio Content-Type to a safe cache-file suffix."""
|
|
media_type = (content_type or "").split(";", 1)[0].strip().lower()
|
|
return _ESP32_AUDIO_EXTENSIONS.get(media_type, ".wav")
|
|
|
|
@staticmethod
|
|
def _audio_response_mime(path: str) -> str:
|
|
"""Return an HTTP Content-Type for a generated audio file path."""
|
|
suffix = Path(path).suffix.lower()
|
|
return _AUDIO_RESPONSE_MIME_BY_SUFFIX.get(suffix, "application/octet-stream")
|
|
|
|
@staticmethod
|
|
|
|
def _convert_audio_to_esp32_wav(input_path: str, *, output_dir: Path, device_id: str) -> Path:
|
|
"""Normalize generated TTS audio to ESP32-friendly WAV.
|
|
|
|
Hermes TTS providers commonly emit MP3 (Edge) or OGG. The ESP32 test
|
|
client expects a small, predictable RIFF/WAVE file, so decode with
|
|
ffmpeg to raw PCM and rewrite the WAV container with Python's ``wave``
|
|
module. That produces a minimal ``fmt`` + ``data`` WAV: mono, 16 kHz,
|
|
signed 16-bit little-endian PCM.
|
|
"""
|
|
import subprocess
|
|
import wave
|
|
|
|
safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default"
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
raw_path = output_dir / f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}.s16le"
|
|
wav_path = raw_path.with_suffix(".wav")
|
|
cmd = [
|
|
"ffmpeg",
|
|
"-y",
|
|
"-hide_banner",
|
|
"-loglevel",
|
|
"error",
|
|
"-i",
|
|
input_path,
|
|
"-ac",
|
|
"1",
|
|
"-ar",
|
|
"16000",
|
|
"-f",
|
|
"s16le",
|
|
str(raw_path),
|
|
]
|
|
try:
|
|
subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
raw = raw_path.read_bytes()
|
|
with wave.open(str(wav_path), "wb") as wav:
|
|
wav.setnchannels(1)
|
|
wav.setsampwidth(2)
|
|
wav.setframerate(16000)
|
|
wav.writeframes(raw)
|
|
return wav_path
|
|
finally:
|
|
try:
|
|
raw_path.unlink(missing_ok=True)
|
|
except Exception:
|
|
pass
|
|
|
|
@staticmethod
|
|
def _safe_audio_header(value: Any, max_length: int) -> str:
|
|
"""Make diagnostic text safe for HTTP response headers."""
|
|
text = str(value or "")[:max_length]
|
|
return text.replace("\r", " ").replace("\n", " ").replace("\x00", " ")
|
|
|
|
@staticmethod
|
|
def _esp32_reply_mode(request: "web.Request") -> str:
|
|
"""Return 'sync' or 'ack' for the ESP32 voice response contract."""
|
|
value = (
|
|
request.headers.get("X-Hermes-Reply-Mode")
|
|
or request.query.get("reply_mode")
|
|
or request.query.get("mode")
|
|
or "sync"
|
|
)
|
|
value = str(value).strip().lower()
|
|
if value in {"ack", "async", "background", "quick_ack", "quick-ack"}:
|
|
return "ack"
|
|
return "sync"
|
|
|
|
@staticmethod
|
|
def _looks_like_http_url(value: str) -> bool:
|
|
try:
|
|
parsed = urlparse(value)
|
|
return parsed.scheme in {"http", "https"} and bool(parsed.netloc)
|
|
except Exception:
|
|
return False
|
|
|
|
@staticmethod
|
|
def _load_local_device_url(device_name: str) -> Optional[str]:
|
|
"""Best-effort local MCP screen lookup from ~/.hermes/local_devices.yaml."""
|
|
if not device_name:
|
|
return None
|
|
try:
|
|
import yaml
|
|
registry = Path.home() / ".hermes" / "local_devices.yaml"
|
|
data = yaml.safe_load(registry.read_text()) if registry.exists() else {}
|
|
aliases = data.get("aliases") if isinstance(data, dict) else None
|
|
devices = data.get("devices") if isinstance(data, dict) else None
|
|
names = [str(device_name).strip(), str(device_name).strip().replace("-", "_")]
|
|
seen: set[str] = set()
|
|
while names:
|
|
name = names.pop(0)
|
|
if not name or name in seen:
|
|
continue
|
|
seen.add(name)
|
|
if isinstance(aliases, dict):
|
|
alias = aliases.get(name)
|
|
if isinstance(alias, str) and alias:
|
|
if alias.startswith("http"):
|
|
return alias
|
|
names.append(alias)
|
|
info = devices.get(name) if isinstance(devices, dict) else None
|
|
if isinstance(info, dict):
|
|
return str(info.get("fallback_url") or info.get("url") or "") or None
|
|
except Exception:
|
|
return None
|
|
return None
|
|
|
|
@staticmethod
|
|
def _load_local_device_url_by_ip(source_ip: str) -> Optional[str]:
|
|
"""Best-effort local MCP screen lookup by observed device IP.
|
|
|
|
Some MCP-capable devices, notably the iPhone companion app, do not run
|
|
their MCP endpoint on port 80. When the voice gateway sees traffic from
|
|
one of those devices, prefer the registry URL over synthesizing
|
|
http://<source-ip>/api/mcp.
|
|
"""
|
|
value = str(source_ip or "").strip()
|
|
if not value:
|
|
return None
|
|
if value.startswith("::ffff:"):
|
|
value = value.removeprefix("::ffff:")
|
|
try:
|
|
import yaml
|
|
registry = Path.home() / ".hermes" / "local_devices.yaml"
|
|
data = yaml.safe_load(registry.read_text()) if registry.exists() else {}
|
|
devices = data.get("devices") if isinstance(data, dict) else None
|
|
if not isinstance(devices, dict):
|
|
return None
|
|
for info in devices.values():
|
|
if not isinstance(info, dict):
|
|
continue
|
|
candidates = {
|
|
str(info.get("reserved_ip") or "").strip(),
|
|
str(info.get("host") or "").strip(),
|
|
}
|
|
for url_key in ("fallback_url", "url"):
|
|
raw_url = str(info.get(url_key) or "").strip()
|
|
if raw_url:
|
|
parsed = urlparse(raw_url)
|
|
if parsed.hostname:
|
|
candidates.add(parsed.hostname)
|
|
if value in candidates:
|
|
return str(info.get("fallback_url") or info.get("url") or "") or None
|
|
except Exception:
|
|
return None
|
|
return None
|
|
|
|
@staticmethod
|
|
def _esp32_screen_url_from_source_ip(source_ip: str) -> Optional[str]:
|
|
"""Build the MCP endpoint for the ESP32 that made this request.
|
|
|
|
ESP32 boards can move between DHCP leases. For voice ACK follow-ups the
|
|
initiating board is the safest target, so prefer its observed peer IP
|
|
instead of a possibly stale local-device registry entry.
|
|
"""
|
|
value = str(source_ip or "").strip()
|
|
if not value:
|
|
return None
|
|
if value.startswith("::ffff:"):
|
|
value = value.removeprefix("::ffff:")
|
|
# Do not route to loopback/proxy/invalid addresses; registry fallback is
|
|
# better in those cases.
|
|
try:
|
|
ip = ipaddress.ip_address(value)
|
|
except ValueError:
|
|
return None
|
|
if ip.is_loopback or ip.is_unspecified or ip.is_multicast:
|
|
return None
|
|
host = f"[{value}]" if ip.version == 6 else value
|
|
return f"http://{host}/api/mcp"
|
|
|
|
def _resolve_esp32_screen_url(self, request: "web.Request", device_id: str) -> Optional[str]:
|
|
"""Resolve the screen MCP endpoint associated with this voice turn."""
|
|
explicit = (
|
|
request.headers.get("X-Hermes-Screen-Url")
|
|
or request.headers.get("X-Screen-MCP-Url")
|
|
or request.query.get("screen_url")
|
|
)
|
|
if explicit and self._looks_like_http_url(explicit.strip()):
|
|
return explicit.strip()
|
|
|
|
if str(device_id or "").strip().lower().startswith("iphone"):
|
|
iphone_url = self._load_local_device_url(str(device_id).strip())
|
|
if iphone_url and self._looks_like_http_url(iphone_url):
|
|
return iphone_url
|
|
|
|
audit_ctx = self._request_audit_context(request)
|
|
for source_ip in (audit_ctx.get("peer_ip"), audit_ctx.get("remote")):
|
|
registry_url = self._load_local_device_url_by_ip(source_ip or "")
|
|
if registry_url and self._looks_like_http_url(registry_url):
|
|
return registry_url
|
|
|
|
audit_ctx = self._request_audit_context(request)
|
|
for source_ip in (audit_ctx.get("peer_ip"), audit_ctx.get("remote")):
|
|
source_url = self._esp32_screen_url_from_source_ip(source_ip or "")
|
|
if source_url:
|
|
return source_url
|
|
|
|
screen_device = (
|
|
request.headers.get("X-Hermes-Screen-Device")
|
|
or request.query.get("screen_device")
|
|
or device_id
|
|
)
|
|
candidates = [screen_device, "little32", "esp32_screen"]
|
|
for candidate in candidates:
|
|
url = self._load_local_device_url(str(candidate).strip())
|
|
if url and self._looks_like_http_url(url):
|
|
return url
|
|
return None
|
|
|
|
async def _call_screen_mcp(
|
|
self,
|
|
screen_url: Optional[str],
|
|
tool_name: str,
|
|
arguments: Dict[str, Any],
|
|
*,
|
|
timeout: float = 8.0,
|
|
) -> bool:
|
|
"""Call a simple HTTP JSON-RPC MCP screen tool directly."""
|
|
if not screen_url:
|
|
return False
|
|
try:
|
|
from aiohttp import ClientSession, ClientTimeout
|
|
payload = {
|
|
"jsonrpc": "2.0",
|
|
"id": uuid.uuid4().hex[:12],
|
|
"method": "tools/call",
|
|
"params": {"name": tool_name, "arguments": arguments},
|
|
}
|
|
async with ClientSession(timeout=ClientTimeout(total=timeout)) as session:
|
|
async with session.post(screen_url, json=payload) as resp:
|
|
body = await resp.text()
|
|
if resp.status >= 400:
|
|
logger.warning(
|
|
"[api_server] screen MCP %s failed: HTTP %s %s",
|
|
tool_name,
|
|
resp.status,
|
|
body[:300],
|
|
)
|
|
return False
|
|
try:
|
|
data = json.loads(body) if body else {}
|
|
if isinstance(data, dict) and data.get("error"):
|
|
logger.warning("[api_server] screen MCP %s returned error: %s", tool_name, data.get("error"))
|
|
return False
|
|
except Exception:
|
|
pass
|
|
return True
|
|
except Exception as exc:
|
|
logger.warning("[api_server] screen MCP %s failed: %s", tool_name, exc)
|
|
return False
|
|
|
|
@staticmethod
|
|
def _screen_stream_target(screen_url: Optional[str]) -> Optional[tuple[str, int]]:
|
|
"""Return the raw UDP video-stream target for MCP screens that expose one."""
|
|
if not screen_url:
|
|
return None
|
|
try:
|
|
parsed = urlparse(screen_url)
|
|
host = parsed.hostname
|
|
if not host:
|
|
return None
|
|
return (host, 8082)
|
|
except Exception:
|
|
return None
|
|
|
|
@staticmethod
|
|
def _voice_animation_udp_frames(state: str, text: str = "", frames: int = 18) -> List[bytes]:
|
|
"""Render small Hermes voice-state animations as RLCD-compatible frames."""
|
|
from PIL import Image, ImageDraw
|
|
import math
|
|
|
|
width, height = 400, 300
|
|
|
|
def to_rlcd(img: "Image.Image") -> bytes:
|
|
img_1bit = img.convert("1", dither=Image.Dither.FLOYDSTEINBERG)
|
|
px = img_1bit.load()
|
|
buf = bytearray(15000)
|
|
for y in range(height):
|
|
for x in range(width):
|
|
if px[x, y]:
|
|
inv_y = 299 - y
|
|
bx = x // 2
|
|
by = inv_y // 4
|
|
idx = bx * 75 + by
|
|
lx = x % 2
|
|
ly = inv_y % 4
|
|
bit = 7 - (ly * 2 + lx)
|
|
buf[idx] |= (1 << bit)
|
|
return bytes(buf)
|
|
|
|
label_by_state = {
|
|
"ack": "Got it",
|
|
"captioning": "Listening",
|
|
"waiting": "Thinking",
|
|
"speaking": "Replying",
|
|
"success": "Done",
|
|
"error": "Try again",
|
|
}
|
|
label = label_by_state.get(str(state), str(state or "Hermes").title())
|
|
snippet = " ".join(str(text or "").split())[:42]
|
|
rendered: List[bytes] = []
|
|
for frame in range(max(1, int(frames))):
|
|
img = Image.new("1", (width, height), 0)
|
|
draw = ImageDraw.Draw(img)
|
|
cx, cy = width // 2, height // 2
|
|
phase = frame * 0.35
|
|
if state == "captioning":
|
|
draw.rectangle((188, 38, 212, 75), outline=1)
|
|
draw.arc((175, 55, 225, 105), 20, 160, fill=1)
|
|
for x in range(42, width - 42, 5):
|
|
y = cy + int(32 * math.sin(x * 0.045 + phase * 2))
|
|
y2 = cy + int(14 * math.sin(x * 0.085 - phase * 3))
|
|
draw.line((x, y, min(width - 1, x + 4), y2), fill=1)
|
|
elif state in {"waiting", "speaking"}:
|
|
for radius in (34, 64, 94):
|
|
pulse = radius + int(8 * math.sin(phase + radius))
|
|
draw.ellipse((cx - pulse, cy - pulse, cx + pulse, cy + pulse), outline=1)
|
|
for i in range(7):
|
|
x = 93 + i * 34
|
|
bar = int(18 + 42 * (0.5 + 0.5 * math.sin(phase * 2 + i)))
|
|
draw.rectangle((x, 228 - bar, x + 17, 228), fill=1)
|
|
elif state == "error":
|
|
draw.line((150, 88, 250, 188), fill=1)
|
|
draw.line((250, 88, 150, 188), fill=1)
|
|
draw.rectangle((145, 82, 255, 194), outline=1)
|
|
else:
|
|
r = 42 + int(16 * math.sin(phase))
|
|
draw.ellipse((cx - r, cy - r, cx + r, cy + r), outline=1)
|
|
draw.ellipse((cx - 8, cy - 8, cx + 8, cy + 8), fill=1)
|
|
draw.text((max(8, (width - len(label) * 6) // 2), 24), label, fill=1)
|
|
if snippet:
|
|
draw.text((max(8, (width - len(snippet) * 6) // 2), 260), snippet, fill=1)
|
|
rendered.append(to_rlcd(img))
|
|
return rendered
|
|
|
|
async def _send_stream_animation(
|
|
self,
|
|
screen_url: Optional[str],
|
|
state: str,
|
|
*,
|
|
text: str = "",
|
|
frames: Optional[int] = None,
|
|
) -> bool:
|
|
"""Send a server-rendered voice animation over the screen streaming socket."""
|
|
target = self._screen_stream_target(screen_url)
|
|
if not target:
|
|
return False
|
|
frame_count = frames if frames is not None else (12 if state == "ack" else 18)
|
|
try:
|
|
rendered = await asyncio.to_thread(self._voice_animation_udp_frames, state, text, frame_count)
|
|
sock = _socket.socket(_socket.AF_INET, _socket.SOCK_DGRAM)
|
|
try:
|
|
for frame_id, frame in enumerate(rendered):
|
|
fid = frame_id % 256
|
|
for chunk_idx in range(15):
|
|
start = chunk_idx * 1000
|
|
sock.sendto(bytes([fid, chunk_idx]) + frame[start : start + 1000], target)
|
|
await asyncio.sleep(0.045)
|
|
finally:
|
|
sock.close()
|
|
return True
|
|
except Exception as exc:
|
|
logger.warning("[api_server] screen stream animation failed: %s", exc)
|
|
return False
|
|
|
|
async def _send_screen_animation(
|
|
self,
|
|
screen_url: Optional[str],
|
|
state: str,
|
|
*,
|
|
text: str = "",
|
|
frames: Optional[int] = None,
|
|
) -> bool:
|
|
"""Upload and run the ESP32 voice-state animation script on the initiating screen."""
|
|
if not screen_url:
|
|
return False
|
|
if await self._send_stream_animation(screen_url, state, text=text, frames=frames):
|
|
return True
|
|
if screen_url not in self._esp32_voice_animation_uploaded:
|
|
uploaded = await self._call_screen_mcp(
|
|
screen_url,
|
|
"write_file",
|
|
{"path": ESP32_VOICE_ANIMATION_PATH, "content": ESP32_VOICE_ANIMATION_SCRIPT},
|
|
)
|
|
if not uploaded:
|
|
return False
|
|
self._esp32_voice_animation_uploaded.add(screen_url)
|
|
code = (
|
|
f"exec(open({ESP32_VOICE_ANIMATION_PATH!r}).read()); "
|
|
f"run_state({str(state)!r}, text={str(text or '')[:240]!r}, frames={frames!r})"
|
|
)
|
|
return await self._call_screen_mcp(screen_url, "execute_python", {"code": code})
|
|
|
|
async def _send_screen_text(self, screen_url: Optional[str], text: str, *, clear: bool = False) -> None:
|
|
"""Best-effort text update to the initiating MCP screen."""
|
|
if not screen_url:
|
|
return
|
|
if clear:
|
|
await self._call_screen_mcp(screen_url, "clear_screen", {"color": 0})
|
|
# Most Reyna screens accept x/y/text. Extra wrapping is avoided here so
|
|
# firmware-specific clients can keep their own font/layout behavior.
|
|
await self._call_screen_mcp(screen_url, "draw_text", {"x": 8, "y": 8, "text": text[:900]})
|
|
|
|
@staticmethod
|
|
def _extract_first_image_reference(text: str) -> Optional[str]:
|
|
"""Return the first local path, MEDIA path, data URL, or HTTP image URL in text."""
|
|
if not text:
|
|
return None
|
|
patterns = [
|
|
r"MEDIA:([^\s)]+)",
|
|
r"!\[[^\]]*\]\(([^)]+)\)",
|
|
r"(?P<url>https?://[^\s)]+\.(?:png|jpe?g|webp|gif)(?:\?[^\s)]*)?)",
|
|
r"(?P<path>/[^\s)]+\.(?:png|jpe?g|webp|gif|bmp))",
|
|
r"(?P<data>data:image/[^\s)]+)",
|
|
]
|
|
for pattern in patterns:
|
|
for match in re.finditer(pattern, text, flags=re.IGNORECASE):
|
|
ref = match.groupdict().get("url") or match.groupdict().get("path") or match.groupdict().get("data") or match.group(1)
|
|
if ref:
|
|
return ref.strip().strip('"\'')
|
|
return None
|
|
|
|
async def _send_screen_image_response(self, screen_url: Optional[str], text: str) -> bool:
|
|
"""Best-effort render of the first generated/linked image onto the ESP32 screen."""
|
|
if not screen_url:
|
|
return False
|
|
image_ref = self._extract_first_image_reference(text)
|
|
if not image_ref:
|
|
return False
|
|
try:
|
|
import base64
|
|
import io
|
|
from PIL import Image, ImageOps
|
|
|
|
raw: bytes
|
|
lowered = image_ref.lower()
|
|
if lowered.startswith("data:image/"):
|
|
_, _, encoded = image_ref.partition(",")
|
|
raw = base64.b64decode(encoded)
|
|
elif self._looks_like_http_url(image_ref):
|
|
from aiohttp import ClientSession, ClientTimeout
|
|
async with ClientSession(timeout=ClientTimeout(total=20)) as session:
|
|
async with session.get(image_ref) as resp:
|
|
if resp.status >= 400:
|
|
logger.warning("[api_server] screen image fetch failed: HTTP %s", resp.status)
|
|
return False
|
|
raw = await resp.read()
|
|
else:
|
|
path = Path(image_ref).expanduser()
|
|
if not path.exists() or not path.is_file():
|
|
return False
|
|
raw = await asyncio.to_thread(path.read_bytes)
|
|
if not raw:
|
|
return False
|
|
|
|
def render_pbm() -> str:
|
|
with Image.open(io.BytesIO(raw)) as img:
|
|
img = ImageOps.exif_transpose(img).convert("RGB")
|
|
canvas = Image.new("RGB", (320, 240), "white")
|
|
img.thumbnail((320, 240), Image.Resampling.LANCZOS)
|
|
x = (320 - img.width) // 2
|
|
y = (240 - img.height) // 2
|
|
canvas.paste(img, (x, y))
|
|
mono = canvas.convert("1", dither=Image.Dither.FLOYDSTEINBERG)
|
|
out = io.BytesIO()
|
|
mono.save(out, format="PPM") # mode=1 writes raw PBM (P4)
|
|
return base64.b64encode(out.getvalue()).decode("ascii")
|
|
|
|
pbm_base64 = await asyncio.to_thread(render_pbm)
|
|
return await self._call_screen_mcp(
|
|
screen_url,
|
|
"draw_image",
|
|
{"pbm_base64": pbm_base64, "x": 0, "y": 0, "dither": False},
|
|
timeout=15,
|
|
)
|
|
except Exception:
|
|
logger.exception("[api_server] failed to render ESP32 screen image response")
|
|
return False
|
|
|
|
async def _send_screen_audio_response(
|
|
self,
|
|
screen_url: Optional[str],
|
|
text: str,
|
|
*,
|
|
audio_dir: Path,
|
|
device_id: str,
|
|
volume: int = 80,
|
|
) -> bool:
|
|
"""Best-effort spoken follow-up for ACK-mode ESP32 voice turns."""
|
|
if not screen_url or not text.strip():
|
|
return False
|
|
try:
|
|
import base64
|
|
from tools.tts_tool import text_to_speech_tool
|
|
|
|
# Keep board playback short and payloads small. The ESP32 HTTP MCP
|
|
# endpoint resets on large base64 WAV bodies; MP3 is much smaller
|
|
# and the firmware exposes play_mp3_base64 for exactly this path.
|
|
spoken_text = self._short_voice_summary(text, 420)
|
|
tts_raw = await asyncio.to_thread(text_to_speech_tool, spoken_text)
|
|
tts_result = json.loads(tts_raw)
|
|
src = str(tts_result.get("file_path") or "")
|
|
if not tts_result.get("success") or not src or not Path(src).exists():
|
|
logger.warning(
|
|
"[api_server] screen audio TTS failed: %s",
|
|
tts_result.get("error") or "missing TTS output",
|
|
)
|
|
return False
|
|
|
|
src_path = Path(src)
|
|
|
|
# Detect if this is iPhone (has no play_mp3_base64, only play_audio_base64)
|
|
is_iphone = "iphone" in str(device_id).lower() or ":8080" in str(screen_url or "")
|
|
|
|
if not is_iphone and src_path.suffix.lower() == ".mp3":
|
|
mp3_base64 = base64.b64encode(src_path.read_bytes()).decode("ascii")
|
|
if await self._call_screen_mcp(
|
|
screen_url,
|
|
"play_mp3_base64",
|
|
{"mp3_base64": mp3_base64, "volume": volume},
|
|
timeout=75,
|
|
):
|
|
return True
|
|
|
|
wav_path = await asyncio.to_thread(
|
|
self._convert_audio_to_esp32_wav,
|
|
src,
|
|
output_dir=audio_dir,
|
|
device_id=f"{device_id}_response",
|
|
)
|
|
wav_bytes = wav_path.read_bytes()
|
|
# iPhone can handle larger WAVs than ESP32 (tested 180k+), increase limit for iPhone
|
|
max_wav = 600_000 if is_iphone else 220_000
|
|
if len(wav_bytes) > max_wav:
|
|
# For iPhone, truncate to first 6 seconds to fit
|
|
if is_iphone:
|
|
import wave, io
|
|
try:
|
|
with wave.open(str(wav_path), "rb") as wf:
|
|
fr = wf.getframerate()
|
|
nframes = int(fr * 6.0)
|
|
wf.rewind()
|
|
frames = wf.readframes(nframes)
|
|
bio = io.BytesIO()
|
|
with wave.open(bio, "wb") as out:
|
|
out.setnchannels(wf.getnchannels())
|
|
out.setsampwidth(wf.getsampwidth())
|
|
out.setframerate(fr)
|
|
out.writeframes(frames)
|
|
wav_bytes = bio.getvalue()
|
|
except Exception:
|
|
pass
|
|
else:
|
|
logger.warning(
|
|
"[api_server] screen audio WAV too large for MCP payload: %s bytes",
|
|
len(wav_bytes),
|
|
)
|
|
return False
|
|
wav_base64 = base64.b64encode(wav_bytes).decode("ascii")
|
|
return await self._call_screen_mcp(
|
|
screen_url,
|
|
"play_audio_base64",
|
|
{"wav_base64": wav_base64, "volume": volume},
|
|
timeout=45,
|
|
)
|
|
except Exception:
|
|
logger.exception("[api_server] failed to play ESP32 screen audio response")
|
|
return False
|
|
|
|
@staticmethod
|
|
def _short_voice_summary(text: str, limit: int = 220) -> str:
|
|
"""Local fallback summary for voice/screen snippets without another LLM call."""
|
|
clean = " ".join(str(text or "").split())
|
|
if len(clean) <= limit:
|
|
return clean
|
|
return clean[: max(0, limit - 1)].rstrip() + "…"
|
|
|
|
async def _esp32_ack_wav(self, *, audio_dir: Path, device_id: str) -> Path:
|
|
"""Return a reusable quiet ACK WAV; the visual screen animation is the acknowledgement."""
|
|
ack_path = audio_dir / "quick_ack.wav"
|
|
if ack_path.exists():
|
|
return ack_path
|
|
|
|
# ACK-mode clients expect a valid audio/wav response immediately, but
|
|
# Reyna ESP32 screens now show the acknowledgement visually. Keep the
|
|
# HTTP contract while avoiding the old spoken "Got it" message.
|
|
import wave
|
|
sample_rate = 16000
|
|
duration_s = 0.08
|
|
frames = b"\x00\x00" * int(sample_rate * duration_s)
|
|
with wave.open(str(ack_path), "wb") as wav:
|
|
wav.setnchannels(1)
|
|
wav.setsampwidth(2)
|
|
wav.setframerate(sample_rate)
|
|
wav.writeframes(frames)
|
|
return ack_path
|
|
|
|
def _append_esp32_voice_audit(self, audio_dir: Path, event: Dict[str, Any]) -> None:
|
|
"""Append a best-effort ESP32 voice request audit event."""
|
|
try:
|
|
log_path = audio_dir / "voice_requests.jsonl"
|
|
with log_path.open("a", encoding="utf-8") as handle:
|
|
handle.write(json.dumps(event, default=str) + "\n")
|
|
except Exception:
|
|
logger.debug("[api_server] failed to append ESP32 voice audit log", exc_info=True)
|
|
|
|
async def _run_esp32_voice_audio_background(
|
|
self,
|
|
*,
|
|
audio_path: Path,
|
|
audit_event: Dict[str, Any],
|
|
device_id: str,
|
|
instructions: Optional[str],
|
|
audio_dir: Path,
|
|
screen_url: Optional[str],
|
|
) -> None:
|
|
"""Transcribe uploaded ESP32 audio and continue the voice turn after an immediate ACK."""
|
|
try:
|
|
await self._send_screen_animation(screen_url, "captioning")
|
|
from tools.transcription_tools import transcribe_audio
|
|
stt_result = await asyncio.to_thread(transcribe_audio, str(audio_path))
|
|
if not stt_result.get("success"):
|
|
failed_event = dict(audit_event)
|
|
failed_event.update({
|
|
"status": "transcription_failed",
|
|
"error": stt_result.get("error") or "Transcription failed",
|
|
"timestamp": time.time(),
|
|
})
|
|
self._append_esp32_voice_audit(audio_dir, failed_event)
|
|
await self._send_screen_text(
|
|
screen_url,
|
|
"I heard audio, but transcription failed. Try again.",
|
|
clear=True,
|
|
)
|
|
return
|
|
transcript = str(stt_result.get("transcript") or "").strip()
|
|
if not transcript:
|
|
empty_event = dict(audit_event)
|
|
empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()})
|
|
self._append_esp32_voice_audit(audio_dir, empty_event)
|
|
await self._send_screen_text(
|
|
screen_url,
|
|
"I heard audio, but got no words. Try again.",
|
|
clear=True,
|
|
)
|
|
return
|
|
transcript_event = dict(audit_event)
|
|
transcript_event.update({"status": "transcribed", "transcript": transcript, "timestamp": time.time()})
|
|
self._append_esp32_voice_audit(audio_dir, transcript_event)
|
|
await self._run_esp32_voice_background(
|
|
transcript=transcript,
|
|
device_id=device_id,
|
|
instructions=instructions,
|
|
audio_dir=audio_dir,
|
|
screen_url=screen_url,
|
|
)
|
|
except Exception as exc:
|
|
logger.exception("[api_server] ESP32 ACK background voice processing failed")
|
|
failed_event = dict(audit_event)
|
|
failed_event.update({"status": "background_exception", "error": str(exc), "timestamp": time.time()})
|
|
self._append_esp32_voice_audit(audio_dir, failed_event)
|
|
await self._send_screen_text(
|
|
screen_url,
|
|
"I got the audio, but background processing failed.",
|
|
clear=True,
|
|
)
|
|
|
|
async def _run_esp32_voice_background(
|
|
self,
|
|
*,
|
|
transcript: str,
|
|
device_id: str,
|
|
instructions: Optional[str],
|
|
audio_dir: Path,
|
|
screen_url: Optional[str],
|
|
) -> None:
|
|
"""Run the slow ESP32 voice turn after the device already heard an ACK."""
|
|
try:
|
|
animation_ok = await self._send_screen_animation(screen_url, "waiting", text=transcript)
|
|
if not animation_ok:
|
|
await self._send_screen_text(
|
|
screen_url,
|
|
f"Heard: {self._short_voice_summary(transcript, 180)}\n\nWorking…",
|
|
clear=True,
|
|
)
|
|
screen_hint = (
|
|
"\n\nThis request came from an ESP32 voice/screen device. "
|
|
"Keep the spoken follow-up short. The gateway will display the full answer on the initiating screen when possible."
|
|
)
|
|
merged_instructions = (instructions or "") + screen_hint
|
|
response_data, final_text = await self._run_esp32_voice_turn(
|
|
transcript=transcript,
|
|
device_id=device_id,
|
|
instructions=merged_instructions,
|
|
)
|
|
if not final_text.strip():
|
|
final_text = self._extract_response_text(response_data) or "I finished, but I do not have a text answer."
|
|
display_text = final_text
|
|
if len(display_text) > 1200:
|
|
display_text = self._short_voice_summary(display_text, 900) + "\n\nFull answer is long; WhatsApp delivery is the next wiring step."
|
|
await self._send_screen_text(screen_url, display_text, clear=True)
|
|
await self._send_screen_image_response(screen_url, final_text)
|
|
await self._send_screen_audio_response(
|
|
screen_url,
|
|
final_text,
|
|
audio_dir=audio_dir,
|
|
device_id=device_id,
|
|
)
|
|
except Exception:
|
|
logger.exception("[api_server] ESP32 background voice turn failed")
|
|
await self._send_screen_text(screen_url, "Hermes hit an error while preparing the full answer.", clear=True)
|
|
|
|
@staticmethod
|
|
def _extract_response_text(response_data: Dict[str, Any]) -> str:
|
|
"""Extract assistant text from an OpenAI Responses-shaped payload."""
|
|
chunks: List[str] = []
|
|
for item in response_data.get("output") or []:
|
|
if not isinstance(item, dict) or item.get("type") != "message":
|
|
continue
|
|
for part in item.get("content") or []:
|
|
if isinstance(part, dict) and part.get("type") in {"output_text", "text"}:
|
|
text = part.get("text")
|
|
if isinstance(text, str) and text:
|
|
chunks.append(text)
|
|
return "\n".join(chunks).strip()
|
|
|
|
async def _run_esp32_voice_turn(
|
|
self,
|
|
*,
|
|
transcript: str,
|
|
device_id: str,
|
|
instructions: Optional[str] = None,
|
|
) -> tuple[Dict[str, Any], str]:
|
|
"""Run one persistent Responses-style turn for an ESP32 voice device."""
|
|
conversation = f"esp32:{device_id}"
|
|
previous_response_id = self._response_store.get_conversation(conversation)
|
|
conversation_history: List[Dict[str, Any]] = []
|
|
stored_session_id = None
|
|
stored_instructions = None
|
|
|
|
if previous_response_id:
|
|
stored = self._response_store.get(previous_response_id)
|
|
if stored is not None:
|
|
conversation_history = list(stored.get("conversation_history", []))
|
|
stored_session_id = stored.get("session_id")
|
|
stored_instructions = stored.get("instructions")
|
|
|
|
if instructions is None:
|
|
instructions = stored_instructions
|
|
|
|
session_id = stored_session_id or str(uuid.uuid4())
|
|
user_message = (
|
|
f'[Voice input from ESP32 device "{device_id}"]\n'
|
|
f'{transcript}'
|
|
)
|
|
|
|
result, usage = await self._run_agent(
|
|
user_message=user_message,
|
|
conversation_history=conversation_history,
|
|
ephemeral_system_prompt=instructions,
|
|
session_id=session_id,
|
|
gateway_session_key=conversation,
|
|
)
|
|
|
|
final_response = result.get("final_response", "") or result.get("error", "") or "(No response generated)"
|
|
response_id = f"resp_{uuid.uuid4().hex[:28]}"
|
|
created_at = int(time.time())
|
|
full_history = self._build_response_conversation_history(
|
|
conversation_history,
|
|
user_message,
|
|
result,
|
|
final_response,
|
|
)
|
|
output_start_index = self._response_messages_turn_start_index(
|
|
conversation_history,
|
|
user_message,
|
|
result,
|
|
)
|
|
output_items = self._extract_output_items(result, start_index=output_start_index)
|
|
response_data = {
|
|
"id": response_id,
|
|
"object": "response",
|
|
"status": "completed",
|
|
"created_at": created_at,
|
|
"model": self._model_name,
|
|
"output": output_items,
|
|
"usage": {
|
|
"input_tokens": usage.get("input_tokens", 0),
|
|
"output_tokens": usage.get("output_tokens", 0),
|
|
"total_tokens": usage.get("total_tokens", 0),
|
|
},
|
|
}
|
|
self._response_store.put(response_id, {
|
|
"response": response_data,
|
|
"conversation_history": full_history,
|
|
"instructions": instructions,
|
|
"session_id": session_id,
|
|
})
|
|
self._response_store.set_conversation(conversation, response_id)
|
|
return response_data, final_response
|
|
|
|
@staticmethod
|
|
def _esp32_audio_cache_dir() -> Path:
|
|
"""Return the cache directory for ESP32 voice artifacts."""
|
|
try:
|
|
from hermes_constants import get_hermes_dir
|
|
return get_hermes_dir("cache/audio", "audio_cache") / "esp32"
|
|
except Exception:
|
|
return Path.home() / ".hermes" / "cache" / "audio" / "esp32"
|
|
|
|
@staticmethod
|
|
def _write_pcm_s16le_wav(raw_path: Path, wav_path: Path, *, sample_rate: int, channels: int = 1) -> None:
|
|
"""Wrap a raw PCM s16le file in a minimal RIFF/WAVE container."""
|
|
import wave
|
|
|
|
with wave.open(str(wav_path), "wb") as wav:
|
|
wav.setnchannels(channels)
|
|
wav.setsampwidth(2)
|
|
wav.setframerate(sample_rate)
|
|
with raw_path.open("rb") as raw_handle:
|
|
while True:
|
|
chunk = raw_handle.read(64 * 1024)
|
|
if not chunk:
|
|
break
|
|
wav.writeframes(chunk)
|
|
|
|
# --- Live draft helpers (from your whisper-translation/apple_speech pipe protocol) ---
|
|
def _load_macmini_mcp_config(self) -> dict:
|
|
try:
|
|
if ESP32_MACMINI_MCP_URL and ESP32_MACMINI_MCP_TOKEN:
|
|
return {"url": ESP32_MACMINI_MCP_URL, "token": ESP32_MACMINI_MCP_TOKEN}
|
|
cfg_path = Path.home() / ".hermes" / "config.yaml"
|
|
if cfg_path.exists():
|
|
import yaml
|
|
cfg = yaml.safe_load(cfg_path.read_text()) or {}
|
|
m = (cfg.get("mcp_servers") or {}).get("macmini") or {}
|
|
url = m.get("url") or ""
|
|
token = ""
|
|
headers = m.get("headers") or {}
|
|
auth = headers.get("Authorization") or ""
|
|
if auth.startswith("Bearer "):
|
|
token = auth[7:]
|
|
if url:
|
|
return {"url": url, "token": token}
|
|
except Exception:
|
|
pass
|
|
return {}
|
|
|
|
@staticmethod
|
|
def _s16le_to_wav_bytes(pcm_bytes: bytes, sample_rate: int = 16000, channels: int = 1) -> bytes:
|
|
import io, wave
|
|
bio = io.BytesIO()
|
|
with wave.open(bio, "wb") as wf:
|
|
wf.setnchannels(channels)
|
|
wf.setsampwidth(2)
|
|
wf.setframerate(sample_rate)
|
|
wf.writeframes(pcm_bytes)
|
|
return bio.getvalue()
|
|
|
|
async def _transcribe_draft_via_macmini(self, wav_bytes: bytes, locale: str = "en-US") -> str | None:
|
|
if not ESP32_LIVE_DRAFT_USE_MACMINI:
|
|
return None
|
|
cfg = self._load_macmini_mcp_config()
|
|
url = cfg.get("url")
|
|
token = cfg.get("token")
|
|
if not url:
|
|
return None
|
|
try:
|
|
import base64
|
|
b64 = base64.b64encode(wav_bytes).decode("ascii")
|
|
# First try new live pipe tool
|
|
for tool_name in ["speech_live_transcribe", "speech_transcribe_file"]:
|
|
if tool_name == "speech_transcribe_file":
|
|
# file tool expects filePath on Mac, not base64 - can't use unless we upload, skip for draft
|
|
continue
|
|
payload = {
|
|
"jsonrpc": "2.0",
|
|
"id": 1,
|
|
"method": "tools/call",
|
|
"params": {"name": tool_name, "arguments": {"audioBase64": b64, "locale": locale} if tool_name=="speech_live_transcribe" else {"filePath": "/tmp/dummy.wav", "locale": locale}},
|
|
}
|
|
headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"}
|
|
if token:
|
|
headers["Authorization"] = f"Bearer {token}"
|
|
import aiohttp
|
|
async with aiohttp.ClientSession() as session:
|
|
async with session.post(url, json=payload, headers=headers, timeout=aiohttp.ClientTimeout(total=8)) as resp:
|
|
txt = await resp.text()
|
|
if resp.status != 200:
|
|
logger.debug(f"[api_server] macmini {tool_name} HTTP {resp.status}: {txt[:500]}")
|
|
continue
|
|
for line in txt.splitlines():
|
|
if not line.startswith("data: "):
|
|
continue
|
|
try:
|
|
data = json.loads(line[6:])
|
|
if data.get("result", {}).get("isError"):
|
|
# log error to help debugging
|
|
err_txt = (data.get("result", {}).get("content") or [{}])[0].get("text","")[:500]
|
|
if "not found" in err_txt.lower():
|
|
logger.warning(f"[api_server] macmini tool {tool_name} not found — need git pull on macmini (have {err_txt[:100]})")
|
|
else:
|
|
logger.debug(f"[api_server] macmini {tool_name} error: {err_txt}")
|
|
continue
|
|
content = (data.get("result", {}).get("content") or [{}])[0].get("text", "") or ""
|
|
if not content:
|
|
continue
|
|
inner = json.loads(content)
|
|
t = inner.get("text") or ""
|
|
if t:
|
|
return t.strip()
|
|
except Exception as ex:
|
|
logger.debug(f"[api_server] macmini parse err: {ex}")
|
|
continue
|
|
except Exception as e:
|
|
logger.debug(f"[api_server] macmini live draft exception: {e}")
|
|
return None
|
|
|
|
async def _transcribe_draft_local(self, wav_bytes: bytes) -> str | None:
|
|
try:
|
|
import tempfile
|
|
fd, tmp_path = tempfile.mkstemp(suffix=".wav")
|
|
os.close(fd)
|
|
Path(tmp_path).write_bytes(wav_bytes)
|
|
def _do():
|
|
try:
|
|
from faster_whisper import WhisperModel
|
|
model = None
|
|
try:
|
|
from tools.transcription_tools import _local_model
|
|
if _local_model is not None:
|
|
model = _local_model
|
|
except Exception:
|
|
pass
|
|
if model is None:
|
|
model = WhisperModel("tiny", device="cpu", compute_type="int8", num_workers=1)
|
|
segs, _ = model.transcribe(tmp_path, beam_size=1, best_of=1, language="en", condition_on_previous_text=False, vad_filter=True)
|
|
return " ".join(s.text.strip() for s in segs).strip()
|
|
except Exception as ex:
|
|
logger.debug(f"[api_server] local draft err: {ex}")
|
|
return None
|
|
finally:
|
|
try:
|
|
Path(tmp_path).unlink(missing_ok=True)
|
|
except Exception:
|
|
pass
|
|
return await asyncio.to_thread(_do)
|
|
except Exception:
|
|
return None
|
|
|
|
async def _apple_llm_quick_reply_via_macmini(self, draft: str, context: str = "", instructions: str = "") -> str | None:
|
|
"""Call Mac mini apple_llm_quick_reply for instant smart ACK from draft."""
|
|
if not ESP32_LIVE_DRAFT_USE_MACMINI:
|
|
return None
|
|
cfg = self._load_macmini_mcp_config()
|
|
url = cfg.get("url")
|
|
token = cfg.get("token")
|
|
if not url:
|
|
return None
|
|
try:
|
|
payload = {
|
|
"jsonrpc": "2.0",
|
|
"id": 1,
|
|
"method": "tools/call",
|
|
"params": {"name": "apple_llm_quick_reply", "arguments": {"draft": draft, "context": context or "", "instructions": instructions or ""}},
|
|
}
|
|
headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"}
|
|
if token:
|
|
headers["Authorization"] = f"Bearer {token}"
|
|
import aiohttp
|
|
async with aiohttp.ClientSession() as session:
|
|
async with session.post(url, json=payload, headers=headers, timeout=aiohttp.ClientTimeout(total=4)) as resp:
|
|
txt = await resp.text()
|
|
if resp.status != 200:
|
|
logger.debug(f"[api_server] macmini quick_reply HTTP {resp.status}: {txt[:500]}")
|
|
return None
|
|
for line in txt.splitlines():
|
|
if not line.startswith("data: "):
|
|
continue
|
|
try:
|
|
data = json.loads(line[6:])
|
|
if data.get("result", {}).get("isError"):
|
|
err_txt = (data.get("result", {}).get("content") or [{}])[0].get("text","")[:500]
|
|
if "not found" in err_txt.lower():
|
|
logger.warning(f"[api_server] macmini apple_llm_quick_reply not found — need git pull + npm restart on mac_mini (192.168.68.102). Have: {err_txt[:120]}")
|
|
else:
|
|
logger.debug(f"[api_server] quick_reply error: {err_txt}")
|
|
continue
|
|
content = (data.get("result", {}).get("content") or [{}])[0].get("text", "") or ""
|
|
if not content:
|
|
continue
|
|
inner = json.loads(content)
|
|
t = inner.get("text") or ""
|
|
if t:
|
|
return t.strip()
|
|
except Exception as ex:
|
|
logger.debug(f"[api_server] quick_reply parse err: {ex}")
|
|
continue
|
|
except Exception as e:
|
|
logger.debug(f"[api_server] macmini quick_reply exception: {e}")
|
|
return None
|
|
|
|
async def _run_quick_reply_then_full(self, draft_text: str, final_transcript: str, device_id: str, screen_url, instructions: str | None, audio_dir):
|
|
"""New flow: when voice stops, we already have draft. Emit instant smart reply via Apple LLM, then full Hermes turn."""
|
|
# This is kept for future speculative pre-start; for now we just log — actual quick reply emitted in WS handler before background start
|
|
pass
|
|
|
|
async def _speak_instant_via_macmini(self, text: str, voice: str = "Alex", use_voicebox: bool = False) -> str | None:
|
|
"""Synthesize instant reply text via Mac mini voice for immediate audio while Hermes thinks.
|
|
- Default: use Mac say Alex boy voice via speech_synthesize_base64 (fast <500ms, reliable)
|
|
- If use_voicebox=True, use Voicebox Qwen Aiden boy voice via voicebox_quick_reply (higher quality, slower 2-4s when queue not stuck)
|
|
Returns base64 wav if success, else None.
|
|
"""
|
|
cfg = self._load_macmini_mcp_config()
|
|
url = cfg.get("url")
|
|
token = cfg.get("token")
|
|
if not url:
|
|
return None
|
|
import aiohttp, json
|
|
headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream"}
|
|
if token:
|
|
headers["Authorization"] = f"Bearer {token}"
|
|
|
|
tool_name = "voicebox_quick_reply" if use_voicebox else "speech_synthesize_base64"
|
|
args = {"text": text[:300]}
|
|
if use_voicebox:
|
|
args["profile"] = "Aiden"
|
|
else:
|
|
args["voice"] = voice
|
|
|
|
try:
|
|
payload = {
|
|
"jsonrpc": "2.0",
|
|
"id": 1,
|
|
"method": "tools/call",
|
|
"params": {"name": tool_name, "arguments": args},
|
|
}
|
|
async with aiohttp.ClientSession() as session:
|
|
async with session.post(url, json=payload, headers=headers, timeout=aiohttp.ClientTimeout(total=10 if not use_voicebox else 20)) as resp:
|
|
txt = await resp.text()
|
|
if resp.status != 200:
|
|
logger.debug(f"[api_server] macmini {tool_name} HTTP {resp.status}: {txt[:500]}")
|
|
return None
|
|
for line in txt.splitlines():
|
|
if not line.startswith("data: "):
|
|
continue
|
|
try:
|
|
data = json.loads(line[6:])
|
|
if data.get("result", {}).get("isError"):
|
|
err_txt = (data.get("result", {}).get("content") or [{}])[0].get("text","")[:500]
|
|
logger.debug(f"[api_server] macmini {tool_name} error: {err_txt}")
|
|
continue
|
|
content = (data.get("result", {}).get("content") or [{}])[0].get("text", "") or ""
|
|
if not content:
|
|
continue
|
|
inner = json.loads(content)
|
|
b64 = inner.get("wavBase64") or inner.get("wav_base64")
|
|
if b64:
|
|
return b64
|
|
except Exception as ex:
|
|
logger.debug(f"[api_server] {tool_name} parse err: {ex}")
|
|
continue
|
|
except Exception as e:
|
|
logger.debug(f"[api_server] {tool_name} exception: {e}")
|
|
return None
|
|
|
|
|
|
|
|
async def _handle_esp32_voice_ws(self, request: "web.Request") -> "web.WebSocketResponse":
|
|
"""GET /api/esp32/voice/ws — ESP32 PCM streaming with LIVE draft captioning.
|
|
LIVE DRAFT MODE (from whisper-translation/engine_apple_transcribe.py):
|
|
- While ingesting binary s16le, every ~800ms snapshots current PCM -> in-mem WAV
|
|
-> tries Mac mini speech_live_transcribe (ApplePipeTranscriber --pipe volatile,
|
|
<200ms drafts) fallback to local tiny for fast feedback.
|
|
- Emits WS events: draft + interim_transcript for UI. Final via macmini or local base.
|
|
"""
|
|
auth_err = self._check_auth(request)
|
|
if auth_err:
|
|
return auth_err # type: ignore[return-value]
|
|
|
|
ws = web.WebSocketResponse(heartbeat=None, max_msg_size=512 * 1024)
|
|
await ws.prepare(request)
|
|
|
|
async def _send_event(event: str, **payload):
|
|
if not ws.closed:
|
|
await ws.send_str(json.dumps({"event": event, **payload}))
|
|
|
|
device_id = (
|
|
request.headers.get("X-Device-ID")
|
|
or request.query.get("device_id")
|
|
or "default"
|
|
).strip()[:80] or "default"
|
|
safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default"
|
|
audio_dir = self._esp32_audio_cache_dir()
|
|
audio_dir.mkdir(parents=True, exist_ok=True)
|
|
stamp = f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}"
|
|
raw_path = audio_dir / f"{stamp}.s16le"
|
|
wav_path = audio_dir / f"{stamp}.wav"
|
|
sample_rate = 16000
|
|
channels = 1
|
|
sample_width = 2
|
|
total_audio_bytes = 0
|
|
started = False
|
|
stopped = False
|
|
first_audio_at: float | None = None
|
|
started_at = time.time()
|
|
# live draft state (directly from your whisper-translation ApplePipeTranscriber pattern)
|
|
live_enabled = ESP32_LIVE_DRAFT_ENABLED
|
|
pcm_buffer = bytearray()
|
|
last_draft_at = 0.0
|
|
last_draft_bytes = 0
|
|
last_draft_text = ""
|
|
draft_in_progress = False
|
|
draft_count = 0
|
|
locale_req = request.headers.get("X-Hermes-Locale") or request.query.get("locale") or "en-US"
|
|
|
|
audit_ctx = self._request_audit_context(request)
|
|
audit_event = {
|
|
"timestamp": started_at,
|
|
"device_id": device_id,
|
|
"source_ip": audit_ctx.get("remote") or audit_ctx.get("peer_ip") or "",
|
|
"peer_ip": audit_ctx.get("peer_ip") or "",
|
|
"forwarded_for": audit_ctx.get("forwarded_for") or "",
|
|
"real_ip": audit_ctx.get("real_ip") or "",
|
|
"user_agent": audit_ctx.get("user_agent") or "",
|
|
"transport": "websocket",
|
|
"raw_path": str(raw_path),
|
|
"audio_path": str(wav_path),
|
|
"status": "accepted",
|
|
"live_draft_enabled": live_enabled,
|
|
}
|
|
self._append_esp32_voice_audit(audio_dir, audit_event)
|
|
screen_url = self._resolve_esp32_screen_url(request, device_id)
|
|
visual_listening_sent = False
|
|
|
|
def _schedule_voice_state(state: str, *, text: str = "", frames: int | None = None) -> None:
|
|
if not screen_url:
|
|
return
|
|
task = asyncio.create_task(self._send_screen_animation(screen_url, state, text=text, frames=frames))
|
|
try:
|
|
self._background_tasks.add(task)
|
|
except TypeError:
|
|
pass
|
|
if hasattr(task, "add_done_callback"):
|
|
task.add_done_callback(self._background_tasks.discard)
|
|
|
|
async def _do_live_draft(snapshot_bytes: bytes):
|
|
nonlocal last_draft_text, draft_in_progress, draft_count
|
|
if draft_in_progress:
|
|
return
|
|
if len(snapshot_bytes) < int(sample_rate * 0.5 * 2): # at least 500ms of s16le at 16k mono
|
|
return
|
|
draft_in_progress = True
|
|
try:
|
|
wav_b = self._s16le_to_wav_bytes(snapshot_bytes, sample_rate=sample_rate, channels=channels)
|
|
txt = None
|
|
if ESP32_LIVE_DRAFT_USE_MACMINI:
|
|
txt = await self._transcribe_draft_via_macmini(wav_b, locale=locale_req)
|
|
if not txt:
|
|
txt = await self._transcribe_draft_local(wav_b)
|
|
if not txt:
|
|
return
|
|
txt = txt.strip()
|
|
if not txt or txt == last_draft_text or len(txt) < 2:
|
|
return
|
|
last_draft_text = txt
|
|
draft_count += 1
|
|
await _send_event("draft", text=txt, is_final=False, isFinal=False, chunk=draft_count, bytes=len(snapshot_bytes))
|
|
await _send_event("interim_transcript", text=txt, is_final=False, draft_count=draft_count)
|
|
if screen_url and len(txt) > 1:
|
|
try:
|
|
t = asyncio.create_task(self._send_screen_text(screen_url, txt, clear=False))
|
|
self._background_tasks.add(t)
|
|
t.add_done_callback(self._background_tasks.discard)
|
|
except Exception:
|
|
pass
|
|
finally:
|
|
draft_in_progress = False
|
|
|
|
await _send_event("ready", device_id=device_id, screen_url=screen_url or "", live_draft=live_enabled, live_model="apple-pipe-volatile")
|
|
|
|
try:
|
|
raw_handle = raw_path.open("wb")
|
|
try:
|
|
async for msg in ws:
|
|
if msg.type == web.WSMsgType.TEXT:
|
|
try:
|
|
payload = json.loads(msg.data)
|
|
except json.JSONDecodeError:
|
|
await _send_event("error", code="bad_json", message="Text frames must be JSON")
|
|
continue
|
|
event = str(payload.get("event") or payload.get("type") or "").strip().lower()
|
|
if event == "start":
|
|
if started:
|
|
await _send_event("error", code="already_started", message="Audio stream already started")
|
|
continue
|
|
fmt = str(payload.get("format") or "pcm_s16le").strip().lower()
|
|
channels = int(payload.get("channels") or 1)
|
|
sample_width = int(payload.get("sample_width") or 2)
|
|
sample_rate = int(payload.get("sample_rate") or 16000)
|
|
device_id = str(payload.get("device_id") or device_id).strip()[:80] or device_id
|
|
if fmt not in {"pcm_s16le", "s16le", "pcm"} or channels != 1 or sample_width != 2:
|
|
await _send_event("error", code="unsupported_audio_format", message="Use 16 kHz mono signed 16-bit little-endian PCM")
|
|
await ws.close()
|
|
return ws
|
|
if sample_rate < 8000 or sample_rate > 48000:
|
|
await _send_event("error", code="bad_sample_rate", message="sample_rate must be 8000-48000")
|
|
await ws.close()
|
|
return ws
|
|
started = True
|
|
if not visual_listening_sent:
|
|
_schedule_voice_state("captioning", frames=8)
|
|
visual_listening_sent = True
|
|
await _send_event("listening", sample_rate=sample_rate, channels=channels, sample_width=sample_width)
|
|
elif event in {"stop", "end"}:
|
|
stopped = True
|
|
break
|
|
elif event in {"ping", "keepalive"}:
|
|
await _send_event("pong")
|
|
else:
|
|
await _send_event("error", code="unknown_event", message=f"Unknown event: {event or '<empty>'}")
|
|
elif msg.type == web.WSMsgType.BINARY:
|
|
if not started:
|
|
started = True
|
|
if not visual_listening_sent:
|
|
_schedule_voice_state("captioning", frames=8)
|
|
visual_listening_sent = True
|
|
await _send_event("listening", sample_rate=sample_rate, channels=channels, sample_width=sample_width)
|
|
chunk = bytes(msg.data or b"")
|
|
if not chunk:
|
|
continue
|
|
if first_audio_at is None:
|
|
first_audio_at = time.time()
|
|
total_audio_bytes += len(chunk)
|
|
pcm_buffer.extend(chunk)
|
|
if total_audio_bytes > ESP32_WS_MAX_AUDIO_BYTES:
|
|
await _send_event("error", code="audio_too_large", message=f"Audio stream exceeded {ESP32_WS_MAX_AUDIO_BYTES} bytes")
|
|
await ws.close()
|
|
return ws
|
|
raw_handle.write(chunk)
|
|
# live draft trigger (every 800ms & 8KB new data) — direct captioning fluff stripped
|
|
if live_enabled:
|
|
now = time.time()
|
|
elapsed_ms = (now - last_draft_at) * 1000
|
|
new_bytes = len(pcm_buffer) - last_draft_bytes
|
|
total_ms = len(pcm_buffer) / 2 / sample_rate * 1000
|
|
if total_ms >= ESP32_LIVE_DRAFT_MIN_TOTAL_MS and elapsed_ms >= ESP32_LIVE_DRAFT_INTERVAL_MS and new_bytes >= ESP32_LIVE_DRAFT_MIN_BYTES:
|
|
last_draft_at = now
|
|
last_draft_bytes = len(pcm_buffer)
|
|
# copy snapshot, fire-and-forget draft (but guard concurrent)
|
|
snapshot = bytes(pcm_buffer)
|
|
asyncio.create_task(_do_live_draft(snapshot))
|
|
elif msg.type == web.WSMsgType.ERROR:
|
|
logger.warning("[api_server] ESP32 voice websocket error: %s", ws.exception())
|
|
break
|
|
finally:
|
|
raw_handle.close()
|
|
|
|
if not stopped:
|
|
await _send_event("error", code="missing_stop", message="Send a stop event after audio chunks")
|
|
await ws.close()
|
|
return ws
|
|
if total_audio_bytes < ESP32_WS_MIN_AUDIO_BYTES:
|
|
await _send_event("error", code="audio_too_short", message="Not enough audio received")
|
|
await ws.close()
|
|
return ws
|
|
|
|
await asyncio.to_thread(
|
|
self._write_pcm_s16le_wav,
|
|
raw_path,
|
|
wav_path,
|
|
sample_rate=sample_rate,
|
|
channels=channels,
|
|
)
|
|
audit_event.update({
|
|
"status": "received",
|
|
"audio_bytes": total_audio_bytes,
|
|
"sample_rate": sample_rate,
|
|
"first_audio_latency_ms": int(((first_audio_at or started_at) - started_at) * 1000),
|
|
"stop_latency_ms": int((time.time() - started_at) * 1000),
|
|
"draft_count": draft_count,
|
|
"last_draft": last_draft_text,
|
|
})
|
|
self._append_esp32_voice_audit(audio_dir, dict(audit_event))
|
|
|
|
_schedule_voice_state("captioning", frames=10)
|
|
await _send_event("thinking", stage="transcribing", draft_count=draft_count)
|
|
|
|
# Parallel instant: start quick_reply from draft ASAP, but prefer full transcript for context
|
|
# Previously we used last_draft_text which was often just "Perfect. Perfect." leading to generic reply
|
|
t_quick_start = time.time()
|
|
instant_future = None
|
|
draft_for_llm = last_draft_text if last_draft_text and len(last_draft_text.strip()) > 2 else None
|
|
# Only start early if draft seems substantial (>30 chars and not just generic starter) - else wait for final transcript
|
|
if draft_for_llm and ESP32_LIVE_DRAFT_USE_MACMINI and len(draft_for_llm.strip()) > 30:
|
|
try:
|
|
instant_future = asyncio.create_task(
|
|
self._apple_llm_quick_reply_via_macmini(
|
|
draft=draft_for_llm,
|
|
context="",
|
|
instructions=request.headers.get("X-Hermes-Instructions") or ""
|
|
)
|
|
)
|
|
except Exception:
|
|
instant_future = None
|
|
|
|
# Final transcription: prefer Mac mini live pipe (ApplePipeTranscriber)
|
|
transcript = None
|
|
final_source = "none"
|
|
t_transcribe_start = time.time()
|
|
|
|
try:
|
|
if ESP32_LIVE_DRAFT_USE_MACMINI and len(pcm_buffer) > 0:
|
|
full_wav = self._s16le_to_wav_bytes(bytes(pcm_buffer), sample_rate=sample_rate, channels=channels)
|
|
mac_txt = await self._transcribe_draft_via_macmini(full_wav, locale=locale_req)
|
|
if mac_txt and len(mac_txt.strip()) >= 2:
|
|
transcript = mac_txt.strip()
|
|
final_source = "macmini-live-final"
|
|
if instant_future is None and mac_txt:
|
|
# Start instant from final if no draft was available earlier
|
|
try:
|
|
draft_for_llm = mac_txt[:200]
|
|
instant_future = asyncio.create_task(
|
|
self._apple_llm_quick_reply_via_macmini(
|
|
draft=mac_txt[:200],
|
|
context="",
|
|
instructions=request.headers.get("X-Hermes-Instructions") or ""
|
|
)
|
|
)
|
|
except Exception:
|
|
pass
|
|
except Exception as e:
|
|
logger.debug(f"[api_server] macmini final transcribe failed: {e}")
|
|
|
|
if not transcript:
|
|
from tools.transcription_tools import transcribe_audio
|
|
try:
|
|
if not wav_path.exists():
|
|
await asyncio.to_thread(
|
|
self._write_pcm_s16le_wav,
|
|
raw_path,
|
|
wav_path,
|
|
sample_rate=sample_rate,
|
|
channels=channels,
|
|
)
|
|
except Exception:
|
|
pass
|
|
stt_result = await asyncio.to_thread(transcribe_audio, str(wav_path) if wav_path.exists() else str(raw_path))
|
|
if not stt_result.get("success"):
|
|
message = stt_result.get("error") or "Transcription failed"
|
|
self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "transcription_failed", "error": message, "timestamp": time.time()})
|
|
await _send_event("error", code="transcription_failed", message=message)
|
|
await ws.close()
|
|
return ws
|
|
transcript = str(stt_result.get("transcript") or "").strip()
|
|
final_source = "local-faster-whisper"
|
|
|
|
if not transcript:
|
|
self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "empty_transcript", "timestamp": time.time()})
|
|
await _send_event("error", code="empty_transcript", message="Transcription was empty")
|
|
await ws.close()
|
|
return ws
|
|
|
|
transcribe_ms = int((time.time() - t_transcribe_start) * 1000)
|
|
|
|
self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "transcribed", "transcript": transcript, "timestamp": time.time(), "draft_count": draft_count, "final_source": final_source, "transcribe_ms": transcribe_ms})
|
|
await _send_event("transcript", text=transcript, is_final=True, isFinal=True, draft_count=draft_count, source=final_source, transcribe_ms=transcribe_ms)
|
|
await _send_event("final", text=transcript, isFinal=True, source=final_source, transcribe_ms=transcribe_ms)
|
|
|
|
summary = self._short_voice_summary(transcript, 180)
|
|
|
|
# Now await instant that was started in parallel on stop
|
|
instant_reply = None
|
|
instant_ok = False
|
|
quick_ms = 0
|
|
if draft_for_llm is None or len(draft_for_llm.strip()) < 20:
|
|
draft_for_llm = transcript[:300]
|
|
# For contextual reply, always feed final transcript if draft was short/generic
|
|
draft_to_use_for_llm = transcript[:400] if (not draft_for_llm or len(transcript) > len(draft_for_llm) or len(draft_for_llm.strip()) < 30) else draft_for_llm
|
|
try:
|
|
if instant_future is not None:
|
|
try:
|
|
instant_reply = await asyncio.wait_for(instant_future, timeout=3.0)
|
|
# If instant from short draft is generic, discard and regenerate from full transcript
|
|
if instant_reply and len(draft_for_llm.strip()) < 30 and len(transcript) > len(draft_for_llm):
|
|
# Prefer transcript-based reply for context
|
|
instant_reply = None
|
|
except asyncio.TimeoutError:
|
|
instant_reply = None
|
|
try:
|
|
instant_future.cancel()
|
|
except Exception:
|
|
pass
|
|
if not instant_reply and ESP32_LIVE_DRAFT_USE_MACMINI:
|
|
# No draft, or future failed — try once more from final transcript (contextual)
|
|
instant_reply = await self._apple_llm_quick_reply_via_macmini(
|
|
draft=draft_to_use_for_llm[:400],
|
|
context="",
|
|
instructions=request.headers.get("X-Hermes-Instructions") or ""
|
|
)
|
|
if instant_reply:
|
|
quick_ms = int((time.time() - t_quick_start) * 1000)
|
|
instant_ok = True
|
|
await _send_event("instant_response", text=instant_reply, is_quick=True, quick_ms=quick_ms, draft=draft_for_llm[:200], source="apple-llm-3b-ane", final=False, persistent=True)
|
|
await _send_event("quick_reply", text=instant_reply, ms=quick_ms, persistent=True)
|
|
await _send_event("response_text", text=instant_reply, response_id="", final=False, is_instant=True, quick_ms=quick_ms, source="apple-llm-3b-ane")
|
|
# Also speak instant reply via Mac mini voicebox/boy voice while Hermes generates full answer
|
|
# This gives user audio feedback immediately instead of silence
|
|
try:
|
|
# First try fast Mac say Alex (boy voice) for speed <500ms, voicebox Aiden if available but slower
|
|
t_speak_start = time.time()
|
|
# Try Aiden voicebox first if you prefer quality, else Alex for speed. Config: try Alex first for speed.
|
|
audio_b64 = await self._speak_instant_via_macmini(instant_reply, voice="Alex", use_voicebox=False)
|
|
if not audio_b64:
|
|
# Fallback try voicebox Aiden boy voice (Qwen 1.7B) if say failed or you prefer quality
|
|
try:
|
|
audio_b64 = await self._speak_instant_via_macmini(instant_reply, voice="Aiden", use_voicebox=True)
|
|
except Exception:
|
|
pass
|
|
if audio_b64 and screen_url:
|
|
speak_ms = int((time.time() - t_speak_start) * 1000)
|
|
# Send audio to ESP32 screen via play_audio_base64 - this was your request: hear fast reply while Hermes thinks
|
|
try:
|
|
# Use play_audio_base64 (WAV 16k mono) - Mac say already 16k
|
|
audio_task = asyncio.create_task(
|
|
self._call_screen_mcp(
|
|
screen_url,
|
|
"play_audio_base64",
|
|
{"wav_base64": audio_b64, "volume": 80},
|
|
timeout=20,
|
|
)
|
|
)
|
|
self._background_tasks.add(audio_task)
|
|
audio_task.add_done_callback(self._background_tasks.discard)
|
|
except Exception as e:
|
|
logger.debug(f"[api_server] instant audio play failed: {e}")
|
|
await _send_event("instant_audio", text=instant_reply, quick_ms=quick_ms, speak_ms=speak_ms, voice="Alex/Aiden boy via Mac mini voicebox", source="macmini-say/voicebox")
|
|
self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "instant_audio", "speak_ms": speak_ms, "timestamp": time.time()})
|
|
except Exception as e:
|
|
logger.debug(f"[api_server] instant speak failed: {e}")
|
|
|
|
if screen_url:
|
|
try:
|
|
task = asyncio.create_task(self._send_screen_text(screen_url, instant_reply, clear=True))
|
|
self._background_tasks.add(task)
|
|
task.add_done_callback(self._background_tasks.discard)
|
|
except Exception:
|
|
pass
|
|
self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "instant_reply", "instant_reply": instant_reply, "quick_ms": quick_ms, "timestamp": time.time()})
|
|
except Exception as e:
|
|
logger.debug(f"[api_server] instant reply failed: {e}")
|
|
|
|
if not instant_ok:
|
|
ack_text = f"Heard: {summary}\n\nWorking on the full answer now."
|
|
await _send_event("response_text", text=ack_text, response_id="", final=False, is_fallback=True)
|
|
|
|
self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "queued_background", "transcript": transcript, "timestamp": time.time(), "instant_ok": instant_ok})
|
|
|
|
try:
|
|
await _send_event("done", mode="background", screen_url=screen_url or "", draft_count=draft_count, source=final_source, instant_reply=instant_reply or "", instant_ok=instant_ok, quick_ms=quick_ms, transcribe_ms=transcribe_ms)
|
|
if instant_ok:
|
|
await _send_event("thinking", stage="full_answer", keep_instant=True, instant_text=instant_reply, persistent_instant=True)
|
|
await ws.close()
|
|
except Exception:
|
|
pass
|
|
background_task = asyncio.create_task(
|
|
self._run_esp32_voice_background(
|
|
transcript=transcript,
|
|
device_id=device_id,
|
|
instructions=request.headers.get("X-Hermes-Instructions") or None,
|
|
audio_dir=audio_dir,
|
|
screen_url=screen_url,
|
|
)
|
|
)
|
|
try:
|
|
self._background_tasks.add(background_task)
|
|
except TypeError:
|
|
pass
|
|
if hasattr(background_task, "add_done_callback"):
|
|
background_task.add_done_callback(self._background_tasks.discard)
|
|
return ws
|
|
except Exception as exc:
|
|
logger.exception("[api_server] ESP32 voice websocket processing failed")
|
|
self._append_esp32_voice_audit(audio_dir, {**audit_event, "status": "websocket_exception", "error": str(exc), "timestamp": time.time()})
|
|
try:
|
|
await _send_event("error", code="internal_error", message=str(exc))
|
|
await ws.close()
|
|
except Exception:
|
|
pass
|
|
return ws
|
|
finally:
|
|
try:
|
|
raw_path.unlink(missing_ok=True)
|
|
except Exception:
|
|
pass
|
|
|
|
async def _handle_esp32_voice(self, request: "web.Request") -> "web.Response":
|
|
|
|
"""POST /api/esp32/voice — receive audio, run Hermes, return TTS audio."""
|
|
auth_err = self._check_auth(request)
|
|
if auth_err:
|
|
return auth_err
|
|
|
|
device_id = (
|
|
request.headers.get("X-Device-ID")
|
|
or request.query.get("device_id")
|
|
or "default"
|
|
).strip()[:80] or "default"
|
|
|
|
try:
|
|
audio_bytes = await request.read()
|
|
except Exception as exc:
|
|
return web.json_response(_openai_error(f"Failed to read audio body: {exc}"), status=400)
|
|
|
|
if not audio_bytes:
|
|
return web.json_response(_openai_error("Empty audio body", code="empty_audio"), status=400)
|
|
if len(audio_bytes) > ESP32_AUDIO_MAX_BYTES:
|
|
return web.json_response(
|
|
_openai_error(
|
|
f"Audio body is too large ({len(audio_bytes)} bytes); max is {ESP32_AUDIO_MAX_BYTES}",
|
|
code="audio_too_large",
|
|
),
|
|
status=413,
|
|
)
|
|
|
|
suffix = self._audio_extension_for_content_type(request.headers.get("Content-Type", ""))
|
|
try:
|
|
from hermes_constants import get_hermes_dir
|
|
audio_dir = get_hermes_dir("cache/audio", "audio_cache") / "esp32"
|
|
except Exception:
|
|
audio_dir = Path.home() / ".hermes" / "cache" / "audio" / "esp32"
|
|
audio_dir.mkdir(parents=True, exist_ok=True)
|
|
safe_device = re.sub(r"[^A-Za-z0-9_.-]+", "_", device_id)[:80] or "default"
|
|
audio_path = audio_dir / f"{safe_device}_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}{suffix}"
|
|
audio_path.write_bytes(audio_bytes)
|
|
|
|
audit_ctx = self._request_audit_context(request)
|
|
audit_event: Dict[str, Any] = {
|
|
"timestamp": time.time(),
|
|
"device_id": device_id,
|
|
"source_ip": audit_ctx.get("remote") or audit_ctx.get("peer_ip") or "",
|
|
"peer_ip": audit_ctx.get("peer_ip") or "",
|
|
"forwarded_for": audit_ctx.get("forwarded_for") or "",
|
|
"real_ip": audit_ctx.get("real_ip") or "",
|
|
"user_agent": audit_ctx.get("user_agent") or "",
|
|
"reply_mode": self._esp32_reply_mode(request),
|
|
"screen_device": request.headers.get("X-Hermes-Screen-Device") or request.query.get("screen_device") or "",
|
|
"screen_url": request.headers.get("X-Hermes-Screen-Url") or request.headers.get("X-Screen-MCP-Url") or request.query.get("screen_url") or "",
|
|
"content_type": request.headers.get("Content-Type", ""),
|
|
"audio_bytes": len(audio_bytes),
|
|
"audio_path": str(audio_path),
|
|
"status": "received",
|
|
}
|
|
|
|
def _append_audit(event: Dict[str, Any]) -> None:
|
|
try:
|
|
log_path = audio_dir / "voice_requests.jsonl"
|
|
with log_path.open("a", encoding="utf-8") as handle:
|
|
handle.write(json.dumps(event, default=str) + "\n")
|
|
except Exception:
|
|
logger.debug("[api_server] failed to append ESP32 voice audit log", exc_info=True)
|
|
|
|
_append_audit(audit_event)
|
|
|
|
if self._esp32_reply_mode(request) == "ack":
|
|
screen_url = self._resolve_esp32_screen_url(request, device_id)
|
|
|
|
async def _ack_background() -> None:
|
|
try:
|
|
await self._send_screen_animation(screen_url, "ack")
|
|
await self._send_screen_animation(screen_url, "captioning")
|
|
from tools.transcription_tools import transcribe_audio
|
|
stt_result_bg = await asyncio.wait_for(
|
|
asyncio.to_thread(transcribe_audio, str(audio_path)),
|
|
timeout=45,
|
|
)
|
|
if not stt_result_bg.get("success"):
|
|
failed_event = dict(audit_event)
|
|
failed_event.update({
|
|
"status": "transcription_failed",
|
|
"error": stt_result_bg.get("error") or "Transcription failed",
|
|
"timestamp": time.time(),
|
|
})
|
|
_append_audit(failed_event)
|
|
await self._send_screen_text(
|
|
screen_url,
|
|
"I heard audio, but transcription failed. Try again.",
|
|
clear=True,
|
|
)
|
|
return
|
|
transcript_bg = str(stt_result_bg.get("transcript") or "").strip()
|
|
if not transcript_bg:
|
|
empty_event = dict(audit_event)
|
|
empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()})
|
|
_append_audit(empty_event)
|
|
await self._send_screen_text(
|
|
screen_url,
|
|
"I heard audio, but got no words. Try again.",
|
|
clear=True,
|
|
)
|
|
return
|
|
transcript_event = dict(audit_event)
|
|
transcript_event.update({"status": "transcribed", "transcript": transcript_bg, "timestamp": time.time()})
|
|
_append_audit(transcript_event)
|
|
await self._run_esp32_voice_background(
|
|
transcript=transcript_bg,
|
|
device_id=device_id,
|
|
instructions=request.headers.get("X-Hermes-Instructions") or None,
|
|
audio_dir=audio_dir,
|
|
screen_url=screen_url,
|
|
)
|
|
except asyncio.TimeoutError:
|
|
timeout_event = dict(audit_event)
|
|
timeout_event.update({"status": "transcription_timeout", "error": "Transcription exceeded 45s", "timestamp": time.time()})
|
|
_append_audit(timeout_event)
|
|
await self._send_screen_text(
|
|
screen_url,
|
|
"I heard audio, but captioning timed out. Try again.",
|
|
clear=True,
|
|
)
|
|
except Exception as exc:
|
|
logger.exception("[api_server] ESP32 ack-mode background processing failed")
|
|
failed_event = dict(audit_event)
|
|
failed_event.update({"status": "background_exception", "error": str(exc), "timestamp": time.time()})
|
|
_append_audit(failed_event)
|
|
|
|
task = asyncio.create_task(_ack_background())
|
|
try:
|
|
self._background_tasks.add(task)
|
|
except TypeError:
|
|
pass
|
|
if hasattr(task, "add_done_callback"):
|
|
task.add_done_callback(self._background_tasks.discard)
|
|
ack_path = await self._esp32_ack_wav(audio_dir=audio_dir, device_id=device_id)
|
|
headers = {
|
|
"X-Hermes-Conversation": f"esp32:{device_id}",
|
|
"X-Hermes-Reply-Mode": "ack",
|
|
"X-Hermes-Screen-Url": screen_url or "",
|
|
"Content-Type": "audio/wav",
|
|
}
|
|
return web.FileResponse(ack_path, headers=headers)
|
|
|
|
try:
|
|
from tools.transcription_tools import transcribe_audio
|
|
stt_result = await asyncio.to_thread(transcribe_audio, str(audio_path))
|
|
except Exception as exc:
|
|
logger.exception("[api_server] ESP32 transcription failed")
|
|
failed_event = dict(audit_event)
|
|
failed_event.update({"status": "transcription_exception", "error": str(exc), "timestamp": time.time()})
|
|
_append_audit(failed_event)
|
|
return web.json_response(_openai_error(f"Transcription failed: {exc}"), status=500)
|
|
|
|
if not stt_result.get("success"):
|
|
failed_event = dict(audit_event)
|
|
failed_event.update({
|
|
"status": "transcription_failed",
|
|
"error": stt_result.get("error") or "Transcription failed",
|
|
"timestamp": time.time(),
|
|
})
|
|
_append_audit(failed_event)
|
|
return web.json_response(
|
|
_openai_error(stt_result.get("error") or "Transcription failed", code="transcription_failed"),
|
|
status=502,
|
|
)
|
|
|
|
transcript = str(stt_result.get("transcript") or "").strip()
|
|
if not transcript:
|
|
empty_event = dict(audit_event)
|
|
empty_event.update({"status": "empty_transcript", "transcript": "", "timestamp": time.time()})
|
|
_append_audit(empty_event)
|
|
return web.json_response(_openai_error("Transcription was empty", code="empty_transcript"), status=422)
|
|
|
|
transcript_event = dict(audit_event)
|
|
transcript_event.update({"status": "transcribed", "transcript": transcript, "timestamp": time.time()})
|
|
_append_audit(transcript_event)
|
|
|
|
instructions = request.headers.get("X-Hermes-Instructions") or None
|
|
try:
|
|
response_data, final_text = await self._run_esp32_voice_turn(
|
|
transcript=transcript,
|
|
device_id=device_id,
|
|
instructions=instructions,
|
|
)
|
|
except Exception as exc:
|
|
logger.exception("[api_server] ESP32 agent turn failed")
|
|
return web.json_response(_openai_error(f"Agent turn failed: {exc}"), status=500)
|
|
|
|
if not final_text.strip():
|
|
final_text = self._extract_response_text(response_data) or "I heard you, but I do not have a spoken answer."
|
|
|
|
try:
|
|
from tools.tts_tool import text_to_speech_tool
|
|
tts_raw = await asyncio.to_thread(text_to_speech_tool, final_text)
|
|
tts_result = json.loads(tts_raw)
|
|
except Exception as exc:
|
|
logger.exception("[api_server] ESP32 TTS failed")
|
|
return web.json_response(
|
|
_openai_error(f"TTS failed after response was generated: {exc}", code="tts_failed"),
|
|
status=500,
|
|
headers={
|
|
"X-Hermes-Transcript": self._safe_audio_header(transcript, 1000),
|
|
"X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000),
|
|
},
|
|
)
|
|
|
|
if not tts_result.get("success"):
|
|
return web.json_response(
|
|
_openai_error(tts_result.get("error") or "TTS failed", code="tts_failed"),
|
|
status=502,
|
|
headers={
|
|
"X-Hermes-Transcript": self._safe_audio_header(transcript, 1000),
|
|
"X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000),
|
|
},
|
|
)
|
|
|
|
tts_path = str(tts_result.get("file_path") or "")
|
|
if not tts_path or not Path(tts_path).exists():
|
|
return web.json_response(
|
|
_openai_error("TTS succeeded but no audio file was produced", code="tts_file_missing"),
|
|
status=500,
|
|
)
|
|
|
|
try:
|
|
response_audio_path = self._convert_audio_to_esp32_wav(
|
|
tts_path,
|
|
output_dir=audio_dir,
|
|
device_id=device_id,
|
|
)
|
|
except Exception as exc:
|
|
logger.exception("[api_server] ESP32 WAV normalization failed")
|
|
return web.json_response(
|
|
_openai_error(f"Could not convert TTS audio to ESP32 WAV: {exc}", code="wav_conversion_failed"),
|
|
status=500,
|
|
headers={
|
|
"X-Hermes-Transcript": self._safe_audio_header(transcript, 1000),
|
|
"X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000),
|
|
},
|
|
)
|
|
|
|
headers = {
|
|
"X-Hermes-Transcript": self._safe_audio_header(transcript, 1000),
|
|
"X-Hermes-Text-Response": self._safe_audio_header(final_text, 2000),
|
|
"X-Hermes-Response-Id": str(response_data.get("id") or ""),
|
|
"X-Hermes-Conversation": f"esp32:{device_id}",
|
|
"X-Hermes-TTS-Source-Type": self._audio_response_mime(tts_path),
|
|
"Content-Type": "audio/wav",
|
|
}
|
|
return web.FileResponse(
|
|
response_audio_path,
|
|
headers=headers,
|
|
)
|
|
|