adolforeyna/mcp_screen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement WebSocket real-time audio streaming for Hermes gateway

Browse Source
This commit is contained in:
Adolfo Reyna
2026-06-19 13:58:42 -04:00
parent b2bcdb5b79
commit b88cb34196
4 changed files with 935 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
demo_websocket_voice.py
+268
View File
@@ -0,0 +1,268 @@
# pyright: reportMissingImports=false, reportAttributeAccessIssue=false
"""Client firmware application for real-time WebSocket audio streaming to Hermes."""
import time
import struct
import machine
from machine import Pin, SPI, I2C, I2S
import json
import ili9341
from ft6336u import FT6336U
from audio_util import ES8311
from websocket_client import WebSocketClient
# --- CONFIGURATION ---
HERMES_WS_URL = "ws://192.168.68.126:8642/api/esp32/voice/ws"
DEVICE_ID = "kitchen-button"
# Placeholder for API Key. Since the key is stored on the Hermes server,
# please copy-paste the token string here.
HERMES_API_KEY = "mcT1YA1vOr9wXSiHpCYalweEGGZKX-PIfZv2drp8BSg"
def main():
print("--- Starting Hermes WebSocket Voice Assistant Demo ---")
# 1. Initialize display
spi = SPI(1, baudrate=40000000, polarity=0, phase=0, sck=Pin(12), mosi=Pin(11), miso=Pin(13))
display = ili9341.ILI9341(spi, cs=Pin(10), dc=Pin(46), bl=Pin(45), rst=None)
display.clear(0)
display.text("Hermes Assistant", 10, 10, 1)
display.show()
# 2. Initialize touch
i2c = I2C(0, sda=Pin(16), scl=Pin(15))
touch = FT6336U(i2c, rst_pin=18, int_pin=17, width=320, height=240, swap_xy=True, invert_x=False, invert_y=True)
# 3. Configure audio codec ES8311
# We need MCLK pin (GPIO 4) active at 6.144 MHz
mclk_pin = Pin(4, Pin.OUT)
mclk_pwm = machine.PWM(mclk_pin)
mclk_pwm.freq(6144000)
mclk_pwm.duty_u16(32768)
codec = ES8311(i2c)
if not codec.init(sample_rate=16000):
print("ES8311 init failed!")
return
codec.set_volume(90)
# Configure microphone registers on ES8311
try:
codec._write(0x14, 0x1A) # Enable analog mic input & set PGA gain
codec._write(0x16, 0x01) # Boost MIC digital gain to +6dB
codec._write(0x17, 0xC8) # Set ADC digital volume
print("Microphone registers initialized.")
except Exception as e:
print("Failed to write microphone registers:", e)
amp_pin = Pin(1, Pin.OUT, value=1) # start with amp disabled (1 = disabled)
buffer = bytearray(3200) # 100ms chunk at 16kHz mono 16-bit PCM (3200 bytes)
while True:
display.clear(0)
display.text("Hermes Assistant", 10, 10, 1)
display.line(10, 22, 310, 22, 1)
display.text("Hold screen & ask", 50, 70, 1)
display.text("a question...", 50, 90, 1)
display.text("Status: Idle (WS)", 10, 220, 1)
display.show()
# Wait for touch
while not touch.is_touched():
time.sleep_ms(30)
print("Touch detected! Connecting WebSocket...")
display.clear(0)
display.text("Connecting...", 80, 80, 1)
display.show()
headers = {
"Authorization": f"Bearer {HERMES_API_KEY}",
"X-Device-ID": DEVICE_ID
}
ws = WebSocketClient(HERMES_WS_URL, headers=headers)
try:
ws.connect()
# Send start event
ws.send_text(json.dumps({
"event": "start",
"device_id": DEVICE_ID,
"sample_rate": 16000,
"channels": 1,
"sample_width": 2,
"format": "pcm_s16le"
}))
# Read ready and listening events from server
evt1_opcode, evt1_payload = ws.recv_frame()
evt2_opcode, evt2_payload = ws.recv_frame()
print("WebSocket connected and streaming started.")
display.clear(0)
display.text("Hermes Assistant", 10, 10, 1)
display.line(10, 22, 310, 22, 1)
display.text("Listening...", 80, 80, 1)
display.fill_rect(130, 110, 30, 30, 1)
display.text("Status: Streaming", 10, 220, 1)
display.show()
# 4. Open I2S RX for recording (Mono 16kHz)
i2s_rx = I2S(1,
sck=Pin(5),
ws=Pin(7),
sd=Pin(6),
mode=I2S.RX,
ibuf=8000,
rate=16000,
bits=16,
format=I2S.MONO)
total_data_bytes = 0
try:
# Record loop - as long as touch is held
while touch.is_touched():
bytes_read = i2s_rx.readinto(buffer)
if bytes_read > 0:
ws.send_binary(buffer[:bytes_read])
total_data_bytes += bytes_read
print(f"Touch released! Sent {total_data_bytes} bytes.")
except Exception as e:
print("Error recording/streaming:", e)
finally:
i2s_rx.deinit()
if total_data_bytes < 3200:
print("Recording too short, cancelling session.")
ws.send_text(json.dumps({"event": "cancel"}))
ws.close()
continue
# Send stop event
ws.send_text(json.dumps({"event": "stop"}))
display.clear(0)
display.text("Hermes Assistant", 10, 10, 1)
display.line(10, 22, 310, 22, 1)
display.text("Processing...", 50, 80, 1)
display.text("Status: Thinking", 10, 220, 1)
display.show()
# Playback/Events loop
i2s_tx = None
while True:
opcode, payload = ws.recv_frame()
if opcode is None:
break
if opcode == 0x1: # Text frame (JSON event)
try:
event_data = json.loads(payload.decode('utf-8'))
evt = event_data.get("event")
if evt == "transcript":
txt = event_data.get("text", "")
print(f"Heard: {txt}")
display.clear(0)
display.text("Hermes Assistant", 10, 10, 1)
display.line(10, 22, 310, 22, 1)
display.text("Heard:", 10, 40, 1)
# Wrap text onto lines
lines = [txt[i:i+30] for i in range(0, min(len(txt), 120), 30)]
y_offset = 60
for line in lines:
display.text(line, 10, y_offset, 1)
y_offset += 20
display.show()
elif evt == "thinking":
pass
elif evt == "response_text":
txt = event_data.get("text", "")
print(f"Response: {txt}")
display.clear(0)
display.text("Hermes Assistant", 10, 10, 1)
display.line(10, 22, 310, 22, 1)
display.text("Response:", 10, 40, 1)
lines = [txt[i:i+30] for i in range(0, min(len(txt), 120), 30)]
y_offset = 60
for line in lines:
display.text(line, 10, y_offset, 1)
y_offset += 20
display.show()
elif evt == "audio_start":
print("Audio response started.")
# Enable amp and open I2S TX
amp_pin.value(0) # Active Low Enable
i2s_tx = I2S(1,
sck=Pin(5),
ws=Pin(7),
sd=Pin(8),
mode=I2S.TX,
ibuf=4096,
rate=16000,
bits=16,
format=I2S.MONO)
elif evt == "audio_end":
print("Audio response ended.")
if i2s_tx:
time.sleep_ms(150)
amp_pin.value(1) # Disable amp
i2s_tx.deinit()
i2s_tx = None
elif evt == "done":
break
elif evt == "error":
msg = event_data.get("message", "Unknown error")
print(f"Error from server: {msg}")
display.clear(0)
display.text("Error", 10, 10, 1)
display.line(10, 22, 310, 22, 1)
display.text(msg[:100], 10, 80, 1)
display.show()
time.sleep(3)
break
except Exception as e:
print("Error parsing event text:", e)
elif opcode == 0x2: # Binary frame (Audio WAV chunk)
if i2s_tx:
chunk = payload
if chunk.startswith(b'RIFF') and len(chunk) > 44:
chunk = chunk[44:] # Skip WAV header for direct I2S mono play
try:
i2s_tx.write(chunk)
except Exception as e:
print("Error writing to speaker:", e)
ws.close()
except Exception as e:
print("Failed to stream to Hermes server:", e)
display.clear(0)
display.text("Server Error", 10, 10, 1)
display.line(10, 22, 310, 22, 1)
display.text("Could not connect", 50, 80, 1)
display.text("to WebSocket server.", 50, 100, 1)
display.show()
time.sleep(3)
try:
ws.close()
except Exception:
pass
# Debounce touch release
while touch.is_touched():
time.sleep_ms(30)
time.sleep_ms(300)
if __name__ == "__main__":
main()