# pyright: reportMissingImports=false, reportAttributeAccessIssue=false """Client firmware application for real-time WebSocket audio streaming to Hermes.""" import time import struct import machine from machine import Pin, SPI, I2C, I2S import json import ili9341 from ft6336u import FT6336U from audio_util import ES8311 from websocket_client import WebSocketClient # --- CONFIGURATION --- HERMES_WS_URL = "ws://192.168.68.126:8642/api/esp32/voice/ws" DEVICE_ID = "kitchen-button" # Placeholder for API Key. Since the key is stored on the Hermes server, # please copy-paste the token string here. HERMES_API_KEY = "mcT1YA1vOr9wXSiHpCYalweEGGZKX-PIfZv2drp8BSg" def main(): print("--- Starting Hermes WebSocket Voice Assistant Demo ---") # 1. Initialize display spi = SPI(1, baudrate=40000000, polarity=0, phase=0, sck=Pin(12), mosi=Pin(11), miso=Pin(13)) display = ili9341.ILI9341(spi, cs=Pin(10), dc=Pin(46), bl=Pin(45), rst=None) display.clear(0) display.text("Hermes Assistant", 10, 10, 1) display.show() # 2. Initialize touch i2c = I2C(0, sda=Pin(16), scl=Pin(15)) touch = FT6336U(i2c, rst_pin=18, int_pin=17, width=320, height=240, swap_xy=True, invert_x=False, invert_y=True) # 3. Configure audio codec ES8311 # We need MCLK pin (GPIO 4) active at 6.144 MHz mclk_pin = Pin(4, Pin.OUT) mclk_pwm = machine.PWM(mclk_pin) mclk_pwm.freq(6144000) mclk_pwm.duty_u16(32768) codec = ES8311(i2c) if not codec.init(sample_rate=16000): print("ES8311 init failed!") return codec.set_volume(90) # Configure microphone registers on ES8311 try: codec._write(0x14, 0x1A) # Enable analog mic input & set PGA gain codec._write(0x16, 0x01) # Boost MIC digital gain to +6dB codec._write(0x17, 0xC8) # Set ADC digital volume print("Microphone registers initialized.") except Exception as e: print("Failed to write microphone registers:", e) amp_pin = Pin(1, Pin.OUT, value=1) # start with amp disabled (1 = disabled) buffer = bytearray(3200) # 100ms chunk at 16kHz mono 16-bit PCM (3200 bytes) while True: display.clear(0) display.text("Hermes Assistant", 10, 10, 1) display.line(10, 22, 310, 22, 1) display.text("Hold screen & ask", 50, 70, 1) display.text("a question...", 50, 90, 1) display.text("Status: Idle (WS)", 10, 220, 1) display.show() # Wait for touch while not touch.is_touched(): time.sleep_ms(30) print("Touch detected! Connecting WebSocket...") display.clear(0) display.text("Connecting...", 80, 80, 1) display.show() headers = { "Authorization": f"Bearer {HERMES_API_KEY}", "X-Device-ID": DEVICE_ID } ws = WebSocketClient(HERMES_WS_URL, headers=headers) try: ws.connect() # Send start event ws.send_text(json.dumps({ "event": "start", "device_id": DEVICE_ID, "sample_rate": 16000, "channels": 1, "sample_width": 2, "format": "pcm_s16le" })) # Read ready and listening events from server evt1_opcode, evt1_payload = ws.recv_frame() evt2_opcode, evt2_payload = ws.recv_frame() print("WebSocket connected and streaming started.") display.clear(0) display.text("Hermes Assistant", 10, 10, 1) display.line(10, 22, 310, 22, 1) display.text("Listening...", 80, 80, 1) display.fill_rect(130, 110, 30, 30, 1) display.text("Status: Streaming", 10, 220, 1) display.show() # 4. Open I2S RX for recording (Mono 16kHz) i2s_rx = I2S(1, sck=Pin(5), ws=Pin(7), sd=Pin(6), mode=I2S.RX, ibuf=8000, rate=16000, bits=16, format=I2S.MONO) total_data_bytes = 0 try: # Record loop - as long as touch is held while touch.is_touched(): bytes_read = i2s_rx.readinto(buffer) if bytes_read > 0: ws.send_binary(buffer[:bytes_read]) total_data_bytes += bytes_read print(f"Touch released! Sent {total_data_bytes} bytes.") except Exception as e: print("Error recording/streaming:", e) finally: i2s_rx.deinit() if total_data_bytes < 3200: print("Recording too short, cancelling session.") ws.send_text(json.dumps({"event": "cancel"})) ws.close() continue # Send stop event ws.send_text(json.dumps({"event": "stop"})) display.clear(0) display.text("Hermes Assistant", 10, 10, 1) display.line(10, 22, 310, 22, 1) display.text("Processing...", 50, 80, 1) display.text("Status: Thinking", 10, 220, 1) display.show() # Playback/Events loop i2s_tx = None while True: opcode, payload = ws.recv_frame() if opcode is None: break if opcode == 0x1: # Text frame (JSON event) try: event_data = json.loads(payload.decode('utf-8')) evt = event_data.get("event") if evt == "transcript": txt = event_data.get("text", "") print(f"Heard: {txt}") display.clear(0) display.text("Hermes Assistant", 10, 10, 1) display.line(10, 22, 310, 22, 1) display.text("Heard:", 10, 40, 1) # Wrap text onto lines lines = [txt[i:i+30] for i in range(0, min(len(txt), 120), 30)] y_offset = 60 for line in lines: display.text(line, 10, y_offset, 1) y_offset += 20 display.show() elif evt == "thinking": pass elif evt == "response_text": txt = event_data.get("text", "") print(f"Response: {txt}") display.clear(0) display.text("Hermes Assistant", 10, 10, 1) display.line(10, 22, 310, 22, 1) display.text("Response:", 10, 40, 1) lines = [txt[i:i+30] for i in range(0, min(len(txt), 120), 30)] y_offset = 60 for line in lines: display.text(line, 10, y_offset, 1) y_offset += 20 display.show() elif evt == "audio_start": print("Audio response started.") # Enable amp and open I2S TX amp_pin.value(0) # Active Low Enable i2s_tx = I2S(1, sck=Pin(5), ws=Pin(7), sd=Pin(8), mode=I2S.TX, ibuf=4096, rate=16000, bits=16, format=I2S.MONO) elif evt == "audio_end": print("Audio response ended.") if i2s_tx: time.sleep_ms(150) amp_pin.value(1) # Disable amp i2s_tx.deinit() i2s_tx = None elif evt == "done": break elif evt == "error": msg = event_data.get("message", "Unknown error") print(f"Error from server: {msg}") display.clear(0) display.text("Error", 10, 10, 1) display.line(10, 22, 310, 22, 1) display.text(msg[:100], 10, 80, 1) display.show() time.sleep(3) break except Exception as e: print("Error parsing event text:", e) elif opcode == 0x2: # Binary frame (Audio WAV chunk) if i2s_tx: chunk = payload if chunk.startswith(b'RIFF') and len(chunk) > 44: chunk = chunk[44:] # Skip WAV header for direct I2S mono play try: i2s_tx.write(chunk) except Exception as e: print("Error writing to speaker:", e) ws.close() except Exception as e: print("Failed to stream to Hermes server:", e) display.clear(0) display.text("Server Error", 10, 10, 1) display.line(10, 22, 310, 22, 1) display.text("Could not connect", 50, 80, 1) display.text("to WebSocket server.", 50, 100, 1) display.show() time.sleep(3) try: ws.close() except Exception: pass # Debounce touch release while touch.is_touched(): time.sleep_ms(30) time.sleep_ms(300) if __name__ == "__main__": main()