Adolfo Reyna
269 lines
11 KiB
Python
269 lines
11 KiB
Python
# pyright: reportMissingImports=false, reportAttributeAccessIssue=false
|
|
"""Client firmware application for real-time WebSocket audio streaming to Hermes."""
|
|
|
|
import time
|
|
import struct
|
|
import machine
|
|
from machine import Pin, SPI, I2C, I2S
|
|
import json
|
|
import ili9341
|
|
from ft6336u import FT6336U
|
|
from audio_util import ES8311
|
|
from websocket_client import WebSocketClient
|
|
|
|
# --- CONFIGURATION ---
|
|
HERMES_WS_URL = "ws://192.168.68.126:8642/api/esp32/voice/ws"
|
|
DEVICE_ID = "kitchen-button"
|
|
|
|
# Placeholder for API Key. Since the key is stored on the Hermes server,
|
|
# please copy-paste the token string here.
|
|
HERMES_API_KEY = "mcT1YA1vOr9wXSiHpCYalweEGGZKX-PIfZv2drp8BSg"
|
|
|
|
def main():
|
|
print("--- Starting Hermes WebSocket Voice Assistant Demo ---")
|
|
|
|
# 1. Initialize display
|
|
spi = SPI(1, baudrate=40000000, polarity=0, phase=0, sck=Pin(12), mosi=Pin(11), miso=Pin(13))
|
|
display = ili9341.ILI9341(spi, cs=Pin(10), dc=Pin(46), bl=Pin(45), rst=None)
|
|
display.clear(0)
|
|
display.text("Hermes Assistant", 10, 10, 1)
|
|
display.show()
|
|
|
|
# 2. Initialize touch
|
|
i2c = I2C(0, sda=Pin(16), scl=Pin(15))
|
|
touch = FT6336U(i2c, rst_pin=18, int_pin=17, width=320, height=240, swap_xy=True, invert_x=False, invert_y=True)
|
|
|
|
# 3. Configure audio codec ES8311
|
|
# We need MCLK pin (GPIO 4) active at 6.144 MHz
|
|
mclk_pin = Pin(4, Pin.OUT)
|
|
mclk_pwm = machine.PWM(mclk_pin)
|
|
mclk_pwm.freq(6144000)
|
|
mclk_pwm.duty_u16(32768)
|
|
|
|
codec = ES8311(i2c)
|
|
if not codec.init(sample_rate=16000):
|
|
print("ES8311 init failed!")
|
|
return
|
|
|
|
codec.set_volume(90)
|
|
|
|
# Configure microphone registers on ES8311
|
|
try:
|
|
codec._write(0x14, 0x1A) # Enable analog mic input & set PGA gain
|
|
codec._write(0x16, 0x01) # Boost MIC digital gain to +6dB
|
|
codec._write(0x17, 0xC8) # Set ADC digital volume
|
|
print("Microphone registers initialized.")
|
|
except Exception as e:
|
|
print("Failed to write microphone registers:", e)
|
|
|
|
amp_pin = Pin(1, Pin.OUT, value=1) # start with amp disabled (1 = disabled)
|
|
buffer = bytearray(3200) # 100ms chunk at 16kHz mono 16-bit PCM (3200 bytes)
|
|
|
|
while True:
|
|
display.clear(0)
|
|
display.text("Hermes Assistant", 10, 10, 1)
|
|
display.line(10, 22, 310, 22, 1)
|
|
display.text("Hold screen & ask", 50, 70, 1)
|
|
display.text("a question...", 50, 90, 1)
|
|
display.text("Status: Idle (WS)", 10, 220, 1)
|
|
display.show()
|
|
|
|
# Wait for touch
|
|
while not touch.is_touched():
|
|
time.sleep_ms(30)
|
|
|
|
print("Touch detected! Connecting WebSocket...")
|
|
display.clear(0)
|
|
display.text("Connecting...", 80, 80, 1)
|
|
display.show()
|
|
|
|
headers = {
|
|
"Authorization": f"Bearer {HERMES_API_KEY}",
|
|
"X-Device-ID": DEVICE_ID
|
|
}
|
|
|
|
ws = WebSocketClient(HERMES_WS_URL, headers=headers, timeout=30)
|
|
try:
|
|
ws.connect()
|
|
|
|
# Send start event
|
|
ws.send_text(json.dumps({
|
|
"event": "start",
|
|
"device_id": DEVICE_ID,
|
|
"sample_rate": 16000,
|
|
"channels": 1,
|
|
"sample_width": 2,
|
|
"format": "pcm_s16le"
|
|
}))
|
|
|
|
# Read ready and listening events from server
|
|
evt1_opcode, evt1_payload = ws.recv_frame()
|
|
evt2_opcode, evt2_payload = ws.recv_frame()
|
|
|
|
print("WebSocket connected and streaming started.")
|
|
display.clear(0)
|
|
display.text("Hermes Assistant", 10, 10, 1)
|
|
display.line(10, 22, 310, 22, 1)
|
|
display.text("Listening...", 80, 80, 1)
|
|
display.fill_rect(130, 110, 30, 30, 1)
|
|
display.text("Status: Streaming", 10, 220, 1)
|
|
display.show()
|
|
|
|
# 4. Open I2S RX for recording (Mono 16kHz)
|
|
i2s_rx = I2S(1,
|
|
sck=Pin(5),
|
|
ws=Pin(7),
|
|
sd=Pin(6),
|
|
mode=I2S.RX,
|
|
ibuf=8000,
|
|
rate=16000,
|
|
bits=16,
|
|
format=I2S.MONO)
|
|
|
|
total_data_bytes = 0
|
|
|
|
try:
|
|
# Record loop - as long as touch is held
|
|
while touch.is_touched():
|
|
bytes_read = i2s_rx.readinto(buffer)
|
|
if bytes_read > 0:
|
|
ws.send_binary(buffer[:bytes_read])
|
|
total_data_bytes += bytes_read
|
|
|
|
print(f"Touch released! Sent {total_data_bytes} bytes.")
|
|
except Exception as e:
|
|
print("Error recording/streaming:", e)
|
|
finally:
|
|
i2s_rx.deinit()
|
|
|
|
if total_data_bytes < 3200:
|
|
print("Recording too short, cancelling session.")
|
|
ws.send_text(json.dumps({"event": "cancel"}))
|
|
ws.close()
|
|
continue
|
|
|
|
# Send stop event
|
|
ws.send_text(json.dumps({"event": "stop"}))
|
|
|
|
display.clear(0)
|
|
display.text("Hermes Assistant", 10, 10, 1)
|
|
display.line(10, 22, 310, 22, 1)
|
|
display.text("Processing...", 50, 80, 1)
|
|
display.text("Status: Thinking", 10, 220, 1)
|
|
display.show()
|
|
|
|
# Playback/Events loop
|
|
i2s_tx = None
|
|
while True:
|
|
opcode, payload = ws.recv_frame()
|
|
if opcode is None:
|
|
break
|
|
|
|
if opcode == 0x1: # Text frame (JSON event)
|
|
try:
|
|
event_data = json.loads(payload.decode('utf-8'))
|
|
evt = event_data.get("event")
|
|
|
|
if evt == "transcript":
|
|
txt = event_data.get("text", "")
|
|
print(f"Heard: {txt}")
|
|
display.clear(0)
|
|
display.text("Hermes Assistant", 10, 10, 1)
|
|
display.line(10, 22, 310, 22, 1)
|
|
display.text("Heard:", 10, 40, 1)
|
|
# Wrap text onto lines
|
|
lines = [txt[i:i+30] for i in range(0, min(len(txt), 120), 30)]
|
|
y_offset = 60
|
|
for line in lines:
|
|
display.text(line, 10, y_offset, 1)
|
|
y_offset += 20
|
|
display.show()
|
|
|
|
elif evt == "thinking":
|
|
pass
|
|
|
|
elif evt == "response_text":
|
|
txt = event_data.get("text", "")
|
|
print(f"Response: {txt}")
|
|
display.clear(0)
|
|
display.text("Hermes Assistant", 10, 10, 1)
|
|
display.line(10, 22, 310, 22, 1)
|
|
display.text("Response:", 10, 40, 1)
|
|
lines = [txt[i:i+30] for i in range(0, min(len(txt), 120), 30)]
|
|
y_offset = 60
|
|
for line in lines:
|
|
display.text(line, 10, y_offset, 1)
|
|
y_offset += 20
|
|
display.show()
|
|
|
|
elif evt == "audio_start":
|
|
print("Audio response started.")
|
|
# Enable amp and open I2S TX
|
|
amp_pin.value(0) # Active Low Enable
|
|
i2s_tx = I2S(1,
|
|
sck=Pin(5),
|
|
ws=Pin(7),
|
|
sd=Pin(8),
|
|
mode=I2S.TX,
|
|
ibuf=4096,
|
|
rate=16000,
|
|
bits=16,
|
|
format=I2S.MONO)
|
|
|
|
elif evt == "audio_end":
|
|
print("Audio response ended.")
|
|
if i2s_tx:
|
|
time.sleep_ms(150)
|
|
amp_pin.value(1) # Disable amp
|
|
i2s_tx.deinit()
|
|
i2s_tx = None
|
|
|
|
elif evt == "done":
|
|
break
|
|
|
|
elif evt == "error":
|
|
msg = event_data.get("message", "Unknown error")
|
|
print(f"Error from server: {msg}")
|
|
display.clear(0)
|
|
display.text("Error", 10, 10, 1)
|
|
display.line(10, 22, 310, 22, 1)
|
|
display.text(msg[:100], 10, 80, 1)
|
|
display.show()
|
|
time.sleep(3)
|
|
break
|
|
except Exception as e:
|
|
print("Error parsing event text:", e)
|
|
|
|
elif opcode == 0x2: # Binary frame (Audio WAV chunk)
|
|
if i2s_tx:
|
|
chunk = payload
|
|
if chunk.startswith(b'RIFF') and len(chunk) > 44:
|
|
chunk = chunk[44:] # Skip WAV header for direct I2S mono play
|
|
try:
|
|
i2s_tx.write(chunk)
|
|
except Exception as e:
|
|
print("Error writing to speaker:", e)
|
|
|
|
ws.close()
|
|
except Exception as e:
|
|
print("Failed to stream to Hermes server:", e)
|
|
display.clear(0)
|
|
display.text("Server Error", 10, 10, 1)
|
|
display.line(10, 22, 310, 22, 1)
|
|
display.text("Could not connect", 50, 80, 1)
|
|
display.text("to WebSocket server.", 50, 100, 1)
|
|
display.show()
|
|
time.sleep(3)
|
|
try:
|
|
ws.close()
|
|
except Exception:
|
|
pass
|
|
|
|
# Debounce touch release
|
|
while touch.is_touched():
|
|
time.sleep_ms(30)
|
|
time.sleep_ms(300)
|
|
|
|
if __name__ == "__main__":
|
|
main()
|