adolforeyna/mcp_screen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix socket read stream crashes and NameError on boot

Browse Source
This commit is contained in:
Adolfo Reyna
2026-06-18 11:35:06 -04:00
parent 5968b4934d
commit d0fcc5fc7b
4 changed files with 208 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/audio_util.py
+1 -1
View File
@@ -73,7 +73,7 @@ class ES7210:
self.i2c.writeto_mem(self.ADDR, reg, bytes([val])) self.i2c.writeto_mem(self.ADDR, reg, bytes([val]))
def record_audio(duration_seconds=5, filename='recording.pcm'): def record_audio(duration_seconds=10, filename='recording.pcm'):
"""Records raw stereo PCM data from the dual microphones to a file. """Records raw stereo PCM data from the dual microphones to a file.
Args: Args:
main.py
+201 -105
View File
@@ -56,6 +56,99 @@ def create_wav_header(data_size):
audio_format, channels, sample_rate, byte_rate, audio_format, channels, sample_rate, byte_rate,
block_align, bits_per_sample, data_label, data_size) block_align, bits_per_sample, data_label, data_size)
def socket_readline(s):
line = bytearray()
while True:
try:
char = s.recv(1)
except OSError:
break
if not char:
break
line.extend(char)
if char == b'\n':
break
return line
def socket_read_exactly(s, n):
res = bytearray()
while len(res) < n:
try:
chunk = s.recv(n - len(res))
except OSError:
break
if not chunk:
break
res.extend(chunk)
return res
def stream_hermes_request(url, headers, filename):
# Parse URL
proto, _, host_port_path = url.split('/', 2)
host_port = host_port_path.split('/', 1)[0]
path = '/' + host_port_path.split('/', 1)[1] if '/' in host_port_path else '/'
if ':' in host_port:
host, port = host_port.split(':')
port = int(port)
else:
host, port = host_port, 80
import socket
addr = socket.getaddrinfo(host, port)[0][-1]
s = socket.socket()
s.settimeout(30.0)
s.connect(addr)
# Calculate file size
import os
try:
file_size = os.stat(filename)[6]
except OSError:
file_size = 0
content_length = file_size + 44 # WAV header + PCM
# Send request headers
s.write(f"POST {path} HTTP/1.1\r\n".encode())
s.write(f"Host: {host_port}\r\n".encode())
for k, v in headers.items():
s.write(f"{k}: {v}\r\n".encode())
s.write(f"Content-Length: {content_length}\r\n".encode())
s.write(b"\r\n")
# Write WAV header
s.write(create_wav_header(file_size))
# Stream audio file from flash
if file_size > 0:
buf = bytearray(2048)
with open(filename, "rb") as f:
while True:
n = f.readinto(buf)
if n == 0:
break
s.write(buf[:n])
# Read status line
status_line = socket_readline(s).decode()
parts = status_line.split(' ')
status_code = int(parts[1]) if len(parts) >= 2 else 500
# Read headers
resp_headers = {}
while True:
line = socket_readline(s)
if line == b"\r\n" or not line:
break
p = line.decode().split(':', 1)
if len(p) == 2:
resp_headers[p[0].strip().lower()] = p[1].strip()
return status_code, resp_headers, s
# LED mode options for manual cycling # LED mode options for manual cycling
led_modes = [ led_modes = [
("Red (Breathing)", lambda led: led.set_color(40, 0, 0), "breath"), ("Red (Breathing)", lambda led: led.set_color(40, 0, 0), "breath"),
@@ -244,29 +337,10 @@ def main():
draw_status_bar("PTT Voice: Initializing...") draw_status_bar("PTT Voice: Initializing...")
# State 1: Wait for user to release the initial trigger press # Start recording immediately!
draw_status_bar("Release button/screen...") tap_started = True
release_start = time.ticks_ms()
while is_talk_trigger_active():
if time.ticks_diff(time.ticks_ms(), release_start) > 2000:
print("Release timeout occurred")
break
time.sleep_ms(30)
# State 2: Wait for tap-to-start recording
draw_status_bar("Ready. Tap key/screen to record...")
tap_started = False
start_wait = time.ticks_ms()
while time.ticks_diff(time.ticks_ms(), start_wait) < 10000: # 10s timeout
if is_talk_trigger_active():
tap_started = True
break
time.sleep_ms(30)
if tap_started: if tap_started:
# User tapped. Let's record! draw_status_bar("Recording: 10s...")
draw_status_bar("Recording: 15s (Tap to stop)")
# 1. Start MCLK PWM and configure mic path based on board config # 1. Start MCLK PWM and configure mic path based on board config
mclk_pwm = None mclk_pwm = None
@@ -303,40 +377,26 @@ def main():
bits=16, bits=16,
format=I2S.MONO) format=I2S.MONO)
audio_chunks = []
total_data_bytes = 0 total_data_bytes = 0
buffer = bytearray(1024) buffer = bytearray(1024)
# Track release of the start-tap first
start_tap_released = False
rec_start_time = time.ticks_ms() rec_start_time = time.ticks_ms()
max_rec_duration_ms = 15000 # 15 seconds max duration max_rec_duration_ms = 10000 # 10 seconds max duration
max_rec_bytes = 480000 # 15 seconds of mono 16kHz 16-bit PCM (32KB/s)
try: try:
while True: with open("voice_rec.pcm", "wb") as f:
# Safety: check duration and byte size while True:
elapsed = time.ticks_diff(time.ticks_ms(), rec_start_time) # Safety: check duration
if elapsed >= max_rec_duration_ms or total_data_bytes >= max_rec_bytes: elapsed = time.ticks_diff(time.ticks_ms(), rec_start_time)
print("Recording stopped: maximum duration/size reached") if elapsed >= max_rec_duration_ms:
break print("Recording stopped: maximum duration reached")
# Read I2S chunk
bytes_read = i2s_rx.readinto(buffer)
if bytes_read > 0:
audio_chunks.append(bytes(buffer[:bytes_read]))
total_data_bytes += bytes_read
# Check release and stop-tap
touched = is_talk_trigger_active()
if not start_tap_released:
if not touched:
start_tap_released = True
print("Start-tap released. Listening for stop-tap...")
else:
if touched:
print("Stop-tap detected. Stopping recording...")
break break
# Read I2S chunk
bytes_read = i2s_rx.readinto(buffer)
if bytes_read > 0:
f.write(buffer[:bytes_read])
total_data_bytes += bytes_read
except Exception as e: except Exception as e:
print("Error recording:", e) print("Error recording:", e)
finally: finally:
@@ -350,80 +410,116 @@ def main():
if total_data_bytes >= 1000: if total_data_bytes >= 1000:
draw_status_bar("Sending & processing...") draw_status_bar("Sending & processing...")
# Prepend WAV header # Determine dynamic device ID based on board configuration
raw_pcm = b"".join(audio_chunks) device_id = "esp32_screen" if board_config.BOARD_TYPE == 'WAVESHARE_RLCD' else "little32"
wav_data = create_wav_header(len(raw_pcm)) + raw_pcm
# Headers # Headers
headers = { headers = {
"Authorization": "Bearer mcT1YA1vOr9wXSiHpCYalweEGGZKX-PIfZv2drp8BSg", "Authorization": "Bearer mcT1YA1vOr9wXSiHpCYalweEGGZKX-PIfZv2drp8BSg",
"Content-Type": "audio/wav", "Content-Type": "audio/wav",
"X-Device-ID": "kitchen-button" "X-Device-ID": device_id,
"X-Hermes-Screen-Device": device_id,
"X-Hermes-Reply-Mode": "ack"
} }
s = None
try: try:
import urequests status_code, resp_headers, s = stream_hermes_request(
res = urequests.post("http://192.168.68.126:8642/api/esp32/voice", "http://192.168.68.126:8642/api/esp32/voice",
data=wav_data, headers,
headers=headers, "voice_rec.pcm"
timeout=30) )
if res.status_code == 200:
response_data = res.content if status_code == 200:
content_type = resp_headers.get("content-type", "")
content_len = int(resp_headers.get("content-length", 0))
# Parse response WAV header if "audio" in content_type and content_len >= 44:
if len(response_data) >= 44 and response_data[0:4] == b'RIFF' and response_data[8:12] == b'WAVE':
draw_status_bar("Playing response...") draw_status_bar("Playing response...")
# Parse channel count, sample rate, bits # Read and parse WAV header (first 44 bytes)
import struct header = socket_read_exactly(s, 44)
fmt_chunk_size = struct.unpack('<I', response_data[16:20])[0] if len(header) == 44 and header[0:4] == b'RIFF' and header[8:12] == b'WAVE':
channels, sample_rate, byte_rate, block_align, bits = struct.unpack('<HHIIH', response_data[22:34]) import struct
fmt_idx = header.find(b'fmt ')
audio_start = 20 + fmt_chunk_size if fmt_idx != -1:
while audio_start < len(response_data) - 8: fmt_data = header[fmt_idx+8 : fmt_idx+24]
if response_data[audio_start:audio_start+4] == b'data': audio_format, channels, sample_rate, byte_rate, block_align, bits = struct.unpack('<HHIIHH', fmt_data)
audio_start += 8
break data_idx = header.find(b'data')
chunk_size = struct.unpack('<I', response_data[audio_start+4:audio_start+8])[0] if data_idx != -1:
audio_start += 8 + chunk_size # Skip remaining bytes to start of audio data
bytes_to_skip = (data_idx + 8) - 44
if audio_start < len(response_data): if bytes_to_skip > 0:
# Play response using board_config parameters socket_read_exactly(s, bytes_to_skip)
on_val = 0 if board_config.audio_amp_active_level == 0 else 1
off_val = 1 if board_config.audio_amp_active_level == 0 else 0 # Play using board_config parameters
amp_pin.value(on_val) # Enable Amp on_val = 0 if board_config.audio_amp_active_level == 0 else 1
off_val = 1 if board_config.audio_amp_active_level == 0 else 0
i2s_format = I2S.MONO if channels == 1 else I2S.STEREO amp_pin.value(on_val) # Enable Amp
i2s_tx = I2S(1,
sck=Pin(board_config.audio_i2s_sck), i2s_format = I2S.MONO if channels == 1 else I2S.STEREO
ws=Pin(board_config.audio_i2s_ws), i2s_tx = I2S(1,
sd=Pin(board_config.audio_i2s_tx_sd), sck=Pin(board_config.audio_i2s_sck),
mode=I2S.TX, ws=Pin(board_config.audio_i2s_ws),
ibuf=4096, sd=Pin(board_config.audio_i2s_tx_sd),
rate=sample_rate, mode=I2S.TX,
bits=bits, ibuf=4096,
format=i2s_format) rate=sample_rate,
try: bits=bits,
pos = audio_start format=i2s_format)
chunk_size = 2048 try:
while pos < len(response_data): remaining_bytes = content_len - (data_idx + 8)
chunk = response_data[pos:pos+chunk_size] while remaining_bytes > 0:
i2s_tx.write(chunk) chunk_to_read = min(remaining_bytes, 2048)
pos += chunk_size try:
finally: chunk = s.recv(chunk_to_read)
time.sleep_ms(150) except OSError:
amp_pin.value(off_val) # Disable Amp break
i2s_tx.deinit() if not chunk:
break
i2s_tx.write(chunk)
remaining_bytes -= len(chunk)
finally:
time.sleep_ms(150)
amp_pin.value(off_val) # Disable Amp
i2s_tx.deinit()
else: else:
draw_status_bar("Error: Gateway returned MP3") # Ack or text response
time.sleep(3) resp_text = ""
if content_len > 0:
try:
resp_text = socket_read_exactly(s, content_len).decode('utf-8', errors='ignore')
except:
pass
print("Non-WAV response received:", resp_text)
if "ack" in resp_text.lower() or "ok" in resp_text.lower() or "success" in resp_text.lower():
draw_status_bar("Success (Ack received)")
else:
draw_status_bar("Response processed.")
time.sleep(2)
else: else:
draw_status_bar(f"HTTP Error: {res.status_code}") content_len = int(resp_headers.get("content-length", 0))
resp_text = "Unknown error"
if content_len > 0:
try:
resp_text = socket_read_exactly(s, content_len).decode('utf-8', errors='ignore')
except:
pass
print("Error response:", resp_text)
draw_status_bar(f"Error: {status_code}")
time.sleep(2) time.sleep(2)
except Exception as he: except Exception as he:
print("Hermes HTTP post failed:", he) print("Hermes HTTP post failed:", he)
sys.print_exception(he)
draw_status_bar("Connection Error") draw_status_bar("Connection Error")
time.sleep(2) time.sleep(2)
finally:
if s:
try:
s.close()
except:
pass
# Deinit MCLK PWM # Deinit MCLK PWM
if mclk_pwm: if mclk_pwm:
mcp_server.py
+1 -1
View File
@@ -595,7 +595,7 @@ class MCPServer:
raise RuntimeError(f"Failed to play audio file '{filename}'. Check format (16kHz 16-bit PCM WAV recommended).") raise RuntimeError(f"Failed to play audio file '{filename}'. Check format (16kHz 16-bit PCM WAV recommended).")
elif name == "record_voice": elif name == "record_voice":
duration = int(args.get("duration_sec", 4)) duration = int(args.get("duration_sec", 10))
filename = str(args.get("filename", "recording.pcm")) filename = str(args.get("filename", "recording.pcm"))
# Clamp duration to a reasonable range # Clamp duration to a reasonable range
duration = max(1, min(15, duration)) duration = max(1, min(15, duration))
voice_assistant.py
+5 -3
View File
@@ -10,7 +10,9 @@ import wave
import time import time
# Default ESP32 IP # Default ESP32 IP
ESP32_IP = "192.168.68.122" ESP32_IP = "192.168.68.124"
if len(sys.argv) > 1:
ESP32_IP = sys.argv[1]
PORT = 80 PORT = 80
URL = f"http://{ESP32_IP}:{PORT}/api/mcp" URL = f"http://{ESP32_IP}:{PORT}/api/mcp"
@@ -268,7 +270,7 @@ def main():
while True: while True:
print("\nReady for command. Choose an option:") print("\nReady for command. Choose an option:")
print(" [Enter] Record a voice command (4 seconds) on the board") print(" [Enter] Record a voice command (10 seconds) on the board")
print(" [t] Type a text command manually") print(" [t] Type a text command manually")
print(" [q] Quit") print(" [q] Quit")
@@ -284,7 +286,7 @@ def main():
continue continue
else: else:
# Voice recording path # Voice recording path
duration = 4 duration = 10
pcm_filename = "recording.pcm" pcm_filename = "recording.pcm"
wav_filename = "temp_recording.wav" wav_filename = "temp_recording.wav"