Save current changes before reorganizing drivers and cleaning workspace

.gitignore

@@ -10,3 +10,7 @@ __pycache__/
 
 # OS generated files
 .DS_Store
+
+# Log files
+desktop_client/*.log
+

audio_util.py

@@ -79,24 +79,30 @@ def record_audio(duration_seconds=5, filename='recording.pcm'):
         duration_seconds (int): How long to record in seconds.
         filename (str): Name of output raw PCM file on the device.
     """
-    # 1. Start I2C Control Bus (SDA=13, SCL=14)
+    # 1. Start I2C Control Bus (SDA=16, SCL=15)
-    i2c = I2C(0, sda=Pin(13), scl=Pin(14))
+    i2c = I2C(0, sda=Pin(16), scl=Pin(15))
+    
+    # 1a. Setup Master Clock (MCLK) on GPIO 4 using PWM (needed for ES8311 ADC)
+    mclk_pin = Pin(4, Pin.OUT)
+    mclk_pwm = machine.PWM(mclk_pin)
+    mclk_pwm.freq(6144000)
+    mclk_pwm.duty_u16(32768)
     
     # 2. Configure I2S Receiver
-    # Pins: sck=BCLK (GPIO 9), ws=WS/LRCK (GPIO 45), sd=DIN (GPIO 10)
+    # Pins: sck=BCLK (GPIO 5), ws=WS/LRCK (GPIO 7), sd=DIN (GPIO 6)
     i2s = I2S(1, 
-              sck=Pin(9), 
+              sck=Pin(5), 
-              ws=Pin(45), 
+              ws=Pin(7), 
-              sd=Pin(10), 
+              sd=Pin(6), 
               mode=I2S.RX, 
               ibuf=16000, 
               rate=16000, 
               bits=16, 
               format=I2S.STEREO)
               
-    # 3. Wake up and configure the ES7210 microphone chip
+    # 3. Wake up and configure the ES8311 codec chip
-    mic_adc = ES7210(i2c)
+    mic_adc = ES8311(i2c)
-    if not mic_adc.init(sample_rate=16000, bit_width=16):
+    if not mic_adc.init(sample_rate=16000):
         i2s.deinit()
         return False
         
@@ -125,7 +131,8 @@ def record_audio(duration_seconds=5, filename='recording.pcm'):
     finally:
         # Always release the I2S peripheral resources
         i2s.deinit()
-        print("I2S receiver deinitialized.")
+        mclk_pwm.deinit()
+        print("I2S receiver and MCLK deinitialized.")
 
 
 class ES8311:
@@ -155,11 +162,11 @@ class ES8311:
             self._write(0x00, 0x00)
             time.sleep_ms(10)
             
-            # Clock Configuration (16kHz sample rate, MCLK=12.288MHz)
+            # Clock Configuration (16kHz sample rate, MCLK=6.144MHz)
             self._write(0x01, 0x3F) # Enable all clocks, use MCLK pin
             self._write(0x02, 0x48) # pre_div=3, pre_mult=1
             self._write(0x03, 0x10) # fs_mode=0, adc_osr=16
-            self._write(0x04, 0x20) # dac_osr=32
+            self._write(0x04, 0x10) # dac_osr=16
             self._write(0x05, 0x00) # adc_div=1, dac_div=1
             self._write(0x06, 0x03) # bclk_div=4 (4-1=3)
             self._write(0x07, 0x00) # lrck_h=0
@@ -219,14 +226,14 @@ def play_tone(frequency=440, duration_ms=1000, volume=50):
     
     print(f"Playing tone: {frequency}Hz for {duration_ms}ms (vol={volume})...")
     
-    # 1. Setup Master Clock (MCLK) on GPIO 16 using PWM
+    # 1. Setup Master Clock (MCLK) on GPIO 4 using PWM
-    mclk_pin = Pin(16, Pin.OUT)
+    mclk_pin = Pin(4, Pin.OUT)
     mclk_pwm = machine.PWM(mclk_pin)
-    mclk_pwm.freq(12288000)
+    mclk_pwm.freq(6144000)
     mclk_pwm.duty_u16(32768)
     
-    # 2. Start I2C Control Bus (SDA=13, SCL=14)
+    # 2. Start I2C Control Bus (SDA=16, SCL=15)
-    i2c = I2C(0, sda=Pin(13), scl=Pin(14))
+    i2c = I2C(0, sda=Pin(16), scl=Pin(15))
     
     # 3. Initialize the ES8311 DAC
     dac = ES8311(i2c)
@@ -237,10 +244,10 @@ def play_tone(frequency=440, duration_ms=1000, volume=50):
     dac.set_volume(volume)
     
     # 4. Configure I2S TX
-    # Pins: sck=BCLK (GPIO 9), ws=WS/LRCK (GPIO 45), sd=DOUT (GPIO 8)
+    # Pins: sck=BCLK (GPIO 5), ws=WS/LRCK (GPIO 7), sd=DOUT (GPIO 8)
     i2s = I2S(1,
-              sck=Pin(9),
+              sck=Pin(5),
-              ws=Pin(45),
+              ws=Pin(7),
               sd=Pin(8),
               mode=I2S.TX,
               ibuf=8000,
@@ -248,8 +255,8 @@ def play_tone(frequency=440, duration_ms=1000, volume=50):
               bits=16,
               format=I2S.STEREO)
               
-    # 5. Enable Speaker Amplifier (GPIO 46)
+    # 5. Enable Speaker Amplifier (GPIO 1, Active Low)
-    amp_pin = Pin(46, Pin.OUT, value=1)
+    amp_pin = Pin(1, Pin.OUT, value=0)
     
     # 6. Generate sine wave cycle
     # Approximate frequency to make integer number of samples per cycle
@@ -277,7 +284,7 @@ def play_tone(frequency=440, duration_ms=1000, volume=50):
         
     # 7. Clean up
     time.sleep_ms(100) # Let the remaining buffer play out
-    amp_pin.value(0)
+    amp_pin.value(1) # Disable amp
     i2s.deinit()
     mclk_pwm.deinit()
     print("Tone playback complete.")
@@ -337,14 +344,14 @@ def play_wav(filename, volume=50):
                 
         print(f"WAV Info: {sample_rate}Hz, {bits} bits, {'Mono' if channels == 1 else 'Stereo'}")
         
-        # 2. Setup Master Clock (MCLK) on GPIO 16 using PWM
+        # 2. Setup Master Clock (MCLK) on GPIO 4 using PWM
-        mclk_pin = Pin(16, Pin.OUT)
+        mclk_pin = Pin(4, Pin.OUT)
         mclk_pwm = machine.PWM(mclk_pin)
-        mclk_pwm.freq(12288000)
+        mclk_pwm.freq(6144000)
         mclk_pwm.duty_u16(32768)
         
-        # 3. Start I2C Control Bus (SDA=13, SCL=14)
+        # 3. Start I2C Control Bus (SDA=16, SCL=15)
-        i2c = I2C(0, sda=Pin(13), scl=Pin(14))
+        i2c = I2C(0, sda=Pin(16), scl=Pin(15))
         
         # 4. Initialize the ES8311 DAC
         dac = ES8311(i2c)
@@ -356,11 +363,11 @@ def play_wav(filename, volume=50):
         dac.set_volume(volume)
         
         # 5. Configure I2S TX
-        # Pins: sck=BCLK (GPIO 9), ws=WS/LRCK (GPIO 45), sd=DOUT (GPIO 8)
+        # Pins: sck=BCLK (GPIO 5), ws=WS/LRCK (GPIO 7), sd=DOUT (GPIO 8)
         i2s_format = I2S.MONO if channels == 1 else I2S.STEREO
         i2s = I2S(1,
-                  sck=Pin(9),
+                  sck=Pin(5),
-                  ws=Pin(45),
+                  ws=Pin(7),
                   sd=Pin(8),
                   mode=I2S.TX,
                   ibuf=4096,
@@ -368,8 +375,8 @@ def play_wav(filename, volume=50):
                   bits=bits,
                   format=i2s_format)
                   
-        # 6. Enable Speaker Amplifier (GPIO 46)
+        # 6. Enable Speaker Amplifier (GPIO 1, Active Low)
-        amp_pin = Pin(46, Pin.OUT, value=1)
+        amp_pin = Pin(1, Pin.OUT, value=0)
         
         # 7. Read and stream chunks to I2S
         buf = bytearray(2048)
@@ -381,7 +388,7 @@ def play_wav(filename, volume=50):
             
         # 8. Clean up
         time.sleep_ms(100) # Let the remaining buffer play out
-        amp_pin.value(0)
+        amp_pin.value(1) # Disable amp
         i2s.deinit()
         mclk_pwm.deinit()
         f.close()

battery_util.py

@@ -3,7 +3,7 @@ from machine import ADC, Pin
 class BatteryMonitor:
     """Utility class for monitoring battery voltage and capacity on ESP32-S3-RLCD-4.2."""
     
-    def __init__(self, pin_num=4):
+    def __init__(self, pin_num=9):
         import sys
         self.adc = ADC(Pin(pin_num))
         if sys.platform == 'esp32':
@@ -18,16 +18,16 @@ class BatteryMonitor:
         try:
             # Try calibrated reading in microvolts first
             uv = self.adc.read_uv()
-            # 3x voltage divider onboard scales 3.0V-4.2V battery to 1.0V-1.4V
+            # 2x voltage divider onboard scales 3.0V-4.2V battery to 1.5V-2.1V
-            voltage = (uv / 1_000_000.0) * 3.0
+            voltage = (uv / 1_000_000.0) * 2.0
             return round(voltage, 3)
         except AttributeError:
             # Fallback if read_uv() is not supported on older MicroPython builds
             try:
                 # Read 12-bit value (0-4095)
                 raw = self.adc.read()
-                # 3.3V reference at 11dB attenuation, 3x divider
+                # 3.3V reference at 11dB attenuation, 2x divider
-                voltage = (raw / 4095.0) * 3.3 * 3.0
+                voltage = (raw / 4095.0) * 3.3 * 2.0
                 return round(voltage, 3)
             except Exception as e:
                 print(f"Error reading battery raw ADC: {e}")

boot.py

@@ -10,7 +10,7 @@ except ImportError:
 if sys.platform == 'rp2':
     import st7796 as display_module
 else:
-    import rlcd as display_module
+    import ili9341 as display_module
 from machine import Pin, SPI
 import wifi_config
 
@@ -24,13 +24,13 @@ def connect_wifi():
     # Initialize display to show connection progress
     display = None
     try:
-        # ESP32-S3 configuration
+        # ESP32-S3 configuration for Hosyond Board
-        spi = SPI(1, baudrate=20000000, polarity=0, phase=0, sck=Pin(11), mosi=Pin(12))
+        spi = SPI(1, baudrate=40000000, polarity=0, phase=0, sck=Pin(12), mosi=Pin(11), miso=Pin(13))
-        display = display_module.RLCD(spi, cs=Pin(40), dc=Pin(5), rst=Pin(41))
+        display = display_module.ILI9341(spi, cs=Pin(10), dc=Pin(46), bl=Pin(45), rst=None)
         display.clear(0)
         
-        display.text("ESP32-S3-RLCD-4.2", 10, 10, 1)
+        display.text("Hosyond ESP32-S3", 10, 10, 1)
-        display.line(10, 20, 390, 20, 1)
+        display.line(10, 20, 310, 20, 1)
         display.text("Connecting to Wi-Fi...", 10, 35, 1)
         display.text(f"SSID: {wifi_config.WIFI_SSID}", 10, 50, 1)
         display.show()
@@ -71,8 +71,8 @@ def connect_wifi():
         print(f"Wi-Fi Connected! IP Address: {ip}")
         if display:
             display.clear(0)
-            display.text("ESP32-S3-RLCD-4.2", 10, 10, 1)
+            display.text("Hosyond ESP32-S3", 10, 10, 1)
-            display.line(10, 20, 390, 20, 1)
+            display.line(10, 20, 310, 20, 1)
             display.text("Wi-Fi Status: Connected!", 10, 35, 1)
             display.text(f"SSID: {wifi_config.WIFI_SSID}", 10, 50, 1)
             display.text(f"IP:   {ip}", 10, 65, 1)
@@ -83,9 +83,9 @@ def connect_wifi():
         print("Wi-Fi Connection Failed.")
         if display:
             display.clear(0)
-            display.text("ESP32-S3-RLCD-4.2", 10, 10, 1)
+            display.text("Hosyond ESP32-S3", 10, 10, 1)
-            display.line(10, 20, 390, 20, 1)
+            display.line(10, 20, 310, 20, 1)
-            display.text("Wi-Fi Status: Connection Failed!", 10, 35, 1)
+            display.text("Wi-Fi Status: Failed!", 10, 35, 1)
             display.text("Check credentials in:", 10, 55, 1)
             display.text("wifi_config.py", 20, 70, 1)
             display.text("Starting offline...", 10, 95, 1)

demo_hermes_voice.py

@@ -0,0 +1,248 @@
+import time
+import struct
+import machine
+from machine import Pin, SPI, I2C, I2S
+import urequests
+import ili9341
+from ft6336u import FT6336U
+from audio_util import ES8311
+
+# --- CONFIGURATION ---
+HERMES_API_URL = "http://192.168.68.126:8642/api/esp32/voice"
+DEVICE_ID = "kitchen-button"
+
+# Placeholder for API Key. Since the key is stored on the Hermes server,
+# please copy-paste the token string here.
+HERMES_API_KEY = "mcT1YA1vOr9wXSiHpCYalweEGGZKX-PIfZv2drp8BSg"
+
+def create_wav_header(data_size):
+    # Generates a 44-byte WAV header for 16kHz, 16-bit mono PCM
+    riff = b'RIFF'
+    file_size = data_size + 36
+    wave = b'WAVE'
+    fmt = b'fmt '
+    chunk_size = 16
+    audio_format = 1 # PCM
+    channels = 1 # Mono
+    sample_rate = 16000
+    bits_per_sample = 16
+    byte_rate = sample_rate * channels * (bits_per_sample // 8)
+    block_align = channels * (bits_per_sample // 8)
+    data_label = b'data'
+    
+    return struct.pack('<4sI4s4sIHHIIHH4sI', 
+                       riff, file_size, wave, fmt, chunk_size, 
+                       audio_format, channels, sample_rate, byte_rate, 
+                       block_align, bits_per_sample, data_label, data_size)
+
+def main():
+    print("--- Starting Hermes Voice Assistant Demo ---")
+    
+    # 1. Initialize display
+    spi = SPI(1, baudrate=40000000, polarity=0, phase=0, sck=Pin(12), mosi=Pin(11), miso=Pin(13))
+    display = ili9341.ILI9341(spi, cs=Pin(10), dc=Pin(46), bl=Pin(45), rst=None)
+    display.clear(0)
+    display.text("Hermes Assistant", 10, 10, 1)
+    display.show()
+    
+    # 2. Initialize touch
+    i2c = I2C(0, sda=Pin(16), scl=Pin(15))
+    touch = FT6336U(i2c, rst_pin=18, int_pin=17, width=320, height=240, swap_xy=True, invert_x=False, invert_y=True)
+    
+    # 3. Configure audio codec ES8311
+    # We need MCLK pin (GPIO 4) active at 6.144 MHz
+    mclk_pin = Pin(4, Pin.OUT)
+    mclk_pwm = machine.PWM(mclk_pin)
+    mclk_pwm.freq(6144000)
+    mclk_pwm.duty_u16(32768)
+    
+    codec = ES8311(i2c)
+    if not codec.init(sample_rate=16000):
+        print("ES8311 init failed!")
+        return
+        
+    codec.set_volume(90)
+    
+    # Configure microphone registers on ES8311
+    try:
+        codec._write(0x14, 0x1A) # Enable analog mic input & set PGA gain
+        codec._write(0x16, 0x01) # Boost MIC digital gain to +6dB
+        codec._write(0x17, 0xC8) # Set ADC digital volume
+        print("Microphone registers initialized.")
+    except Exception as e:
+        print("Failed to write microphone registers:", e)
+        
+    amp_pin = Pin(1, Pin.OUT, value=1) # start with amp disabled (1 = disabled)
+    
+    buffer = bytearray(1024)
+    
+    while True:
+        display.clear(0)
+        display.text("Hermes Assistant", 10, 10, 1)
+        display.line(10, 22, 310, 22, 1)
+        display.text("Hold screen & ask", 50, 70, 1)
+        display.text("a question...", 50, 90, 1)
+        display.text("Status: Idle", 10, 220, 1)
+        display.show()
+        
+        # Wait for touch
+        while not touch.is_touched():
+            time.sleep_ms(30)
+            
+        print("Touch detected! Starting recording to RAM...")
+        display.clear(0)
+        display.text("Hermes Assistant", 10, 10, 1)
+        display.line(10, 22, 310, 22, 1)
+        display.text("Listening...", 80, 80, 1)
+        display.fill_rect(130, 110, 30, 30, 1)
+        display.text("Status: Recording", 10, 220, 1)
+        display.show()
+        
+        # 4. Open I2S RX for recording (Mono 16kHz for Hermes STT)
+        i2s_rx = I2S(1, 
+                     sck=Pin(5), 
+                     ws=Pin(7), 
+                     sd=Pin(6), 
+                     mode=I2S.RX, 
+                     ibuf=8000, 
+                     rate=16000, 
+                     bits=16, 
+                     format=I2S.MONO)
+                     
+        audio_chunks = []
+        total_data_bytes = 0
+        
+        try:
+            # Record loop - as long as touch is held
+            while touch.is_touched():
+                bytes_read = i2s_rx.readinto(buffer)
+                if bytes_read > 0:
+                    audio_chunks.append(bytes(buffer[:bytes_read]))
+                    total_data_bytes += bytes_read
+                
+            print(f"Touch released! Recorded {total_data_bytes} bytes.")
+                
+        except Exception as e:
+            print("Error recording:", e)
+        finally:
+            i2s_rx.deinit()
+            
+        if total_data_bytes < 1000:
+            print("Recording too short, ignoring.")
+            continue
+            
+        # 5. Connect and POST to Hermes voice endpoint
+        display.clear(0)
+        display.text("Hermes Assistant", 10, 10, 1)
+        display.line(10, 22, 310, 22, 1)
+        display.text("Sending audio...", 50, 80, 1)
+        display.text("Waiting for agent...", 50, 100, 1)
+        display.text("Status: Processing", 10, 220, 1)
+        display.show()
+        
+        # Join chunks and prepend WAV header
+        raw_pcm = b"".join(audio_chunks)
+        wav_data = create_wav_header(len(raw_pcm)) + raw_pcm
+        
+        # Headers
+        headers = {
+            "Authorization": f"Bearer {HERMES_API_KEY}",
+            "Content-Type": "audio/wav",
+            "X-Device-ID": DEVICE_ID
+        }
+        
+        print(f"Posting raw WAV ({len(wav_data)} bytes) to {HERMES_API_URL}...")
+        try:
+            res = urequests.post(HERMES_API_URL, headers=headers, data=wav_data)
+            print(f"HTTP Status: {res.status_code}")
+            
+            if res.status_code == 200:
+                response_data = res.content
+                print(f"Received {len(response_data)} bytes of audio response.")
+                
+                # Check response format
+                if response_data[0:4] == b'RIFF' and response_data[8:12] == b'WAVE':
+                    # Parse WAV header
+                    idx = response_data.find(b'fmt ')
+                    if idx != -1:
+                        fmt_chunk = response_data[idx:idx+24]
+                        audio_format, channels, sample_rate, byte_rate, block_align, bits = struct.unpack('<HHIIHH', fmt_chunk[8:24])
+                        print(f"Playing WAV: {sample_rate}Hz, {channels}ch, {bits}bit")
+                        
+                        # Find data chunk
+                        data_idx = response_data.find(b'data')
+                        if data_idx != -1:
+                            audio_start = data_idx + 8
+                            
+                            display.clear(0)
+                            display.text("Hermes Assistant", 10, 10, 1)
+                            display.line(10, 22, 310, 22, 1)
+                            display.text("Playing response...", 50, 80, 1)
+                            display.text("Status: Speaking", 10, 220, 1)
+                            display.show()
+                            
+                            # Enable amp & play
+                            amp_pin.value(0) # Active Low Enable
+                            i2s_format = I2S.MONO if channels == 1 else I2S.STEREO
+                            i2s_tx = I2S(1,
+                                         sck=Pin(5),
+                                         ws=Pin(7),
+                                         sd=Pin(8),
+                                         mode=I2S.TX,
+                                         ibuf=4096,
+                                         rate=sample_rate,
+                                         bits=bits,
+                                         format=i2s_format)
+                            try:
+                                pos = audio_start
+                                chunk_size = 2048
+                                while pos < len(response_data):
+                                    chunk = response_data[pos:pos+chunk_size]
+                                    i2s_tx.write(chunk)
+                                    pos += chunk_size
+                            finally:
+                                time.sleep_ms(150)
+                                amp_pin.value(1) # Disable
+                                i2s_tx.deinit()
+                    else:
+                        print("Error: Could not parse WAV format.")
+                else:
+                    # Check if it is MP3
+                    if response_data[0:3] == b'ID3' or (len(response_data) > 2 and response_data[0] == 0xFF and (response_data[1] & 0xE0) == 0xE0):
+                        print("Error: Server returned MP3 format. ESP32 requires WAV.")
+                        display.clear(0)
+                        display.text("Error: Got MP3", 10, 10, 1)
+                        display.line(10, 22, 310, 22, 1)
+                        display.text("Hermes returned MP3.", 30, 80, 1)
+                        display.text("Configure server to", 30, 100, 1)
+                        display.text("convert MP3 to WAV.", 30, 120, 1)
+                        display.show()
+                        time.sleep(4)
+                    else:
+                        print(f"Error: Unknown response format. Starts with: {response_data[0:16]}")
+            else:
+                print(f"Request failed with status {res.status_code}: {res.text}")
+                display.clear(0)
+                display.text("HTTP Error", 10, 10, 1)
+                display.line(10, 22, 310, 22, 1)
+                display.text(f"Status: {res.status_code}", 50, 80, 1)
+                display.show()
+                time.sleep(3)
+                
+        except Exception as e:
+            print("Failed to contact Hermes server:", e)
+            display.clear(0)
+            display.text("Server Error", 10, 10, 1)
+            display.line(10, 22, 310, 22, 1)
+            display.text("Could not connect", 50, 80, 1)
+            display.text("to API gateway.", 50, 100, 1)
+            display.show()
+            time.sleep(3)
+            
+        # Debounce touch release
+        while touch.is_touched():
+            time.sleep_ms(30)
+        time.sleep_ms(300)
+
+if __name__ == "__main__":
+    main()

demo_touch_mic.py

@@ -0,0 +1,144 @@
+import time
+import machine
+from machine import Pin, SPI, I2C, I2S
+import ili9341
+from ft6336u import FT6336U
+from audio_util import ES8311
+
+def main():
+    print("--- Starting RAM-based Touch Mic Demo ---")
+    
+    # 1. Initialize display
+    spi = SPI(1, baudrate=40000000, polarity=0, phase=0, sck=Pin(12), mosi=Pin(11), miso=Pin(13))
+    display = ili9341.ILI9341(spi, cs=Pin(10), dc=Pin(46), bl=Pin(45), rst=None)
+    display.clear(0)
+    display.text("RAM Touch Mic", 10, 10, 1)
+    display.show()
+    
+    # 2. Initialize touch
+    i2c = I2C(0, sda=Pin(16), scl=Pin(15))
+    touch = FT6336U(i2c, rst_pin=18, int_pin=17, width=320, height=240, swap_xy=True, invert_x=False, invert_y=True)
+    
+    # 3. Configure audio codec ES8311
+    # Generate MCLK PWM on GPIO 4 at 6.144 MHz
+    mclk_pin = Pin(4, Pin.OUT)
+    mclk_pwm = machine.PWM(mclk_pin)
+    mclk_pwm.freq(6144000)
+    mclk_pwm.duty_u16(32768)
+    
+    codec = ES8311(i2c)
+    if not codec.init(sample_rate=16000):
+        print("ES8311 init failed!")
+        return
+        
+    # Set speaker volume to 95%
+    codec.set_volume(95)
+    
+    # Configure microphone registers on ES8311
+    try:
+        codec._write(0x14, 0x1A) # Enable analog mic input & set PGA gain
+        codec._write(0x16, 0x01) # Boost MIC digital gain to +6dB to prevent saturation
+        codec._write(0x17, 0xC8) # Set ADC digital volume
+        print("Microphone registers initialized with gain boost.")
+    except Exception as e:
+        print("Failed to write microphone registers:", e)
+        
+    amp_pin = Pin(1, Pin.OUT, value=1) # start with amp disabled (1 = disabled)
+    
+    # We allocate a buffer for I2S read
+    buffer = bytearray(2048)
+    
+    while True:
+        display.clear(0)
+        display.text("RAM Touch Mic", 10, 10, 1)
+        display.line(10, 22, 310, 22, 1)
+        display.text("Press & hold screen", 50, 80, 1)
+        display.text("to record voice...", 50, 100, 1)
+        display.show()
+        
+        # Wait for touch
+        while not touch.is_touched():
+            time.sleep_ms(30)
+            
+        print("Touch detected! Starting recording to RAM...")
+        display.clear(0)
+        display.text("Recording...", 10, 10, 1)
+        display.line(10, 22, 310, 22, 1)
+        display.text("SPEAK NOW!", 80, 80, 1)
+        display.fill_rect(130, 110, 30, 30, 1)
+        display.show()
+        
+        # 4. Open I2S RX for recording
+        i2s_rx = I2S(1, 
+                     sck=Pin(5), 
+                     ws=Pin(7), 
+                     sd=Pin(6), 
+                     mode=I2S.RX, 
+                     ibuf=8000, 
+                     rate=16000, 
+                     bits=16, 
+                     format=I2S.STEREO)
+                     
+        audio_chunks = []
+        total_data_bytes = 0
+        
+        try:
+            # Record loop - append chunks directly in RAM
+            while touch.is_touched():
+                bytes_read = i2s_rx.readinto(buffer)
+                if bytes_read > 0:
+                    audio_chunks.append(bytes(buffer[:bytes_read]))
+                    total_data_bytes += bytes_read
+                
+            print(f"Touch released! Recorded {total_data_bytes} bytes in RAM.")
+                
+        except Exception as e:
+            print("Error recording:", e)
+        finally:
+            i2s_rx.deinit()
+            
+        # 5. Playback recorded audio
+        display.clear(0)
+        display.text("Playing back...", 10, 10, 1)
+        display.line(10, 22, 310, 22, 1)
+        display.text("Listening to RAM...", 50, 80, 1)
+        display.show()
+        
+        print("Starting playback from RAM...")
+        amp_pin.value(0) # Enable amp
+        
+        i2s_tx = I2S(1,
+                     sck=Pin(5),
+                     ws=Pin(7),
+                     sd=Pin(8),
+                     mode=I2S.TX,
+                     ibuf=4096,
+                     rate=16000,
+                     bits=16,
+                     format=I2S.STEREO)
+                     
+        try:
+            for chunk in audio_chunks:
+                i2s_tx.write(chunk)
+        except Exception as e:
+            print("Error during playback:", e)
+        finally:
+            time.sleep_ms(100) # wait to clear buffer
+            amp_pin.value(1) # Disable amp
+            i2s_tx.deinit()
+            
+        print("Playback finished.")
+        display.clear(0)
+        display.text("Finished!", 10, 10, 1)
+        display.line(10, 22, 310, 22, 1)
+        display.text("Touch screen to", 50, 80, 1)
+        display.text("record again.", 50, 100, 1)
+        display.show()
+        
+        # Debounce touch release
+        while touch.is_touched():
+            time.sleep_ms(30)
+        time.sleep_ms(400)
+
+if __name__ == "__main__":
+    main()

desktop_client/README.md

@@ -0,0 +1,79 @@
+# Desktop Companion Client for AI Agents
+
+This is a cross-platform (Linux, macOS, Windows) client that allows your computer to be used directly by AI Agents as a physical-like communication and display companion.
+
+It implements the Model Context Protocol (MCP) and exposes the host machine's actual hardware:
+- **Audio Output**: Plays pure tones and WAV files on your actual speakers.
+- **Audio Input**: Records voice messages from your microphone.
+- **Battery Status**: Exposes your laptop's real battery voltage and percentage.
+- **System Telemetry**: Reads host metrics (uptime, CPU loading).
+- **Dedicated Canvas Window**: Renders images and text drawn by the agent.
+- **Video Streaming**: Renders incoming 1-bit monochrome video streams (from sources streaming to TCP port 8081 or UDP port 8082).
+- **UDP Discovery**: Responds to local discovery beacons.
+
+## Requirements
+
+- Python 3.10+
+- **Pillow** (PIL) library (already installed in your environment)
+- **Audio Playback**: Uses native system players (`afplay` on macOS, `aplay`/`paplay`/`pw-play` on Linux, `winsound` on Windows).
+- **Audio Recording**:
+  - **Linux**: Standard `arecord` (pre-installed via `alsa-utils`).
+  - **macOS / Windows**: If `sounddevice` or `pyaudio` Python packages are installed, they will be used. Otherwise, it falls back to `sox`/`rec` if available, or generates a clean simulated wave format if no audio recorder is present.
+
+## Running the Client
+
+Start the client manually with:
+```bash
+python3 desktop_client/client.py
+```
+
+Options:
+- `--port`: HTTP JSON-RPC port (default is `8080`)
+- `--width`: Canvas width (default `480`)
+- `--height`: Canvas height (default `320`)
+- `--headless`: Run headlessly without Pygame GUI window (ideal for remote SSH servers)
+
+### Keyboard Controls (GUI Mode)
+- **`D`**: Switch the screen canvas back to the local **Host Telemetry Dashboard**.
+- **`S`**: Save a screenshot of the companion window.
+
+### Hidden Mode Behavior
+To avoid cluttering your screen, the GUI window starts **hidden** on boot. It runs silently in the background and only pops to the foreground when a message (drawing or video stream) is sent by the AI Agent.
+
+---
+
+## Running as a macOS Service (LaunchAgent)
+
+To run the companion client persistently as a background service that launches automatically on login, use the provided scripts:
+
+1. **Install and Start the Service**:
+   ```bash
+   ./desktop_client/setup_service.sh
+   ```
+2. **Uninstall/Stop the Service**:
+   ```bash
+   ./desktop_client/uninstall_service.sh
+   ```
+
+Logs are captured dynamically at:
+- Standard Out: `desktop_client/stdout.log`
+- Standard Error: `desktop_client/stderr.log`
+
+---
+
+## Configuring Claude Desktop / Agent Host
+
+To add this desktop client as an MCP tool provider, add the following to your `claude_desktop_config.json`:
+
+```json
+{
+  "mcpServers": {
+    "desktop-companion": {
+      "command": "/Users/adolforeyna/.pyenv/versions/3.10.12/bin/python3",
+      "args": [
+        "/Users/adolforeyna/Projects/MicroPython/test1/Screen/desktop_client/client.py"
+      ]
+    }
+  }
+}
+```

desktop_client/com.adolforeyna.companionclient.plist

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>Label</key>
+    <string>com.adolforeyna.companionclient</string>
+    <key>ProgramArguments</key>
+    <array>
+        <string>/Users/adolforeyna/.pyenv/versions/3.10.12/bin/python3</string>
+        <string>/Users/adolforeyna/Projects/MicroPython/test1/Screen/desktop_client/client.py</string>
+    </array>
+    <key>WorkingDirectory</key>
+    <string>/Users/adolforeyna/Projects/MicroPython/test1/Screen</string>
+    <key>RunAtLoad</key>
+    <true/>
+    <key>KeepAlive</key>
+    <true/>
+    <key>StandardOutPath</key>
+    <string>/Users/adolforeyna/Projects/MicroPython/test1/Screen/desktop_client/stdout.log</string>
+    <key>StandardErrorPath</key>
+    <string>/Users/adolforeyna/Projects/MicroPython/test1/Screen/desktop_client/stderr.log</string>
+</dict>
+</plist>

desktop_client/setup_service.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+PLIST_NAME="com.adolforeyna.companionclient.plist"
+LOCAL_PLIST="/Users/adolforeyna/Projects/MicroPython/test1/Screen/desktop_client/${PLIST_NAME}"
+TARGET_DIR="${HOME}/Library/LaunchAgents"
+TARGET_PLIST="${TARGET_DIR}/${PLIST_NAME}"
+
+echo "=== Installing Desktop Companion LaunchAgent ==="
+
+# 1. Create target directory if it doesn't exist
+mkdir -p "${TARGET_DIR}"
+
+# 2. Unload the service if it's already running
+echo "Stopping existing agent if running..."
+launchctl bootout gui/$(id -u) "${TARGET_PLIST}" 2>/dev/null
+launchctl unload "${TARGET_PLIST}" 2>/dev/null
+
+# 3. Copy the plist file
+echo "Copying plist configuration..."
+cp "${LOCAL_PLIST}" "${TARGET_PLIST}"
+chmod 644 "${TARGET_PLIST}"
+
+# 4. Load/start the LaunchAgent
+echo "Loading and starting the service..."
+launchctl bootstrap gui/$(id -u) "${TARGET_PLIST}"
+# Or fallback if bootstrap fails
+if [ $? -ne 0 ]; then
+    launchctl load "${TARGET_PLIST}"
+fi
+
+echo "Service successfully installed and started!"
+echo "Check status using: launchctl list | grep com.adolforeyna.companionclient"
+echo "Logs are available at:"
+echo "  STDOUT: /Users/adolforeyna/Projects/MicroPython/test1/Screen/desktop_client/stdout.log"
+echo "  STDERR: /Users/adolforeyna/Projects/MicroPython/test1/Screen/desktop_client/stderr.log"

desktop_client/uninstall_service.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+PLIST_NAME="com.adolforeyna.companionclient.plist"
+TARGET_PLIST="${HOME}/Library/LaunchAgents/${PLIST_NAME}"
+
+echo "=== Uninstalling Desktop Companion LaunchAgent ==="
+
+if [ -f "${TARGET_PLIST}" ]; then
+    echo "Stopping and unloading the service..."
+    launchctl bootout gui/$(id -u) "${TARGET_PLIST}" 2>/dev/null
+    launchctl unload "${TARGET_PLIST}" 2>/dev/null
+    
+    echo "Removing plist configuration..."
+    rm "${TARGET_PLIST}"
+    echo "Service successfully uninstalled!"
+else
+    echo "Service is not installed."
+fi

ft6336u.py

@@ -78,8 +78,18 @@ class FT6336U:
         return True
 
     def is_touched(self):
-        """Returns True if screen is touched (the active-low INT pin is pulled LOW)."""
+        """Returns True if screen is touched by checking the TD_STATUS register and clearing coordinates."""
-        return self.int() == 0
+        if not self.initialized:
+            return False
+        td_status = self.read_reg(self.REG_TD_STATUS)
+        if td_status is None:
+            return False
+        touch_count = td_status[0] & 0x0F
+        if touch_count > 0 and touch_count < 3:
+            # Read P1 coordinates to clear register/interrupt state on the chip
+            self.read_reg(self.REG_P1_XH, 6)
+            return True
+        return False
 
     def read_touch(self):
         """Reads touch point coordinates.

ili9341.py

@@ -0,0 +1,261 @@
+import time
+from machine import Pin, SPI
+import framebuf
+import micropython
+
+class ILI9341:
+    def __init__(self, spi, cs, dc, rst=None, bl=None, width=320, height=240, invert_color=True):
+        self.spi = spi
+        self.cs = cs
+        self.dc = dc
+        self.rst = rst
+        self.bl = bl
+        self.width = width
+        self.height = height
+        self.invert_color = invert_color
+        
+        # 1. 1-bit Canvas Buffer (Standard MONO_HLSB for drawing)
+        self.hw_len = (self.width * self.height) // 8
+        self.canvas_buffer = bytearray(self.hw_len)
+        self.canvas = framebuf.FrameBuffer(self.canvas_buffer, self.width, self.height, framebuf.MONO_HLSB)
+        
+        # Pre-allocate chunk buffer for conversion (16 rows: 320 * 16 * 2 = 10240 bytes)
+        self.chunk_rows = 16
+        self.row_buffer = bytearray(self.width * self.chunk_rows * 2)
+        
+        # Initialize pins
+        self.cs.init(self.cs.OUT, value=1)
+        self.dc.init(self.dc.OUT, value=0)
+        
+        if self.rst is not None:
+            self.rst.init(self.rst.OUT, value=1)
+            
+        if self.bl is not None:
+            self.bl.init(self.bl.OUT, value=1)
+                
+        self.reset()
+        self.init_display()
+        self.clear(0)
+        self.show()
+
+    # --- DRAWING WRAPPERS ---
+    def pixel(self, x, y, c): self.canvas.pixel(x, y, c)
+    def line(self, x1, y1, x2, y2, c): self.canvas.line(x1, y1, x2, y2, c)
+    def rect(self, x, y, w, h, c): self.canvas.rect(x, y, w, h, c)
+    def fill_rect(self, x, y, w, h, c): self.canvas.fill_rect(x, y, w, h, c)
+    def text(self, msg, x, y, c=1): self.canvas.text(msg, x, y, c)
+    def clear(self, c=0): self.canvas.fill(c)
+
+    # --- SCALABLE TEXT ---
+    def text_large(self, msg, x, y, scale=2, c=1):
+        char_w = 8; char_h = 8
+        tmp_buf = bytearray(char_w * char_h // 8)
+        tmp_fb = framebuf.FrameBuffer(tmp_buf, char_w, char_h, framebuf.MONO_HLSB)
+        for char in msg:
+            tmp_fb.fill(0); tmp_fb.text(char, 0, 0, 1)
+            for py in range(8):
+                for px in range(8):
+                    if tmp_fb.pixel(px, py):
+                        self.canvas.fill_rect(x + (px * scale), y + (py * scale), scale, scale, c)
+            x += (8 * scale)
+
+    # --- RAW BITMAPS (1:1 scale) ---
+    def bitmap(self, x, y, w, h, pixel_data):
+        img = framebuf.FrameBuffer(pixel_data, w, h, framebuf.MONO_HLSB)
+        self.canvas.blit(img, x, y)
+
+    # --- PBM FILE LOADER WITH SCALING ---
+    def draw_pbm(self, filename, x, y, scale=1):
+        try:
+            with open(filename, 'rb') as f:
+                line1 = f.readline()
+                if not line1.startswith(b'P4'): print("Err: Not P4 PBM"); return
+                while True:
+                    line = f.readline()
+                    if not line.startswith(b'#'): break
+                dims = line.split(); w = int(dims[0]); h = int(dims[1])
+                data = bytearray(f.read())
+                
+                src_fb = framebuf.FrameBuffer(data, w, h, framebuf.MONO_HLSB)
+                
+                if scale == 1:
+                     self.canvas.blit(src_fb, x, y)
+                else:
+                    for sy in range(h):
+                        for sx in range(w):
+                            if src_fb.pixel(sx, sy):
+                                self.canvas.fill_rect(x + (sx * scale), y + (sy * scale), scale, scale, 1)
+                print(f"Loaded {filename} (scale {scale})")
+        except OSError:
+            print(f"Error: Could not open {filename}")
+
+    # --- SCREENSHOT ---
+    def save_screenshot(self, filename):
+        print(f"Saving screenshot to {filename}...")
+        try:
+            with open(filename, 'wb') as f:
+                f.write(b'P4\n')
+                f.write(f"{self.width} {self.height}\n".encode())
+                f.write(self.canvas_buffer)
+            print("Saved!")
+        except Exception as e:
+            print(f"Error saving screenshot: {e}")
+
+    # --- HARDWARE LOGIC ---
+    def reset(self):
+        if self.rst is not None:
+            self.rst(1); time.sleep_ms(5); self.rst(0); time.sleep_ms(15); self.rst(1); time.sleep_ms(15)
+
+    def write_cmd(self, cmd):
+        self.dc(0); self.cs(0); self.spi.write(bytearray([cmd])); self.cs(1)
+
+    def write_data(self, data):
+        self.dc(1); self.cs(0)
+        if isinstance(data, int): self.spi.write(bytearray([data]))
+        elif isinstance(data, list): self.spi.write(bytearray(data))
+        else: self.spi.write(data)
+        self.cs(1)
+
+    def init_display(self):
+        # ILI9341 Initialization Sequence
+        self.write_cmd(0x01) # SWRESET
+        time.sleep_ms(150)
+        
+        self.write_cmd(0xCF); self.write_data(b"\x00\xC1\x30") 
+        self.write_cmd(0xED); self.write_data(b"\x64\x03\x12\x81")
+        self.write_cmd(0xE8); self.write_data(b"\x85\x00\x78")
+        self.write_cmd(0xCB); self.write_data(b"\x39\x2C\x00\x34\x02")
+        self.write_cmd(0xF7); self.write_data(b"\x20")
+        self.write_cmd(0xEA); self.write_data(b"\x00\x00")
+        
+        self.write_cmd(0xC0); self.write_data(b"\x13") # Power Control 1
+        self.write_cmd(0xC1); self.write_data(b"\x13") # Power Control 2
+        self.write_cmd(0xC5); self.write_data(b"\x22\x35") # VCOM Control 1
+        self.write_cmd(0xC7); self.write_data(b"\xBD") # VCOM Control 2
+        
+        # Memory Access Control (MADCTL) = 0x68 (Landscape: MV=1, MX=1, MY=0, BGR color filter)
+        self.write_cmd(0x36); self.write_data(b"\x68")
+        
+        self.write_cmd(0xB6); self.write_data(b"\x0A\xA2") # Display Function Control
+        self.write_cmd(0x3A); self.write_data(b"\x55") # Pixel Format (COLMOD) = 16-bit RGB565
+        self.write_cmd(0xF6); self.write_data(b"\x01\x30")
+        self.write_cmd(0xB1); self.write_data(b"\x00\x1B") # Frame Rate Control
+        self.write_cmd(0xF2); self.write_data(b"\x00")
+        self.write_cmd(0x26); self.write_data(b"\x01") # Gamma Curve
+        
+        self.write_cmd(0xE0); self.write_data(b"\x0F\x35\x31\x0B\x0E\x06\x49\xA7\x33\x07\x0F\x03\x0C\x0A\x00") # Positive Gamma Correction
+        self.write_cmd(0xE1); self.write_data(b"\x00\x0A\x0F\x04\x11\x08\x36\x58\x4D\x07\x10\x0C\x32\x34\x0F") # Negative Gamma Correction
+        
+        if self.invert_color:
+            self.write_cmd(0x21) # INVON
+        else:
+            self.write_cmd(0x20) # INVOFF
+            
+        self.write_cmd(0x11) # SLPOUT (Exit sleep mode)
+        time.sleep_ms(120)
+        self.write_cmd(0x29) # DISPON (Display on)
+        time.sleep_ms(10)
+
+    def invert(self, enable):
+        if enable:
+            self.write_cmd(0x21) # INVON
+        else:
+            self.write_cmd(0x20) # INVOFF
+
+    def set_window(self, x0, y0, x1, y1):
+        # Column Address Set (CASET)
+        self.write_cmd(0x2A)
+        self.write_data(bytearray([x0 >> 8, x0 & 0xFF, x1 >> 8, x1 & 0xFF]))
+        
+        # Row Address Set (RASET)
+        self.write_cmd(0x2B)
+        self.write_data(bytearray([y0 >> 8, y0 & 0xFF, y1 >> 8, y1 & 0xFF]))
+        
+        # Memory Write (RAMWR)
+        self.write_cmd(0x2C)
+
+    @micropython.native
+    def _convert_rows(self, start_row, num_rows, row_buf):
+        """Converts 1-bit monochrome row segment to 16-bit RGB565 format.
+        Compiles block-wise bitwise operations at native speed.
+        """
+        width = self.width
+        canvas_buf = self.canvas_buffer
+        idx = 0
+        
+        for y in range(start_row, start_row + num_rows):
+            byte_offset = y * (width // 8)
+            for x_byte_idx in range(width // 8):
+                val = canvas_buf[byte_offset + x_byte_idx]
+                
+                # Unroll 8 bits for speed
+                # Bit 7
+                if val & 0x80:
+                    row_buf[idx] = 0xFF; row_buf[idx+1] = 0xFF
+                else:
+                    row_buf[idx] = 0x00; row_buf[idx+1] = 0x00
+                idx += 2
+                
+                # Bit 6
+                if val & 0x40:
+                    row_buf[idx] = 0xFF; row_buf[idx+1] = 0xFF
+                else:
+                    row_buf[idx] = 0x00; row_buf[idx+1] = 0x00
+                idx += 2
+                
+                # Bit 5
+                if val & 0x20:
+                    row_buf[idx] = 0xFF; row_buf[idx+1] = 0xFF
+                else:
+                    row_buf[idx] = 0x00; row_buf[idx+1] = 0x00
+                idx += 2
+                
+                # Bit 4
+                if val & 0x10:
+                    row_buf[idx] = 0xFF; row_buf[idx+1] = 0xFF
+                else:
+                    row_buf[idx] = 0x00; row_buf[idx+1] = 0x00
+                idx += 2
+                
+                # Bit 3
+                if val & 0x08:
+                    row_buf[idx] = 0xFF; row_buf[idx+1] = 0xFF
+                else:
+                    row_buf[idx] = 0x00; row_buf[idx+1] = 0x00
+                idx += 2
+                
+                # Bit 2
+                if val & 0x04:
+                    row_buf[idx] = 0xFF; row_buf[idx+1] = 0xFF
+                else:
+                    row_buf[idx] = 0x00; row_buf[idx+1] = 0x00
+                idx += 2
+                
+                # Bit 1
+                if val & 0x02:
+                    row_buf[idx] = 0xFF; row_buf[idx+1] = 0xFF
+                else:
+                    row_buf[idx] = 0x00; row_buf[idx+1] = 0x00
+                idx += 2
+                
+                # Bit 0
+                if val & 0x01:
+                    row_buf[idx] = 0xFF; row_buf[idx+1] = 0xFF
+                else:
+                    row_buf[idx] = 0x00; row_buf[idx+1] = 0x00
+                idx += 2
+
+    def show(self):
+        """Refreshes the screen by writing the frame buffer segment-by-segment."""
+        self.set_window(0, 0, self.width - 1, self.height - 1)
+        
+        self.dc(1)
+        self.cs(0)
+        
+        num_chunks = self.height // self.chunk_rows
+        for chunk in range(num_chunks):
+            start_row = chunk * self.chunk_rows
+            self._convert_rows(start_row, self.chunk_rows, self.row_buffer)
+            self.spi.write(self.row_buffer)
+            
+        self.cs(1)

main.py

@@ -1,7 +1,7 @@
 import time
 import sys
 import machine
-from machine import Pin, SPI, I2C
+from machine import Pin, SPI, I2C, I2S
 
 try:
     import network
@@ -12,7 +12,7 @@ except ImportError:
 if sys.platform == 'rp2':
     import st7796 as display_module
 else:
-    import rlcd as display_module
+    import ili9341 as display_module
 
 class DummyMCP:
     def __init__(self):
@@ -36,6 +36,29 @@ from rgb_led_util import BoardLED
 from ble_util import BLEUART
 from mcp_server import MCPServer
 from video_stream import VideoStreamServer
+from audio_util import ES8311
+import struct
+import urequests
+
+def create_wav_header(data_size):
+    # Generates a 44-byte WAV header for 16kHz, 16-bit mono PCM
+    riff = b'RIFF'
+    file_size = data_size + 36
+    wave = b'WAVE'
+    fmt = b'fmt '
+    chunk_size = 16
+    audio_format = 1 # PCM
+    channels = 1 # Mono
+    sample_rate = 16000
+    bits_per_sample = 16
+    byte_rate = sample_rate * channels * (bits_per_sample // 8)
+    block_align = channels * (bits_per_sample // 8)
+    data_label = b'data'
+    
+    return struct.pack('<4sI4s4sIHHIIHH4sI', 
+                       riff, file_size, wave, fmt, chunk_size, 
+                       audio_format, channels, sample_rate, byte_rate, 
+                       block_align, bits_per_sample, data_label, data_size)
 
 # LED mode options for manual cycling
 led_modes = [
@@ -57,7 +80,7 @@ def main():
     i2c = None
     if sys.platform != 'rp2':
         try:
-            i2c = I2C(0, sda=Pin(13), scl=Pin(14))
+            i2c = I2C(0, sda=Pin(16), scl=Pin(15))
         except Exception as e:
             print("Failed to initialize I2C0:", e)
     
@@ -74,13 +97,18 @@ def main():
         except Exception as te:
             print("Failed to initialize touch:", te)
     else:
-        spi = SPI(1, baudrate=20000000, polarity=0, phase=0, sck=Pin(11), mosi=Pin(12))
+        spi = SPI(1, baudrate=40000000, polarity=0, phase=0, sck=Pin(12), mosi=Pin(11), miso=Pin(13))
-        display = display_module.RLCD(spi, cs=Pin(40), dc=Pin(5), rst=Pin(41))
+        display = display_module.ILI9341(spi, cs=Pin(10), dc=Pin(46), bl=Pin(45), rst=None)
         touch = None
+        try:
+            from ft6336u import FT6336U
+            touch = FT6336U(i2c, rst_pin=18, int_pin=17, width=320, height=240, swap_xy=True, invert_x=False, invert_y=True)
+        except Exception as te:
+            print("Failed to initialize touch:", te)
     
     if sys.platform != 'rp2':
-        # Configure Audio Amp control pin to save power
+        # Configure Audio Amp control pin (GPIO 1, Active Low) to save power
-        amp_pin = Pin(46, Pin.OUT, value=0)
+        amp_pin = Pin(1, Pin.OUT, value=1)
     
     # 2. Initialize utility objects
     sensor = None
@@ -100,7 +128,7 @@ def main():
     buttons = None
     if sys.platform != 'rp2':
         buttons = BoardButtons()
-    ble_uart = BLEUART(name="ESP32-S3-RLCD")
+    ble_uart = BLEUART(name="ESP32-S3-TFT")
     
     # Sync system clock from RTC chip
     if rtc_chip:
@@ -126,7 +154,7 @@ def main():
             
             # 4b. Start MCP Server
             from mcp_server import MCPServer
-            mcp = MCPServer(display, led, battery, sensor, rtc_chip, ble_uart, vstream=vstream)
+            mcp = MCPServer(display, led, battery, sensor, rtc_chip, ble_uart, vstream=vstream, touch=touch)
             mcp.start(port=80)
         except Exception as ne:
             print("Failed to start network services:", ne)
@@ -169,6 +197,7 @@ def main():
     last_dashboard_update = 0
     dashboard_update_interval_ms = 5000
     last_led_update = 0
+    last_wifi_check = 0
     
     print("ESP32 MCP loop running...")
     
@@ -176,16 +205,227 @@ def main():
     while True:
         now = time.ticks_ms()
         
-        # Check touch interaction if available
+        # A. Check Wi-Fi status periodically (every 10 seconds) and auto-reconnect
-        if touch and touch.is_touched():
+        if has_network and time.ticks_diff(now, last_wifi_check) >= 10000:
-            pt = touch.read_touch()
+            last_wifi_check = now
-            if pt:
+            if not wlan.isconnected():
-                tx, ty = pt
+                print("Wi-Fi connection lost. Attempting reconnect...")
-                print(f"Touch detected at: ({tx}, {ty})")
+                last_action_str = "Wi-Fi Disconnected"
-                last_action_str = f"Touch: ({tx}, {ty})"
+                if ip_addr != "Disconnected":
-                mcp.override_active = False
+                    ip_addr = "Disconnected"
+                    force_dashboard_redraw = True
+                try:
+                    wlan.connect(wifi_config.WIFI_SSID, wifi_config.WIFI_PASS)
+                except Exception as e:
+                    print("Wi-Fi reconnect trigger failed:", e)
+            elif ip_addr == "Disconnected" or ip_addr == "Offline (USB)":
+                ip_addr = wlan.ifconfig()[0]
+                print(f"Wi-Fi Connected! IP Address: {ip_addr}")
+                last_action_str = f"Wi-Fi Connected: {ip_addr}"
                 force_dashboard_redraw = True
         
+        # Check touch interaction if available (Push-to-talk Voice Assistant)
+        if touch and touch.is_touched():
+            print("Touch detected! Starting Hermes Voice Assistant...")
+            
+            def draw_status_bar(text):
+                display.line(0, 215, 320, 215, 1)
+                display.fill_rect(0, 216, 320, 24, 0)
+                display.text(text, 10, 222, 1)
+                display.show()
+                
+            def clear_status_bar():
+                display.fill_rect(0, 215, 320, 25, 0)
+                display.show()
+                
+            # State 1: Wait for user to release their finger from the initial touch
+            draw_status_bar("Release finger...")
+            
+            release_start = time.ticks_ms()
+            while touch.is_touched():
+                if time.ticks_diff(time.ticks_ms(), release_start) > 2000:
+                    print("Release timeout occurred")
+                    break
+                time.sleep_ms(30)
+                
+            # State 2: Wait for tap-to-start recording
+            draw_status_bar("Ready. Tap screen to record...")
+            
+            tap_started = False
+            start_wait = time.ticks_ms()
+            while time.ticks_diff(time.ticks_ms(), start_wait) < 10000: # 10s timeout to start recording
+                if touch.is_touched():
+                    tap_started = True
+                    break
+                time.sleep_ms(30)
+                
+            if tap_started:
+                # User tapped the screen. Let's record!
+                draw_status_bar("Recording: 15s (Tap to stop)")
+                
+                # 1. Start MCLK PWM and configure ES8311 mic path
+                mclk_pin = Pin(4, Pin.OUT)
+                mclk_pwm = machine.PWM(mclk_pin)
+                mclk_pwm.freq(6144000)
+                mclk_pwm.duty_u16(32768)
+                
+                codec = ES8311(i2c)
+                if codec.init(sample_rate=16000):
+                    codec.set_volume(80) # Set a balanced volume to prevent speaker saturation
+                    try:
+                        codec._write(0x14, 0x1A) # Enable analog mic input & PGA
+                        codec._write(0x16, 0x01) # Enable +6dB gain boost for mic recording
+                        codec._write(0x17, 0xC8) # Set ADC digital volume
+                    except Exception as e:
+                        print("Failed to set mic gain:", e)
+                
+                # 2. Configure I2S RX for recording (Mono 16kHz)
+                i2s_rx = I2S(1, 
+                             sck=Pin(5), 
+                             ws=Pin(7), 
+                             sd=Pin(6), 
+                             mode=I2S.RX, 
+                             ibuf=8000, 
+                             rate=16000, 
+                             bits=16, 
+                             format=I2S.MONO)
+                
+                audio_chunks = []
+                total_data_bytes = 0
+                buffer = bytearray(1024)
+                
+                # Track release of the start-tap first
+                start_tap_released = False
+                rec_start_time = time.ticks_ms()
+                max_rec_duration_ms = 15000 # 15 seconds max duration
+                max_rec_bytes = 480000 # 15 seconds of mono 16kHz 16-bit PCM (32KB/s)
+                last_sec_left = -1
+                
+                try:
+                    while True:
+                        # Safety: check duration and byte size
+                        elapsed = time.ticks_diff(time.ticks_ms(), rec_start_time)
+                        if elapsed >= max_rec_duration_ms or total_data_bytes >= max_rec_bytes:
+                            print("Recording stopped: maximum duration/size reached")
+                            break
+                            
+                        # Update countdown
+                        sec_left = max(0, 15 - (elapsed // 1000))
+                        if sec_left != last_sec_left:
+                            last_sec_left = sec_left
+                            draw_status_bar(f"Recording: {sec_left}s (Tap to stop)")
+                            
+                        # Read I2S chunk
+                        bytes_read = i2s_rx.readinto(buffer)
+                        if bytes_read > 0:
+                            audio_chunks.append(bytes(buffer[:bytes_read]))
+                            total_data_bytes += bytes_read
+                            
+                        # Check touch state
+                        touched = touch.is_touched()
+                        if not start_tap_released:
+                            if not touched:
+                                start_tap_released = True
+                                print("Start-tap released. Listening for stop-tap...")
+                        else:
+                            if touched:
+                                print("Stop-tap detected. Stopping recording...")
+                                break
+                except Exception as e:
+                    print("Error recording:", e)
+                finally:
+                    i2s_rx.deinit()
+                    try:
+                        codec._write(0x16, 0x00) # Reset mic gain back to 0dB immediately to prevent playback saturation
+                    except:
+                        pass
+                    
+                if total_data_bytes >= 1000:
+                    draw_status_bar("Sending & processing...")
+                    
+                    # Prepend WAV header
+                    raw_pcm = b"".join(audio_chunks)
+                    wav_data = create_wav_header(len(raw_pcm)) + raw_pcm
+                    
+                    # Headers
+                    headers = {
+                        "Authorization": "Bearer mcT1YA1vOr9wXSiHpCYalweEGGZKX-PIfZv2drp8BSg",
+                        "Content-Type": "audio/wav",
+                        "X-Device-ID": "kitchen-button"
+                    }
+                    
+                    print(f"Posting WAV ({len(wav_data)} bytes) to Hermes API...")
+                    try:
+                        res = urequests.post("http://192.168.68.126:8642/api/esp32/voice", headers=headers, data=wav_data)
+                        print(f"HTTP Status: {res.status_code}")
+                        
+                        if res.status_code == 200:
+                            response_data = res.content
+                            print(f"Received response: {len(response_data)} bytes")
+                            
+                            if response_data[0:4] == b'RIFF' and response_data[8:12] == b'WAVE':
+                                idx = response_data.find(b'fmt ')
+                                if idx != -1:
+                                    fmt_chunk = response_data[idx:idx+24]
+                                    audio_format, channels, sample_rate, byte_rate, block_align, bits = struct.unpack('<HHIIHH', fmt_chunk[8:24])
+                                    print(f"Playing response WAV: {sample_rate}Hz, {channels}ch, {bits}bit")
+                                    
+                                    data_idx = response_data.find(b'data')
+                                    if data_idx != -1:
+                                        audio_start = data_idx + 8
+                                        
+                                        draw_status_bar("Speaking...")
+                                        
+                                        # Play response
+                                        amp_pin.value(0) # Enable Amp (Active Low)
+                                        i2s_format = I2S.MONO if channels == 1 else I2S.STEREO
+                                        i2s_tx = I2S(1,
+                                                     sck=Pin(5),
+                                                     ws=Pin(7),
+                                                     sd=Pin(8),
+                                                     mode=I2S.TX,
+                                                     ibuf=4096,
+                                                     rate=sample_rate,
+                                                     bits=bits,
+                                                     format=i2s_format)
+                                        try:
+                                            pos = audio_start
+                                            chunk_size = 2048
+                                            while pos < len(response_data):
+                                                chunk = response_data[pos:pos+chunk_size]
+                                                i2s_tx.write(chunk)
+                                                pos += chunk_size
+                                        finally:
+                                            time.sleep_ms(150)
+                                            amp_pin.value(1) # Disable Amp
+                                            i2s_tx.deinit()
+                            else:
+                                # MP3 warning message on display
+                                draw_status_bar("Error: Gateway returned MP3")
+                                time.sleep(3)
+                        else:
+                            draw_status_bar(f"HTTP Error: {res.status_code}")
+                            time.sleep(2)
+                    except Exception as he:
+                        print("Hermes HTTP post failed:", he)
+                        draw_status_bar("Connection Error")
+                        time.sleep(2)
+                
+                # Deinit MCLK PWM
+                mclk_pwm.deinit()
+                
+            # Debounce touch release at the very end of wizard
+            clear_status_bar()
+            release_end = time.ticks_ms()
+            while touch.is_touched():
+                if time.ticks_diff(time.ticks_ms(), release_end) > 2000:
+                    break
+                time.sleep_ms(30)
+            time.sleep_ms(200)
+            
+            last_action_str = "Voice query finished"
+            force_dashboard_redraw = True
+        
         # A. Handle non-blocking MCP client connection updates
         mcp.update()
         
@@ -214,31 +454,46 @@ def main():
                 
                 # Draw standard status dashboard layout
                 display.clear(0)
-                title_text = "RP2350-TFT MCP SERVER" if sys.platform == 'rp2' else "ESP32-S3-RLCD MCP SERVER"
+                title_text = "RP2350-TFT MCP SERVER" if sys.platform == 'rp2' else "Hosyond ESP32-S3 Server"
-                line_w = 470 if sys.platform == 'rp2' else 390
+                line_w = 470 if sys.platform == 'rp2' else 310
                 
-                display.text(title_text, 10, 10, 1)
+                if sys.platform == 'rp2':
-                display.line(10, 20, line_w, 20, 1)
+                    display.text(title_text, 10, 10, 1)
-                
+                    display.line(10, 20, line_w, 20, 1)
-                display.text_large("ENVIRONMENT", 15, 30, scale=2, c=1)
+                    display.text_large("ENVIRONMENT", 15, 30, scale=2, c=1)
-                display.text(f"Temp        : {t_str}", 25, 55, 1)
+                    display.text(f"Temp        : {t_str}", 25, 55, 1)
-                display.text(f"Humid       : {h_str}", 25, 70, 1)
+                    display.text(f"Humid       : {h_str}", 25, 70, 1)
-                
+                    display.line(10, 95, line_w, 95, 1)
-                display.line(10, 95, line_w, 95, 1)
+                    display.text_large("MCP NET CONNECTION", 15, 105, scale=2, c=1)
-                
+                    display.text(f"IP Address  : {ip_addr}", 25, 130, 1)
-                display.text_large("MCP NET CONNECTION", 15, 105, scale=2, c=1)
+                    display.text(f"Port / Path : 80 /api/mcp", 25, 145, 1)
-                display.text(f"IP Address  : {ip_addr}", 25, 130, 1)
+                    display.text(f"BLE Name    : ESP32-S3-RLCD", 25, 160, 1)
-                display.text(f"Port / Path : 80 /api/mcp", 25, 145, 1)
+                    display.line(10, 185, line_w, 185, 1)
-                display.text(f"BLE Name    : ESP32-S3-RLCD", 25, 160, 1)
+                    display.text_large("SYSTEM STATUS", 15, 195, scale=2, c=1)
-                
+                    display.text(f"Battery     : {bat_str}", 25, 220, 1)
-                display.line(10, 185, line_w, 185, 1)
+                    display.text(f"Time        : {time_str}", 25, 235, 1)
-                
+                    display.line(10, 255, line_w, 255, 1)
-                display.text_large("SYSTEM STATUS", 15, 195, scale=2, c=1)
+                    display.text(f"Status: {last_action_str}", 15, 265, 1)
-                display.text(f"Battery     : {bat_str}", 25, 220, 1)
+                else:
-                display.text(f"Time        : {time_str}", 25, 235, 1)
+                    display.text(title_text, 10, 8, 1)
-                
+                    display.line(10, 18, line_w, 18, 1)
-                display.line(10, 255, line_w, 255, 1)
+                    # Left Column (System & Environment)
-                display.text(f"Status: {last_action_str}", 15, 265, 1)
+                    display.text("SYSTEM & ENV", 10, 28, 1)
+                    display.line(10, 38, 150, 38, 1)
+                    display.text(f"Temp : {t_str}", 10, 46, 1)
+                    display.text(f"Hum  : {h_str}", 10, 58, 1)
+                    display.text(f"Bat  : {bat_str}", 10, 70, 1)
+                    display.text(f"Time : {time_str[11:19]}", 10, 82, 1)
+                    display.text(f"Date : {time_str[0:10]}", 10, 94, 1)
+                    # Right Column (Network)
+                    display.text("MCP NETWORK", 170, 28, 1)
+                    display.line(170, 38, line_w, 38, 1)
+                    display.text(f"IP : {ip_addr}", 170, 46, 1)
+                    display.text("Port: 80/api/mcp", 170, 58, 1)
+                    display.text("BLE : S3-TFT", 170, 70, 1)
+                    # Bottom Status
+                    display.line(10, 115, line_w, 115, 1)
+                    display.text(f"Status: {last_action_str}", 10, 125, 1)
                 display.show()
                 
         # D. Update NeoPixel animation smoothly (runs every 50ms)

mcp_server.py

@@ -5,7 +5,7 @@ import time
 class MCPServer:
     """A lightweight JSON-RPC HTTP server implementing Model Context Protocol (MCP) endpoints."""
     
-    def __init__(self, display, led, battery, sensor, rtc, ble, vstream=None):
+    def __init__(self, display, led, battery, sensor, rtc, ble, vstream=None, touch=None):
         self.display = display
         self.led = led
         self.battery = battery
@@ -13,6 +13,7 @@ class MCPServer:
         self.rtc = rtc
         self.ble = ble
         self.vstream = vstream
+        self.touch = touch
         
         self.sock = None
         self.active_led_mode = "off" # static, breath, rainbow, off
@@ -20,6 +21,7 @@ class MCPServer:
 
     def start(self, port=80):
         """Starts the TCP server non-blockingly."""
+        self.port = port
         self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
         self.sock.bind(('', port))
@@ -39,6 +41,27 @@ class MCPServer:
             print("Failed to bind UDP Discovery socket: {}".format(ue))
             self.udp_sock = None
 
+    def restart(self):
+        """Re-initializes the listening sockets after a fatal error."""
+        print("Restarting MCP Server sockets...")
+        try:
+            if self.sock:
+                self.sock.close()
+        except:
+            pass
+        self.sock = None
+        try:
+            if hasattr(self, 'udp_sock') and self.udp_sock:
+                self.udp_sock.close()
+        except:
+            pass
+        self.udp_sock = None
+        
+        # Add a short delay to allow socket reuse to settle
+        time.sleep_ms(100)
+        port = getattr(self, 'port', 80)
+        self.start(port)
+
     def update(self):
         """Check for incoming HTTP requests and handle them non-blockingly."""
         # 1. Handle UDP Discovery queries
@@ -56,8 +79,18 @@ class MCPServer:
             
         try:
             client, addr = self.sock.accept()
-        except OSError:
+        except OSError as e:
-            # No incoming connection
+            import errno
+            err = getattr(e, 'errno', None)
+            if err is None and e.args:
+                err = e.args[0]
+            ewouldblock = getattr(errno, 'EWOULDBLOCK', errno.EAGAIN)
+            if err in (errno.EAGAIN, ewouldblock) or err is None:
+                # No incoming connection
+                return
+            # Fatal socket error
+            print(f"Fatal socket error in MCP Server accept(): {e}. Re-initializing...")
+            self.restart()
             return
             
         try:
@@ -100,14 +133,21 @@ class MCPServer:
                 rpc_req = json.loads(body)
                 rpc_resp = self._handle_rpc(rpc_req)
                 
-                # Send HTTP Response
                 resp_body = json.dumps(rpc_resp)
                 resp = "HTTP/1.1 200 OK\r\n"
                 resp += "Content-Type: application/json\r\n"
                 resp += f"Content-Length: {len(resp_body)}\r\n"
                 resp += "Connection: close\r\n\r\n"
                 resp += resp_body
-                client.send(resp.encode('utf-8'))
+                
+                data_to_send = resp.encode('utf-8')
+                total_sent = 0
+                while total_sent < len(data_to_send):
+                    sent = client.write(data_to_send[total_sent:])
+                    if sent is None or sent == 0:
+                        time.sleep_ms(10)
+                        continue
+                    total_sent += sent
             else:
                 # Return 404
                 resp = "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
@@ -171,6 +211,11 @@ class MCPServer:
                             "description": "Read the current battery voltage and estimated capacity percentage.",
                             "inputSchema": {"type": "object", "properties": {}}
                         },
+                        {
+                            "name": "get_touch",
+                            "description": "Read the current touch state and coordinates from the capacitive touchscreen.",
+                            "inputSchema": {"type": "object", "properties": {}}
+                        },
                         {
                             "name": "get_sensors",
                             "description": "Read onboard SHTC3 temperature and relative humidity.",
@@ -404,6 +449,17 @@ class MCPServer:
             t, h = self.sensor.read_sensor()
             return json.dumps({"temperature_c": t, "humidity_pct": h})
             
+        elif name == "get_touch":
+            if self.touch is None:
+                return json.dumps({"error": "Touchscreen not configured on this device."})
+            is_t = self.touch.is_touched()
+            x, y = None, None
+            if is_t:
+                pt = self.touch.read_touch()
+                if pt:
+                    x, y = pt
+            return json.dumps({"is_touched": is_t, "x": x, "y": y})
+            
         elif name == "scan_ble":
             dur = int(args.get("duration_ms", 3000))
             # Scan returns dictionary: {mac: {rssi: rssi, name: name}}

rgb_led_util.py

@@ -9,7 +9,7 @@ class BoardLED:
     def __init__(self, pin_num=None):
         import sys
         if pin_num is None:
-            pin_num = 14 if sys.platform == 'rp2' else 38
+            pin_num = 14 if sys.platform == 'rp2' else 42
         self.np = neopixel.NeoPixel(Pin(pin_num), 1)
         self.base_color = (0, 0, 0)
         self.off()

scratch/recover.py

@@ -0,0 +1,75 @@
+import json
+import urllib.request
+import time
+import sys
+
+ESP32_IP = "192.168.68.123"
+URL = f"http://{ESP32_IP}/api/mcp"
+
+def call_mcp_tool(tool_name, arguments):
+    payload = {
+        "jsonrpc": "2.0",
+        "id": 1,
+        "method": "tools/call",
+        "params": {
+            "name": tool_name,
+            "arguments": arguments
+        }
+    }
+    
+    req_data = json.dumps(payload).encode('utf-8')
+    req = urllib.request.Request(
+        URL,
+        data=req_data,
+        headers={"Content-Type": "application/json"},
+        method="POST"
+    )
+    
+    # Try multiple times for each call since the connection might reset
+    for attempt in range(1, 100):
+        try:
+            with urllib.request.urlopen(req, timeout=1.0) as response:
+                resp_body = response.read().decode('utf-8')
+                resp_json = json.loads(resp_body)
+                if "error" in resp_json:
+                    return None
+                return resp_json.get("result", {}).get("content", [{}])[0].get("text", "")
+        except Exception as e:
+            time.sleep(0.01)
+    return None
+
+def upload_file(local_path, remote_path):
+    print(f"Reading local file {local_path}...")
+    with open(local_path, 'r') as f:
+        content = f.read()
+        
+    print(f"Uploading to ESP32: {remote_path}...")
+    for attempt in range(1, 20):
+        res = call_mcp_tool("write_file", {"path": remote_path, "content": content})
+        if res:
+            print(f"Success uploading {remote_path} on attempt {attempt}!")
+            return True
+        print(f"Retry {attempt} uploading {remote_path}...")
+        time.sleep(0.1)
+    return False
+
+def main():
+    print("=== ESP32 OTA Recovery Tool ===")
+    
+    # Upload mcp_server.py first
+    if not upload_file("mcp_server.py", "mcp_server.py"):
+        print("Failed to upload mcp_server.py. Exiting.")
+        sys.exit(1)
+        
+    # Upload video_stream.py
+    if not upload_file("video_stream.py", "video_stream.py"):
+        print("Failed to upload video_stream.py. Exiting.")
+        sys.exit(1)
+        
+    print("Rebooting the board...")
+    reboot_code = "import machine\nmachine.reset()"
+    call_mcp_tool("execute_python", {"code": reboot_code})
+    print("Reboot command sent! The board should recover now.")
+
+if __name__ == "__main__":
+    main()

test_stream.py

@@ -80,6 +80,28 @@ def generate_animation_frame(frame_idx, mode_name):
     
     return img
 
+def connect_socket(esp32_ip, port, protocol):
+    """Establishes connection to the ESP32 stream server, retrying on failure for TCP."""
+    while True:
+        try:
+            if protocol == "tcp":
+                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+                sock.connect((esp32_ip, port))
+                print(f"Connected to stream server at {esp32_ip}:{port}")
+                return sock
+            else: # udp
+                sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+                if esp32_ip.endswith(".255") or esp32_ip == "255.255.255.255":
+                    sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
+                return sock
+        except (socket.error, Exception) as e:
+            if protocol == "tcp":
+                print(f"Failed to connect to stream server: {e}. Retrying in 2 seconds...")
+                time.sleep(2.0)
+            else:
+                # UDP doesn't establish connection, so any error here is fatal
+                raise e
+
 def stream_camera(esp32_ip, port, protocol):
     """Streams camera capture frames using OpenCV (requires opencv-python)."""
     try:
@@ -94,16 +116,8 @@ def stream_camera(esp32_ip, port, protocol):
         print("Error: Could not open camera.")
         sys.exit(1)
         
-    sock = None
+    sock = connect_socket(esp32_ip, port, protocol)
-    if protocol == "tcp":
+        
-        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        sock.connect((esp32_ip, port))
-    else: # udp
-        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        # Enable broadcast if destination is broadcast
-        if esp32_ip.endswith(".255") or esp32_ip == "255.255.255.255":
-            sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
-            
     print(f"Streaming webcam to {esp32_ip}:{port} via {protocol.upper()}...")
     frame_count = 0
     start_time = time.time()
@@ -128,11 +142,19 @@ def stream_camera(esp32_ip, port, protocol):
             # Map pixels to hardware buffer
             raw_bytes = map_to_rlcd_hw_buffer(bg)
             
-            # Transmit
+            # Transmit with reconnect logic
-            if protocol == "tcp":
+            try:
-                sock.sendall(raw_bytes)
+                if protocol == "tcp":
-            else:
+                    sock.sendall(raw_bytes)
-                send_udp_frame_chunked(sock, esp32_ip, port, frame_count, raw_bytes)
+                else:
+                    send_udp_frame_chunked(sock, esp32_ip, port, frame_count, raw_bytes)
+            except (socket.error, Exception) as se:
+                print(f"Transmission failed: {se}. Reconnecting...")
+                try:
+                    sock.close()
+                except:
+                    pass
+                sock = connect_socket(esp32_ip, port, protocol)
                 
             frame_count += 1
             elapsed = time.time() - start_time
@@ -151,14 +173,7 @@ def stream_camera(esp32_ip, port, protocol):
 
 def stream_animation(esp32_ip, port, protocol):
     """Streams a generated Pillow vector animation."""
-    sock = None
+    sock = connect_socket(esp32_ip, port, protocol)
-    if protocol == "tcp":
-        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        sock.connect((esp32_ip, port))
-    else: # udp
-        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        if esp32_ip.endswith(".255") or esp32_ip == "255.255.255.255":
-            sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
             
     print(f"Streaming vector animation to {esp32_ip}:{port} via {protocol.upper()}...")
     frame_idx = 0
@@ -172,11 +187,19 @@ def stream_animation(esp32_ip, port, protocol):
             # Map pixels to hardware buffer
             raw_bytes = map_to_rlcd_hw_buffer(pil_img)
             
-            # Transmit
+            # Transmit with reconnect logic
-            if protocol == "tcp":
+            try:
-                sock.sendall(raw_bytes)
+                if protocol == "tcp":
-            else:
+                    sock.sendall(raw_bytes)
-                send_udp_frame_chunked(sock, esp32_ip, port, frame_idx, raw_bytes)
+                else:
+                    send_udp_frame_chunked(sock, esp32_ip, port, frame_idx, raw_bytes)
+            except (socket.error, Exception) as se:
+                print(f"Transmission failed: {se}. Reconnecting...")
+                try:
+                    sock.close()
+                except:
+                    pass
+                sock = connect_socket(esp32_ip, port, protocol)
                 
             frame_idx += 1
             

upload_files.py

@@ -3,7 +3,7 @@ import json
 import urllib.request
 import sys
 
-ESP32_IP = "192.168.68.123"
+ESP32_IP = "192.168.68.124"
 URL = f"http://{ESP32_IP}/api/mcp"
 
 def call_mcp_tool(tool_name, arguments):

verify_streaming.py

@@ -9,7 +9,7 @@ from PIL import Image
 
 ESP32_IP = "192.168.68.123"
 URL = f"http://{ESP32_IP}/api/mcp"
-ARTIFACTS_DIR = "/Users/adolforeyna/.gemini/antigravity/brain/8b421e8b-10a8-4d06-a7d0-27bd82b42804"
+ARTIFACTS_DIR = "/Users/adolforeyna/.gemini/antigravity/brain/da0e7ff3-d127-4865-bfea-f8ae9198bd40"
 
 def capture_screenshot(output_filename):
     print(f"Requesting screenshot from ESP32 ({ESP32_IP})...")

video_stream.py

@@ -136,6 +136,46 @@ class VideoStreamServer:
         except Exception as e:
             print(f"Failed to start UDP stream server: {e}")
 
+    def restart_tcp_server(self):
+        """Re-initializes the TCP stream server socket after a fatal error."""
+        print("Restarting TCP Stream Server...")
+        self.close_tcp_client()
+        try:
+            if self.tcp_server:
+                self.tcp_server.close()
+        except:
+            pass
+        self.tcp_server = None
+        time.sleep_ms(100)
+        try:
+            self.tcp_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            self.tcp_server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            self.tcp_server.bind(('', self.tcp_port))
+            self.tcp_server.listen(1)
+            self.tcp_server.setblocking(False)
+            print(f"TCP Stream Server restarted on port {self.tcp_port}")
+        except Exception as e:
+            print(f"Failed to restart TCP stream server: {e}")
+
+    def restart_udp_sock(self):
+        """Re-initializes the UDP stream server socket after a fatal error."""
+        print("Restarting UDP Stream Server...")
+        try:
+            if self.udp_sock:
+                self.udp_sock.close()
+        except:
+            pass
+        self.udp_sock = None
+        time.sleep_ms(100)
+        try:
+            self.udp_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+            self.udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            self.udp_sock.bind(('', self.udp_port))
+            self.udp_sock.setblocking(False)
+            print(f"UDP Stream Server restarted on port {self.udp_port}")
+        except Exception as e:
+            print(f"Failed to restart UDP stream server: {e}")
+
     def update(self):
         """Updates connection states and checks for incoming video data.
         Call this periodically in the main execution loop.
@@ -195,7 +235,16 @@ class VideoStreamServer:
                             self._draw_frame()
                             self.chunks_received = 0  # Reset for next frame
                 except OSError as e:
-                    break # EWOULDBLOCK (no data available)
+                    import errno
+                    err = getattr(e, 'errno', None)
+                    if err is None and e.args:
+                        err = e.args[0]
+                    ewouldblock = getattr(errno, 'EWOULDBLOCK', errno.EAGAIN)
+                    if err in (errno.EAGAIN, ewouldblock) or err is None:
+                        break # EAGAIN/EWOULDBLOCK (no data available)
+                    print(f"Fatal UDP socket error in update(): {e}. Re-initializing UDP socket...")
+                    self.restart_udp_sock()
+                    break
 
         # 3. Handle TCP stream
         if self.tcp_server:
@@ -208,8 +257,15 @@ class VideoStreamServer:
                     self.active = True
                     self.last_packet_time = now
                     print(f"TCP Stream client connected from: {addr}")
-                except OSError:
+                except OSError as e:
-                    pass
+                    import errno
+                    err = getattr(e, 'errno', None)
+                    if err is None and e.args:
+                        err = e.args[0]
+                    ewouldblock = getattr(errno, 'EWOULDBLOCK', errno.EAGAIN)
+                    if err not in (errno.EAGAIN, ewouldblock) and err is not None:
+                        print(f"Fatal TCP stream server socket error in accept(): {e}. Re-initializing...")
+                        self.restart_tcp_server()
             
             # If client is connected, read data
             if self.tcp_client is not None: