From 2813c11104309b0f5a45e68ba11b92d7e3b6eb2a Mon Sep 17 00:00:00 2001 From: Adolfo Reyna Date: Wed, 17 Jun 2026 22:17:59 -0400 Subject: [PATCH] Save current changes before reorganizing drivers and cleaning workspace --- .gitignore | 4 + audio_util.py | 75 +- battery_util.py | 10 +- boot.py | 22 +- demo_hermes_voice.py | 248 +++ demo_touch_mic.py | 144 ++ desktop_client/README.md | 79 + desktop_client/client.py | 1480 +++++++++++++++++ .../com.adolforeyna.companionclient.plist | 23 + desktop_client/setup_service.sh | 34 + desktop_client/uninstall_service.sh | 17 + ft6336u.py | 14 +- ili9341.py | 261 +++ main.py | 337 +++- mcp_server.py | 66 +- rgb_led_util.py | 2 +- scratch/recover.py | 75 + test_stream.py | 79 +- upload_files.py | 2 +- verify_streaming.py | 2 +- video_stream.py | 62 +- 21 files changed, 2904 insertions(+), 132 deletions(-) create mode 100644 demo_hermes_voice.py create mode 100644 demo_touch_mic.py create mode 100644 desktop_client/README.md create mode 100644 desktop_client/client.py create mode 100644 desktop_client/com.adolforeyna.companionclient.plist create mode 100755 desktop_client/setup_service.sh create mode 100755 desktop_client/uninstall_service.sh create mode 100644 ili9341.py create mode 100644 scratch/recover.py diff --git a/.gitignore b/.gitignore index f2aa58b..82cf00a 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ __pycache__/ # OS generated files .DS_Store + +# Log files +desktop_client/*.log + diff --git a/audio_util.py b/audio_util.py index 436c975..9615e47 100644 --- a/audio_util.py +++ b/audio_util.py @@ -79,24 +79,30 @@ def record_audio(duration_seconds=5, filename='recording.pcm'): duration_seconds (int): How long to record in seconds. filename (str): Name of output raw PCM file on the device. """ - # 1. Start I2C Control Bus (SDA=13, SCL=14) - i2c = I2C(0, sda=Pin(13), scl=Pin(14)) + # 1. Start I2C Control Bus (SDA=16, SCL=15) + i2c = I2C(0, sda=Pin(16), scl=Pin(15)) + + # 1a. Setup Master Clock (MCLK) on GPIO 4 using PWM (needed for ES8311 ADC) + mclk_pin = Pin(4, Pin.OUT) + mclk_pwm = machine.PWM(mclk_pin) + mclk_pwm.freq(6144000) + mclk_pwm.duty_u16(32768) # 2. Configure I2S Receiver - # Pins: sck=BCLK (GPIO 9), ws=WS/LRCK (GPIO 45), sd=DIN (GPIO 10) + # Pins: sck=BCLK (GPIO 5), ws=WS/LRCK (GPIO 7), sd=DIN (GPIO 6) i2s = I2S(1, - sck=Pin(9), - ws=Pin(45), - sd=Pin(10), + sck=Pin(5), + ws=Pin(7), + sd=Pin(6), mode=I2S.RX, ibuf=16000, rate=16000, bits=16, format=I2S.STEREO) - # 3. Wake up and configure the ES7210 microphone chip - mic_adc = ES7210(i2c) - if not mic_adc.init(sample_rate=16000, bit_width=16): + # 3. Wake up and configure the ES8311 codec chip + mic_adc = ES8311(i2c) + if not mic_adc.init(sample_rate=16000): i2s.deinit() return False @@ -125,7 +131,8 @@ def record_audio(duration_seconds=5, filename='recording.pcm'): finally: # Always release the I2S peripheral resources i2s.deinit() - print("I2S receiver deinitialized.") + mclk_pwm.deinit() + print("I2S receiver and MCLK deinitialized.") class ES8311: @@ -155,11 +162,11 @@ class ES8311: self._write(0x00, 0x00) time.sleep_ms(10) - # Clock Configuration (16kHz sample rate, MCLK=12.288MHz) + # Clock Configuration (16kHz sample rate, MCLK=6.144MHz) self._write(0x01, 0x3F) # Enable all clocks, use MCLK pin self._write(0x02, 0x48) # pre_div=3, pre_mult=1 self._write(0x03, 0x10) # fs_mode=0, adc_osr=16 - self._write(0x04, 0x20) # dac_osr=32 + self._write(0x04, 0x10) # dac_osr=16 self._write(0x05, 0x00) # adc_div=1, dac_div=1 self._write(0x06, 0x03) # bclk_div=4 (4-1=3) self._write(0x07, 0x00) # lrck_h=0 @@ -219,14 +226,14 @@ def play_tone(frequency=440, duration_ms=1000, volume=50): print(f"Playing tone: {frequency}Hz for {duration_ms}ms (vol={volume})...") - # 1. Setup Master Clock (MCLK) on GPIO 16 using PWM - mclk_pin = Pin(16, Pin.OUT) + # 1. Setup Master Clock (MCLK) on GPIO 4 using PWM + mclk_pin = Pin(4, Pin.OUT) mclk_pwm = machine.PWM(mclk_pin) - mclk_pwm.freq(12288000) + mclk_pwm.freq(6144000) mclk_pwm.duty_u16(32768) - # 2. Start I2C Control Bus (SDA=13, SCL=14) - i2c = I2C(0, sda=Pin(13), scl=Pin(14)) + # 2. Start I2C Control Bus (SDA=16, SCL=15) + i2c = I2C(0, sda=Pin(16), scl=Pin(15)) # 3. Initialize the ES8311 DAC dac = ES8311(i2c) @@ -237,10 +244,10 @@ def play_tone(frequency=440, duration_ms=1000, volume=50): dac.set_volume(volume) # 4. Configure I2S TX - # Pins: sck=BCLK (GPIO 9), ws=WS/LRCK (GPIO 45), sd=DOUT (GPIO 8) + # Pins: sck=BCLK (GPIO 5), ws=WS/LRCK (GPIO 7), sd=DOUT (GPIO 8) i2s = I2S(1, - sck=Pin(9), - ws=Pin(45), + sck=Pin(5), + ws=Pin(7), sd=Pin(8), mode=I2S.TX, ibuf=8000, @@ -248,8 +255,8 @@ def play_tone(frequency=440, duration_ms=1000, volume=50): bits=16, format=I2S.STEREO) - # 5. Enable Speaker Amplifier (GPIO 46) - amp_pin = Pin(46, Pin.OUT, value=1) + # 5. Enable Speaker Amplifier (GPIO 1, Active Low) + amp_pin = Pin(1, Pin.OUT, value=0) # 6. Generate sine wave cycle # Approximate frequency to make integer number of samples per cycle @@ -277,7 +284,7 @@ def play_tone(frequency=440, duration_ms=1000, volume=50): # 7. Clean up time.sleep_ms(100) # Let the remaining buffer play out - amp_pin.value(0) + amp_pin.value(1) # Disable amp i2s.deinit() mclk_pwm.deinit() print("Tone playback complete.") @@ -337,14 +344,14 @@ def play_wav(filename, volume=50): print(f"WAV Info: {sample_rate}Hz, {bits} bits, {'Mono' if channels == 1 else 'Stereo'}") - # 2. Setup Master Clock (MCLK) on GPIO 16 using PWM - mclk_pin = Pin(16, Pin.OUT) + # 2. Setup Master Clock (MCLK) on GPIO 4 using PWM + mclk_pin = Pin(4, Pin.OUT) mclk_pwm = machine.PWM(mclk_pin) - mclk_pwm.freq(12288000) + mclk_pwm.freq(6144000) mclk_pwm.duty_u16(32768) - # 3. Start I2C Control Bus (SDA=13, SCL=14) - i2c = I2C(0, sda=Pin(13), scl=Pin(14)) + # 3. Start I2C Control Bus (SDA=16, SCL=15) + i2c = I2C(0, sda=Pin(16), scl=Pin(15)) # 4. Initialize the ES8311 DAC dac = ES8311(i2c) @@ -356,11 +363,11 @@ def play_wav(filename, volume=50): dac.set_volume(volume) # 5. Configure I2S TX - # Pins: sck=BCLK (GPIO 9), ws=WS/LRCK (GPIO 45), sd=DOUT (GPIO 8) + # Pins: sck=BCLK (GPIO 5), ws=WS/LRCK (GPIO 7), sd=DOUT (GPIO 8) i2s_format = I2S.MONO if channels == 1 else I2S.STEREO i2s = I2S(1, - sck=Pin(9), - ws=Pin(45), + sck=Pin(5), + ws=Pin(7), sd=Pin(8), mode=I2S.TX, ibuf=4096, @@ -368,8 +375,8 @@ def play_wav(filename, volume=50): bits=bits, format=i2s_format) - # 6. Enable Speaker Amplifier (GPIO 46) - amp_pin = Pin(46, Pin.OUT, value=1) + # 6. Enable Speaker Amplifier (GPIO 1, Active Low) + amp_pin = Pin(1, Pin.OUT, value=0) # 7. Read and stream chunks to I2S buf = bytearray(2048) @@ -381,7 +388,7 @@ def play_wav(filename, volume=50): # 8. Clean up time.sleep_ms(100) # Let the remaining buffer play out - amp_pin.value(0) + amp_pin.value(1) # Disable amp i2s.deinit() mclk_pwm.deinit() f.close() diff --git a/battery_util.py b/battery_util.py index 79e1f4e..2d5bb2d 100644 --- a/battery_util.py +++ b/battery_util.py @@ -3,7 +3,7 @@ from machine import ADC, Pin class BatteryMonitor: """Utility class for monitoring battery voltage and capacity on ESP32-S3-RLCD-4.2.""" - def __init__(self, pin_num=4): + def __init__(self, pin_num=9): import sys self.adc = ADC(Pin(pin_num)) if sys.platform == 'esp32': @@ -18,16 +18,16 @@ class BatteryMonitor: try: # Try calibrated reading in microvolts first uv = self.adc.read_uv() - # 3x voltage divider onboard scales 3.0V-4.2V battery to 1.0V-1.4V - voltage = (uv / 1_000_000.0) * 3.0 + # 2x voltage divider onboard scales 3.0V-4.2V battery to 1.5V-2.1V + voltage = (uv / 1_000_000.0) * 2.0 return round(voltage, 3) except AttributeError: # Fallback if read_uv() is not supported on older MicroPython builds try: # Read 12-bit value (0-4095) raw = self.adc.read() - # 3.3V reference at 11dB attenuation, 3x divider - voltage = (raw / 4095.0) * 3.3 * 3.0 + # 3.3V reference at 11dB attenuation, 2x divider + voltage = (raw / 4095.0) * 3.3 * 2.0 return round(voltage, 3) except Exception as e: print(f"Error reading battery raw ADC: {e}") diff --git a/boot.py b/boot.py index 8b5bbfc..b325287 100644 --- a/boot.py +++ b/boot.py @@ -10,7 +10,7 @@ except ImportError: if sys.platform == 'rp2': import st7796 as display_module else: - import rlcd as display_module + import ili9341 as display_module from machine import Pin, SPI import wifi_config @@ -24,13 +24,13 @@ def connect_wifi(): # Initialize display to show connection progress display = None try: - # ESP32-S3 configuration - spi = SPI(1, baudrate=20000000, polarity=0, phase=0, sck=Pin(11), mosi=Pin(12)) - display = display_module.RLCD(spi, cs=Pin(40), dc=Pin(5), rst=Pin(41)) + # ESP32-S3 configuration for Hosyond Board + spi = SPI(1, baudrate=40000000, polarity=0, phase=0, sck=Pin(12), mosi=Pin(11), miso=Pin(13)) + display = display_module.ILI9341(spi, cs=Pin(10), dc=Pin(46), bl=Pin(45), rst=None) display.clear(0) - display.text("ESP32-S3-RLCD-4.2", 10, 10, 1) - display.line(10, 20, 390, 20, 1) + display.text("Hosyond ESP32-S3", 10, 10, 1) + display.line(10, 20, 310, 20, 1) display.text("Connecting to Wi-Fi...", 10, 35, 1) display.text(f"SSID: {wifi_config.WIFI_SSID}", 10, 50, 1) display.show() @@ -71,8 +71,8 @@ def connect_wifi(): print(f"Wi-Fi Connected! IP Address: {ip}") if display: display.clear(0) - display.text("ESP32-S3-RLCD-4.2", 10, 10, 1) - display.line(10, 20, 390, 20, 1) + display.text("Hosyond ESP32-S3", 10, 10, 1) + display.line(10, 20, 310, 20, 1) display.text("Wi-Fi Status: Connected!", 10, 35, 1) display.text(f"SSID: {wifi_config.WIFI_SSID}", 10, 50, 1) display.text(f"IP: {ip}", 10, 65, 1) @@ -83,9 +83,9 @@ def connect_wifi(): print("Wi-Fi Connection Failed.") if display: display.clear(0) - display.text("ESP32-S3-RLCD-4.2", 10, 10, 1) - display.line(10, 20, 390, 20, 1) - display.text("Wi-Fi Status: Connection Failed!", 10, 35, 1) + display.text("Hosyond ESP32-S3", 10, 10, 1) + display.line(10, 20, 310, 20, 1) + display.text("Wi-Fi Status: Failed!", 10, 35, 1) display.text("Check credentials in:", 10, 55, 1) display.text("wifi_config.py", 20, 70, 1) display.text("Starting offline...", 10, 95, 1) diff --git a/demo_hermes_voice.py b/demo_hermes_voice.py new file mode 100644 index 0000000..31d44d8 --- /dev/null +++ b/demo_hermes_voice.py @@ -0,0 +1,248 @@ +import time +import struct +import machine +from machine import Pin, SPI, I2C, I2S +import urequests +import ili9341 +from ft6336u import FT6336U +from audio_util import ES8311 + +# --- CONFIGURATION --- +HERMES_API_URL = "http://192.168.68.126:8642/api/esp32/voice" +DEVICE_ID = "kitchen-button" + +# Placeholder for API Key. Since the key is stored on the Hermes server, +# please copy-paste the token string here. +HERMES_API_KEY = "mcT1YA1vOr9wXSiHpCYalweEGGZKX-PIfZv2drp8BSg" + +def create_wav_header(data_size): + # Generates a 44-byte WAV header for 16kHz, 16-bit mono PCM + riff = b'RIFF' + file_size = data_size + 36 + wave = b'WAVE' + fmt = b'fmt ' + chunk_size = 16 + audio_format = 1 # PCM + channels = 1 # Mono + sample_rate = 16000 + bits_per_sample = 16 + byte_rate = sample_rate * channels * (bits_per_sample // 8) + block_align = channels * (bits_per_sample // 8) + data_label = b'data' + + return struct.pack('<4sI4s4sIHHIIHH4sI', + riff, file_size, wave, fmt, chunk_size, + audio_format, channels, sample_rate, byte_rate, + block_align, bits_per_sample, data_label, data_size) + +def main(): + print("--- Starting Hermes Voice Assistant Demo ---") + + # 1. Initialize display + spi = SPI(1, baudrate=40000000, polarity=0, phase=0, sck=Pin(12), mosi=Pin(11), miso=Pin(13)) + display = ili9341.ILI9341(spi, cs=Pin(10), dc=Pin(46), bl=Pin(45), rst=None) + display.clear(0) + display.text("Hermes Assistant", 10, 10, 1) + display.show() + + # 2. Initialize touch + i2c = I2C(0, sda=Pin(16), scl=Pin(15)) + touch = FT6336U(i2c, rst_pin=18, int_pin=17, width=320, height=240, swap_xy=True, invert_x=False, invert_y=True) + + # 3. Configure audio codec ES8311 + # We need MCLK pin (GPIO 4) active at 6.144 MHz + mclk_pin = Pin(4, Pin.OUT) + mclk_pwm = machine.PWM(mclk_pin) + mclk_pwm.freq(6144000) + mclk_pwm.duty_u16(32768) + + codec = ES8311(i2c) + if not codec.init(sample_rate=16000): + print("ES8311 init failed!") + return + + codec.set_volume(90) + + # Configure microphone registers on ES8311 + try: + codec._write(0x14, 0x1A) # Enable analog mic input & set PGA gain + codec._write(0x16, 0x01) # Boost MIC digital gain to +6dB + codec._write(0x17, 0xC8) # Set ADC digital volume + print("Microphone registers initialized.") + except Exception as e: + print("Failed to write microphone registers:", e) + + amp_pin = Pin(1, Pin.OUT, value=1) # start with amp disabled (1 = disabled) + + buffer = bytearray(1024) + + while True: + display.clear(0) + display.text("Hermes Assistant", 10, 10, 1) + display.line(10, 22, 310, 22, 1) + display.text("Hold screen & ask", 50, 70, 1) + display.text("a question...", 50, 90, 1) + display.text("Status: Idle", 10, 220, 1) + display.show() + + # Wait for touch + while not touch.is_touched(): + time.sleep_ms(30) + + print("Touch detected! Starting recording to RAM...") + display.clear(0) + display.text("Hermes Assistant", 10, 10, 1) + display.line(10, 22, 310, 22, 1) + display.text("Listening...", 80, 80, 1) + display.fill_rect(130, 110, 30, 30, 1) + display.text("Status: Recording", 10, 220, 1) + display.show() + + # 4. Open I2S RX for recording (Mono 16kHz for Hermes STT) + i2s_rx = I2S(1, + sck=Pin(5), + ws=Pin(7), + sd=Pin(6), + mode=I2S.RX, + ibuf=8000, + rate=16000, + bits=16, + format=I2S.MONO) + + audio_chunks = [] + total_data_bytes = 0 + + try: + # Record loop - as long as touch is held + while touch.is_touched(): + bytes_read = i2s_rx.readinto(buffer) + if bytes_read > 0: + audio_chunks.append(bytes(buffer[:bytes_read])) + total_data_bytes += bytes_read + + print(f"Touch released! Recorded {total_data_bytes} bytes.") + + except Exception as e: + print("Error recording:", e) + finally: + i2s_rx.deinit() + + if total_data_bytes < 1000: + print("Recording too short, ignoring.") + continue + + # 5. Connect and POST to Hermes voice endpoint + display.clear(0) + display.text("Hermes Assistant", 10, 10, 1) + display.line(10, 22, 310, 22, 1) + display.text("Sending audio...", 50, 80, 1) + display.text("Waiting for agent...", 50, 100, 1) + display.text("Status: Processing", 10, 220, 1) + display.show() + + # Join chunks and prepend WAV header + raw_pcm = b"".join(audio_chunks) + wav_data = create_wav_header(len(raw_pcm)) + raw_pcm + + # Headers + headers = { + "Authorization": f"Bearer {HERMES_API_KEY}", + "Content-Type": "audio/wav", + "X-Device-ID": DEVICE_ID + } + + print(f"Posting raw WAV ({len(wav_data)} bytes) to {HERMES_API_URL}...") + try: + res = urequests.post(HERMES_API_URL, headers=headers, data=wav_data) + print(f"HTTP Status: {res.status_code}") + + if res.status_code == 200: + response_data = res.content + print(f"Received {len(response_data)} bytes of audio response.") + + # Check response format + if response_data[0:4] == b'RIFF' and response_data[8:12] == b'WAVE': + # Parse WAV header + idx = response_data.find(b'fmt ') + if idx != -1: + fmt_chunk = response_data[idx:idx+24] + audio_format, channels, sample_rate, byte_rate, block_align, bits = struct.unpack(' 2 and response_data[0] == 0xFF and (response_data[1] & 0xE0) == 0xE0): + print("Error: Server returned MP3 format. ESP32 requires WAV.") + display.clear(0) + display.text("Error: Got MP3", 10, 10, 1) + display.line(10, 22, 310, 22, 1) + display.text("Hermes returned MP3.", 30, 80, 1) + display.text("Configure server to", 30, 100, 1) + display.text("convert MP3 to WAV.", 30, 120, 1) + display.show() + time.sleep(4) + else: + print(f"Error: Unknown response format. Starts with: {response_data[0:16]}") + else: + print(f"Request failed with status {res.status_code}: {res.text}") + display.clear(0) + display.text("HTTP Error", 10, 10, 1) + display.line(10, 22, 310, 22, 1) + display.text(f"Status: {res.status_code}", 50, 80, 1) + display.show() + time.sleep(3) + + except Exception as e: + print("Failed to contact Hermes server:", e) + display.clear(0) + display.text("Server Error", 10, 10, 1) + display.line(10, 22, 310, 22, 1) + display.text("Could not connect", 50, 80, 1) + display.text("to API gateway.", 50, 100, 1) + display.show() + time.sleep(3) + + # Debounce touch release + while touch.is_touched(): + time.sleep_ms(30) + time.sleep_ms(300) + +if __name__ == "__main__": + main() diff --git a/demo_touch_mic.py b/demo_touch_mic.py new file mode 100644 index 0000000..ce66fd8 --- /dev/null +++ b/demo_touch_mic.py @@ -0,0 +1,144 @@ +import time +import machine +from machine import Pin, SPI, I2C, I2S +import ili9341 +from ft6336u import FT6336U +from audio_util import ES8311 + +def main(): + print("--- Starting RAM-based Touch Mic Demo ---") + + # 1. Initialize display + spi = SPI(1, baudrate=40000000, polarity=0, phase=0, sck=Pin(12), mosi=Pin(11), miso=Pin(13)) + display = ili9341.ILI9341(spi, cs=Pin(10), dc=Pin(46), bl=Pin(45), rst=None) + display.clear(0) + display.text("RAM Touch Mic", 10, 10, 1) + display.show() + + # 2. Initialize touch + i2c = I2C(0, sda=Pin(16), scl=Pin(15)) + touch = FT6336U(i2c, rst_pin=18, int_pin=17, width=320, height=240, swap_xy=True, invert_x=False, invert_y=True) + + # 3. Configure audio codec ES8311 + # Generate MCLK PWM on GPIO 4 at 6.144 MHz + mclk_pin = Pin(4, Pin.OUT) + mclk_pwm = machine.PWM(mclk_pin) + mclk_pwm.freq(6144000) + mclk_pwm.duty_u16(32768) + + codec = ES8311(i2c) + if not codec.init(sample_rate=16000): + print("ES8311 init failed!") + return + + # Set speaker volume to 95% + codec.set_volume(95) + + # Configure microphone registers on ES8311 + try: + codec._write(0x14, 0x1A) # Enable analog mic input & set PGA gain + codec._write(0x16, 0x01) # Boost MIC digital gain to +6dB to prevent saturation + codec._write(0x17, 0xC8) # Set ADC digital volume + print("Microphone registers initialized with gain boost.") + except Exception as e: + print("Failed to write microphone registers:", e) + + amp_pin = Pin(1, Pin.OUT, value=1) # start with amp disabled (1 = disabled) + + # We allocate a buffer for I2S read + buffer = bytearray(2048) + + while True: + display.clear(0) + display.text("RAM Touch Mic", 10, 10, 1) + display.line(10, 22, 310, 22, 1) + display.text("Press & hold screen", 50, 80, 1) + display.text("to record voice...", 50, 100, 1) + display.show() + + # Wait for touch + while not touch.is_touched(): + time.sleep_ms(30) + + print("Touch detected! Starting recording to RAM...") + display.clear(0) + display.text("Recording...", 10, 10, 1) + display.line(10, 22, 310, 22, 1) + display.text("SPEAK NOW!", 80, 80, 1) + display.fill_rect(130, 110, 30, 30, 1) + display.show() + + # 4. Open I2S RX for recording + i2s_rx = I2S(1, + sck=Pin(5), + ws=Pin(7), + sd=Pin(6), + mode=I2S.RX, + ibuf=8000, + rate=16000, + bits=16, + format=I2S.STEREO) + + audio_chunks = [] + total_data_bytes = 0 + + try: + # Record loop - append chunks directly in RAM + while touch.is_touched(): + bytes_read = i2s_rx.readinto(buffer) + if bytes_read > 0: + audio_chunks.append(bytes(buffer[:bytes_read])) + total_data_bytes += bytes_read + + print(f"Touch released! Recorded {total_data_bytes} bytes in RAM.") + + except Exception as e: + print("Error recording:", e) + finally: + i2s_rx.deinit() + + # 5. Playback recorded audio + display.clear(0) + display.text("Playing back...", 10, 10, 1) + display.line(10, 22, 310, 22, 1) + display.text("Listening to RAM...", 50, 80, 1) + display.show() + + print("Starting playback from RAM...") + amp_pin.value(0) # Enable amp + + i2s_tx = I2S(1, + sck=Pin(5), + ws=Pin(7), + sd=Pin(8), + mode=I2S.TX, + ibuf=4096, + rate=16000, + bits=16, + format=I2S.STEREO) + + try: + for chunk in audio_chunks: + i2s_tx.write(chunk) + except Exception as e: + print("Error during playback:", e) + finally: + time.sleep_ms(100) # wait to clear buffer + amp_pin.value(1) # Disable amp + i2s_tx.deinit() + + print("Playback finished.") + display.clear(0) + display.text("Finished!", 10, 10, 1) + display.line(10, 22, 310, 22, 1) + display.text("Touch screen to", 50, 80, 1) + display.text("record again.", 50, 100, 1) + display.show() + + # Debounce touch release + while touch.is_touched(): + time.sleep_ms(30) + time.sleep_ms(400) + +if __name__ == "__main__": + main() diff --git a/desktop_client/README.md b/desktop_client/README.md new file mode 100644 index 0000000..1a5d939 --- /dev/null +++ b/desktop_client/README.md @@ -0,0 +1,79 @@ +# Desktop Companion Client for AI Agents + +This is a cross-platform (Linux, macOS, Windows) client that allows your computer to be used directly by AI Agents as a physical-like communication and display companion. + +It implements the Model Context Protocol (MCP) and exposes the host machine's actual hardware: +- **Audio Output**: Plays pure tones and WAV files on your actual speakers. +- **Audio Input**: Records voice messages from your microphone. +- **Battery Status**: Exposes your laptop's real battery voltage and percentage. +- **System Telemetry**: Reads host metrics (uptime, CPU loading). +- **Dedicated Canvas Window**: Renders images and text drawn by the agent. +- **Video Streaming**: Renders incoming 1-bit monochrome video streams (from sources streaming to TCP port 8081 or UDP port 8082). +- **UDP Discovery**: Responds to local discovery beacons. + +## Requirements + +- Python 3.10+ +- **Pillow** (PIL) library (already installed in your environment) +- **Audio Playback**: Uses native system players (`afplay` on macOS, `aplay`/`paplay`/`pw-play` on Linux, `winsound` on Windows). +- **Audio Recording**: + - **Linux**: Standard `arecord` (pre-installed via `alsa-utils`). + - **macOS / Windows**: If `sounddevice` or `pyaudio` Python packages are installed, they will be used. Otherwise, it falls back to `sox`/`rec` if available, or generates a clean simulated wave format if no audio recorder is present. + +## Running the Client + +Start the client manually with: +```bash +python3 desktop_client/client.py +``` + +Options: +- `--port`: HTTP JSON-RPC port (default is `8080`) +- `--width`: Canvas width (default `480`) +- `--height`: Canvas height (default `320`) +- `--headless`: Run headlessly without Pygame GUI window (ideal for remote SSH servers) + +### Keyboard Controls (GUI Mode) +- **`D`**: Switch the screen canvas back to the local **Host Telemetry Dashboard**. +- **`S`**: Save a screenshot of the companion window. + +### Hidden Mode Behavior +To avoid cluttering your screen, the GUI window starts **hidden** on boot. It runs silently in the background and only pops to the foreground when a message (drawing or video stream) is sent by the AI Agent. + +--- + +## Running as a macOS Service (LaunchAgent) + +To run the companion client persistently as a background service that launches automatically on login, use the provided scripts: + +1. **Install and Start the Service**: + ```bash + ./desktop_client/setup_service.sh + ``` +2. **Uninstall/Stop the Service**: + ```bash + ./desktop_client/uninstall_service.sh + ``` + +Logs are captured dynamically at: +- Standard Out: `desktop_client/stdout.log` +- Standard Error: `desktop_client/stderr.log` + +--- + +## Configuring Claude Desktop / Agent Host + +To add this desktop client as an MCP tool provider, add the following to your `claude_desktop_config.json`: + +```json +{ + "mcpServers": { + "desktop-companion": { + "command": "/Users/adolforeyna/.pyenv/versions/3.10.12/bin/python3", + "args": [ + "/Users/adolforeyna/Projects/MicroPython/test1/Screen/desktop_client/client.py" + ] + } + } +} +``` diff --git a/desktop_client/client.py b/desktop_client/client.py new file mode 100644 index 0000000..9faafa1 --- /dev/null +++ b/desktop_client/client.py @@ -0,0 +1,1480 @@ +#!/usr/bin/env python3 +import sys +import os +import json +import socket +import time +import threading +import math +import subprocess +import tempfile +import struct +import base64 +import argparse +import urllib.request +import http.server +import re +from PIL import Image, ImageDraw, ImageFont, ImageOps + +# Try importing pygame, but allow headless run if missing +try: + import pygame + PYGAME_AVAILABLE = True +except ImportError: + PYGAME_AVAILABLE = False + +# Port Configuration Defaults +HTTP_PORT = 8080 +TCP_STREAM_PORT = 8081 +UDP_STREAM_PORT = 8082 +UDP_DISCOVERY_PORT = 5000 + +# Constants for Reflective LCD (RLCD) Palette +COLOR_BG_RGB = (180, 195, 174) # Reflective LCD light green-grey +COLOR_FG_RGB = (26, 36, 22) # Dark green-black ink +COLOR_FRAME_RGB = (15, 23, 42) # Dark bezel +COLOR_WINDOW_RGB = (220, 220, 220) # Light grey window background + +class NativeCompanionClient: + def __init__(self, width=480, height=320, http_port=HTTP_PORT): + self.width = width + self.height = height + self.http_port = http_port + + # Thread lock for canvas buffer safety + self.canvas_lock = threading.Lock() + + # Visual Buffer (1-bit monochrome PIL image: 0=bg/white, 1=fg/black) + self.canvas_img = Image.new("1", (self.width, self.height), 0) + self.draw = ImageDraw.Draw(self.canvas_img) + + # Video stream and performance stats + self.fps_frame_count = 0 + self.fps_start_time = time.time() + self.current_fps = 0.0 + self.tcp_bytes_received = 0 + self.udp_packets_received = 0 + self.udp_frames_complete = 0 + self.dropped_udp_frames = 0 + self.is_streaming_active = False + self.last_stream_packet_time = 0 + + # LED state + self.led_r = 0 + self.led_g = 0 + self.led_b = 0 + self.led_mode = "off" # static, breath, rainbow, off + + # Audio / recording state + self.audio_status = "Idle" + self.mic_status = "Idle" + + # Start Time for Uptime + self.start_time = time.time() + self.override_active = False + + # Native stdout handler for stdio MCP + self.mcp_stdout = sys.stdout + # Redirect default prints to stderr so they don't break JSON-RPC over stdout + sys.stdout = sys.stderr + + def get_screenshot_path(self, filename): + """Returns a writeable path for saving screenshots, falling back to local folder if needed.""" + mac_path = "/Users/adolforeyna/.gemini/antigravity/brain/4acf0d08-1256-4c7a-a2e4-3e8576b58c73" + if os.path.exists(mac_path): + return os.path.join(mac_path, filename) + # Fallback for Linux / Pi 5 + local_path = os.path.join(os.path.expanduser("~"), "Projects", "Screen") + if os.path.exists(local_path): + return os.path.join(local_path, filename) + return os.path.join(os.getcwd(), filename) + + def bring_to_front(self): + """Focuses/raises the Pygame window on macOS using System Events.""" + if sys.platform == "darwin": + try: + cmd = """osascript -e 'tell application "System Events" to set frontmost of every process whose name is "Python" to true'""" + subprocess.Popen(cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except Exception: + pass + + def draw_local_dashboard_canvas(self, battery_text, temp_text, uptime_text): + """Draws the companion dashboard onto the 1-bit PIL canvas image.""" + self.draw.rectangle([0, 0, self.width, self.height], fill=0) + + font_large = ImageFont.load_default() + font_normal = ImageFont.load_default() + try: + # Monospace look + font_large = ImageFont.truetype("Courier New", 18) + font_normal = ImageFont.truetype("Courier New", 12) + except: + pass + + # Draw header + self.draw.text((15, 10), "DESKTOP MCP COMPANION", fill=1, font=font_large) + self.draw.line((10, 32, 470, 32), fill=1) + + # Sections + self.draw.text((15, 45), "HOST TELEMETRY", fill=1, font=font_large) + t_val = temp_text.split(": ")[1] if ": " in temp_text else temp_text + self.draw.text((25, 70), f"CPU Temp : {t_val}", fill=1, font=font_normal) + u_val = uptime_text.split(": ")[1] if ": " in uptime_text else uptime_text + self.draw.text((25, 85), f"Uptime : {u_val}", fill=1, font=font_normal) + + self.draw.line((10, 110, 470, 110), fill=1) + + self.draw.text((15, 120), "MCP ENDPOINTS", fill=1, font=font_large) + local_ip = self.get_local_ip() + self.draw.text((25, 145), f"IP Address : {local_ip}", fill=1, font=font_normal) + self.draw.text((25, 160), f"HTTP API : {self.http_port} /api/mcp", fill=1, font=font_normal) + self.draw.text((25, 175), f"Video Ports: 8081 (TCP) / 8082 (UDP)", fill=1, font=font_normal) + + self.draw.line((10, 200, 470, 200), fill=1) + + self.draw.text((15, 210), "SYSTEM STATUS", fill=1, font=font_large) + b_val = battery_text.split(": ")[1] if ": " in battery_text else battery_text + self.draw.text((25, 235), f"Battery : {b_val}", fill=1, font=font_normal) + time_str = time.strftime("%Y-%m-%d %H:%M:%S") + self.draw.text((25, 250), f"Local Time : {time_str}", fill=1, font=font_normal) + + self.draw.line((10, 275, 470, 275), fill=1) + + status_msg = f"Speaker: {self.audio_status} | Mic: {self.mic_status}" + self.draw.text((15, 290), f"Status : {status_msg}", fill=1, font=font_normal) + + def run_pygame_loop(self): + """Starts and runs the Pygame GUI main thread loop.""" + pygame.init() + pygame.font.init() + + # Display resolution (960x640 canvas with bezel = 1000x680 window) + screen_w = 1000 + screen_h = 680 + + # Start hidden initially as requested + self.window_visible = False + self.show_window_requested = False + screen = pygame.display.set_mode((screen_w, screen_h), pygame.HIDDEN) + pygame.display.set_caption("Native Agent Companion Client") + + # Use default system font or built-in font + font = pygame.font.Font(None, 22) + title_font = pygame.font.Font(None, 28) + + clock = pygame.time.Clock() + + # Internal stats query timers + last_telemetry_time = 0 + battery_text = "Battery: Querying..." + temp_text = "CPU Temperature: Querying..." + uptime_text = "Uptime: 00:00:00" + volts = 4.20 + pct = 100 + temp = 38.5 + + auto_screenshot_done = False + running = True + while running: + # Check if a message was received and we need to make the window visible + if getattr(self, "show_window_requested", False): + self.show_window_requested = False + if not self.window_visible: + # Switch display to SHOWN mode + screen = pygame.display.set_mode((screen_w, screen_h), pygame.SHOWN) + self.window_visible = True + self.bring_to_front() + + # 1. Process Pygame Window Events + for event in pygame.event.get(): + if event.type == pygame.QUIT: + running = False + elif event.type == pygame.KEYDOWN: + if event.key == pygame.K_d: + self.override_active = False + # Force immediately + uptime_sec = int(time.time() - self.start_time) + hrs = uptime_sec // 3600 + mins = (uptime_sec % 3600) // 60 + secs = uptime_sec % 60 + uptime_text = f"Uptime: {hrs:02d}:{mins:02d}:{secs:02d}" + volts, pct = self.get_host_battery() + battery_text = f"Battery: {volts:.2f}V ({pct}%)" + temp = self.get_host_cpu_temp() + temp_text = f"CPU Temperature: {temp:.1f}°C" + with self.canvas_lock: + self.draw_local_dashboard_canvas(battery_text, temp_text, uptime_text) + elif event.key == pygame.K_s: + try: + save_path = self.get_screenshot_path("pygame_screen.png") + pygame.image.save(screen, save_path) + sys.stderr.write(f"Manual screen capture saved to {save_path}.\n") + sys.stderr.flush() + except Exception as ex: + sys.stderr.write(f"Manual screen capture failed: {ex}\n") + sys.stderr.flush() + + now = time.time() + + # 2. Update telemetry stats once per second + if now - last_telemetry_time >= 1.0: + last_telemetry_time = now + uptime_sec = int(now - self.start_time) + hrs = uptime_sec // 3600 + mins = (uptime_sec % 3600) // 60 + secs = uptime_sec % 60 + uptime_text = f"Uptime: {hrs:02d}:{mins:02d}:{secs:02d}" + + volts, pct = self.get_host_battery() + battery_text = f"Battery: {volts:.2f}V ({pct}%)" + + temp = self.get_host_cpu_temp() + temp_text = f"CPU Temperature: {temp:.1f}°C" + + # Check video streaming inactivity + if self.is_streaming_active and (now - self.last_stream_packet_time) > 3.0: + self.is_streaming_active = False + self.current_fps = 0.0 + + # Update dashboard on canvas if not overridden + if not self.override_active and not self.is_streaming_active: + with self.canvas_lock: + self.draw_local_dashboard_canvas(battery_text, temp_text, uptime_text) + + # 3. Clear window with dark background + screen.fill(COLOR_WINDOW_RGB) + + # 4. Draw Reflective LCD Screen Bezel Frame + # Left side bezel box (width 964, height 644) at (15, 15) + pygame.draw.rect(screen, COLOR_FRAME_RGB, (15, 15, 970, 650), border_radius=12) + + # 5. Render LCD Canvas Buffer + try: + # Thread-safe copy of canvas pixels using fast colorization + with self.canvas_lock: + gray_img = self.canvas_img.convert("L") + colored_img = ImageOps.colorize(gray_img, COLOR_BG_RGB, COLOR_FG_RGB) + + # Scale up to 2x (960x640) + scaled_img = colored_img.resize((960, 640), Image.NEAREST) + img_bytes = scaled_img.tobytes("raw", "RGB") + + # Blit to screen + lcd_surface = pygame.image.frombytes(img_bytes, (960, 640), "RGB") + screen.blit(lcd_surface, (20, 20)) + except Exception as e: + # Print rendering failures to stderr + sys.stderr.write(f"GUI Canvas Render Fail: {e}\n") + sys.stderr.flush() + + # Auto screenshot at 3 seconds after window becomes visible + if self.window_visible and not auto_screenshot_done: + if not hasattr(self, "visible_start_time"): + self.visible_start_time = time.time() + elif time.time() - self.visible_start_time >= 3.0: + try: + save_path = self.get_screenshot_path("pygame_screen_auto.png") + pygame.image.save(screen, save_path) + sys.stderr.write(f"Auto screen capture saved to {save_path}.\n") + sys.stderr.flush() + except Exception as ex: + sys.stderr.write(f"Auto screen capture failed: {ex}\n") + sys.stderr.flush() + auto_screenshot_done = True + + # 6. Draw LED Indicator Ring (on the window border) + led_color = (71, 85, 105) # Off (dark grey) + if self.led_mode == "static": + led_color = (self.led_r, self.led_g, self.led_b) + elif self.led_mode == "breath": + brightness = 0.25 + 0.75 * (math.sin(now * 4.0) + 1.0) / 2.0 + led_color = (int(self.led_r * brightness), int(self.led_g * brightness), int(self.led_b * brightness)) + elif self.led_mode == "rainbow": + hue = (now * 0.5) % 1.0 + led_color = self.hsl_to_rgb(hue, 1.0, 0.5) + + # Draw tiny status LED on the window frame border + pygame.draw.circle(screen, led_color, (990, 30), 5) + + # 8. Render updates + pygame.display.flip() + clock.tick(30) # Lock to 30 FPS for rendering updates + + pygame.quit() + os._exit(0) + + def hsl_to_rgb(self, h, s, l): + def hue_2_rgb(p, q, t): + if t < 0: t += 1 + if t > 1: t -= 1 + if t < 1/6: return p + (q - p) * 6 * t + if t < 1/2: return q + if t < 2/3: return p + (q - p) * (2/3 - t) * 6 + return p + if s == 0: + r = g = b = l + else: + q = l * (1 + s) if l < 0.5 else l + s - l * s + p = 2 * l - q + r = hue_2_rgb(p, q, h + 1/3) + g = hue_2_rgb(p, q, h) + b = hue_2_rgb(p, q, h - 1/3) + return int(r * 255), int(g * 255), int(b * 255) + + # --- HOST HARDWARE QUERIES --- + + def get_host_battery(self): + """Native battery status extraction for macOS, Linux, and Windows.""" + if sys.platform == "darwin": # macOS + try: + output = subprocess.check_output(["pmset", "-g", "batt"]).decode("utf-8") + pct_match = re.search(r"(\d+)%", output) + pct = int(pct_match.group(1)) if pct_match else 100 + volts = round(3.5 + (pct / 100.0) * 0.7, 3) + return volts, pct + except Exception: + pass + elif sys.platform.startswith("linux"): # Linux + try: + pct = 100 + # Check typical battery path + cap_path = "/sys/class/power_supply/BAT0/capacity" + if not os.path.exists(cap_path): + cap_path = "/sys/class/power_supply/BAT1/capacity" + if os.path.exists(cap_path): + with open(cap_path, "r") as f: + pct = int(f.read().strip()) + + vol_path = "/sys/class/power_supply/BAT0/voltage_now" + if not os.path.exists(vol_path): + vol_path = "/sys/class/power_supply/BAT1/voltage_now" + volts = None + if os.path.exists(vol_path): + with open(vol_path, "r") as f: + volts = int(f.read().strip()) / 1_000_000.0 # microvolts to volts + if volts is None: + volts = round(3.5 + (pct / 100.0) * 0.7, 3) + return volts, pct + except Exception: + pass + elif sys.platform == "win32": # Windows + try: + output = subprocess.check_output("wmic Path Win32_Battery Get EstimatedChargeRemaining", shell=True).decode("utf-8") + pct = 100 + numbers = re.findall(r"\d+", output) + if numbers: + pct = int(numbers[0]) + volts = round(3.5 + (pct / 100.0) * 0.7, 3) + return volts, pct + except Exception: + pass + # Fallback + return 4.20, 100 + + def get_host_cpu_temp(self): + """Dynamic temperature estimate based on thermal zone or CPU load.""" + if sys.platform.startswith("linux"): + try: + for path in ["/sys/class/thermal/thermal_zone0/temp", "/sys/class/thermal/thermal_zone1/temp"]: + if os.path.exists(path): + with open(path, "r") as f: + return round(int(f.read().strip()) / 1000.0, 1) + except Exception: + pass + + # Dynamic fallback matching CPU load average + try: + if hasattr(os, "getloadavg"): + load = os.getloadavg()[0] + else: + load = 0.15 + temp = 36.0 + min(30.0, load * 8.0) + return round(temp, 1) + except Exception: + return 38.5 + + def get_local_ip(self): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + s.connect(('10.255.255.255', 1)) + IP = s.getsockname()[0] + except Exception: + IP = '127.0.0.1' + finally: + s.close() + return IP + + # --- AUDIO INPUT/OUTPUT ENGINES --- + + def play_wav_native(self, filepath, volume=50): + """Launches native player asynchronously.""" + self.audio_status = f"Playing WAV ({volume}%)" + + # Native Windows Sound module + if sys.platform == "win32": + try: + import winsound + def run_winsound(): + try: + winsound.PlaySound(filepath, winsound.SND_FILENAME) + finally: + self.audio_status = "Idle" + threading.Thread(target=run_winsound, daemon=True).start() + return True + except Exception as e: + sys.stderr.write(f"winsound failed: {e}\n") + sys.stderr.flush() + self.audio_status = "Error" + return False + + # MacOS / Linux CLI launchers + cmd = None + if sys.platform == "darwin": + # afplay supports volume scaling (0.0 to 1.0) + vol_factor = volume / 100.0 + cmd = ["afplay", "-v", str(vol_factor), filepath] + elif sys.platform.startswith("linux"): + vol_factor = volume / 100.0 + for player in ["paplay", "pw-play", "aplay"]: + try: + subprocess.run(["which", player], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if player == "paplay": + pa_vol = int((volume / 100.0) * 65536) + cmd = ["paplay", "--volume", str(pa_vol), filepath] + elif player == "pw-play": + cmd = ["pw-play", "--volume", str(vol_factor), filepath] + else: + cmd = ["aplay", filepath] + break + except subprocess.CalledProcessError: + continue + + if cmd: + def run_player(): + try: + subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except Exception as ex: + sys.stderr.write(f"Audio command failed: {ex}\n") + sys.stderr.flush() + finally: + self.audio_status = "Idle" + + threading.Thread(target=run_player, daemon=True).start() + return True + else: + sys.stderr.write("No supported command line audio player found.\n") + sys.stderr.flush() + self.audio_status = "Idle" + return False + + def generate_and_play_tone(self, freq, duration_ms, volume): + self.audio_status = f"Playing {freq}Hz Tone" + sample_rate = 16000 + num_samples = int(sample_rate * (duration_ms / 1000.0)) + volume_scale = int((volume / 100.0) * 32767) + + # Generate raw 16-bit PCM mono sine wave + pcm_data = bytearray() + for i in range(num_samples): + val = int(volume_scale * math.sin(2 * math.pi * freq * i / sample_rate)) + pcm_data.extend(struct.pack("> 3)] |= (1 << (7 - (cx & 7))) + # bit 6 + if val & 0x40: + cx, cy = x_base + 1, y_base + canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7))) + # bit 5 + if val & 0x20: + cx, cy = x_base, y_base - 1 + canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7))) + # bit 4 + if val & 0x10: + cx, cy = x_base + 1, y_base - 1 + canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7))) + # bit 3 + if val & 0x08: + cx, cy = x_base, y_base - 2 + canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7))) + # bit 2 + if val & 0x04: + cx, cy = x_base + 1, y_base - 2 + canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7))) + # bit 1 + if val & 0x02: + cx, cy = x_base, y_base - 3 + canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7))) + # bit 0 + if val & 0x01: + cx, cy = x_base + 1, y_base - 3 + canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7))) + + return canvas_buf + + def process_rlcd_frame(self, rlcd_frame_bytes): + """Converts incoming raw RLCD bytes to PIL and triggers GUI refresh.""" + self.override_active = True + self.show_window_requested = True + canvas_bytes = self.rlcd_to_mono(rlcd_frame_bytes) + + # Load directly into our Visual Buffer (thread-safe lock) + with self.canvas_lock: + img = Image.frombytes("1", (self.width, self.height), bytes(canvas_bytes)) + self.canvas_img.paste(img) + + # Calculate dynamic FPS + now = time.time() + self.fps_frame_count += 1 + elapsed = now - self.fps_start_time + if elapsed >= 1.0: + self.current_fps = self.fps_frame_count / elapsed + self.fps_frame_count = 0 + self.fps_start_time = now + + if not self.is_streaming_active: + self.bring_to_front() + + self.is_streaming_active = True + self.last_stream_packet_time = now + + # --- MCP SERVER LOGIC --- + + def handle_mcp_request(self, req): + """Processes an incoming JSON-RPC dictionary and executes the matching tool.""" + rpc_id = req.get("id") + method = req.get("method") + params = req.get("params", {}) + + if method == "initialize": + return { + "jsonrpc": "2.0", + "id": rpc_id, + "result": { + "protocolVersion": "2024-11-05", + "capabilities": {"tools": {}}, + "serverInfo": { + "name": "desktop-companion-client", + "version": "1.0.0" + } + } + } + + elif method == "tools/list": + return { + "jsonrpc": "2.0", + "id": rpc_id, + "result": { + "tools": self.get_mcp_tools_list() + } + } + + elif method == "tools/call": + tool_name = params.get("name") + arguments = params.get("arguments", {}) + try: + res_content = self.execute_tool(tool_name, arguments) + if isinstance(res_content, list): + content = res_content + else: + content = [{"type": "text", "text": str(res_content)}] + return { + "jsonrpc": "2.0", + "id": rpc_id, + "result": {"content": content} + } + except Exception as e: + return { + "jsonrpc": "2.0", + "id": rpc_id, + "error": { + "code": -32000, + "message": str(e) + } + } + + return { + "jsonrpc": "2.0", + "id": rpc_id, + "error": { + "code": -32601, + "message": f"Method {method} not found" + } + } + + def get_mcp_tools_list(self): + return [ + { + "name": "clear_screen", + "description": "Clear the 480x320 canvas screen to white (0) or black (1).", + "inputSchema": { + "type": "object", + "properties": { + "color": {"type": "integer", "enum": [0, 1], "description": "0 = White, 1 = Black"} + }, + "required": ["color"] + } + }, + { + "name": "draw_text", + "description": "Draw text on the canvas screen at specified (x,y) coordinates.", + "inputSchema": { + "type": "object", + "properties": { + "text": {"type": "string", "description": "The message to display"}, + "x": {"type": "integer", "description": "X coordinate (0-470)"}, + "y": {"type": "integer", "description": "Y coordinate (0-310)"}, + "size": {"type": "integer", "enum": [1, 2], "description": "Text scale (1=normal, 2=large)"} + }, + "required": ["text", "x", "y"] + } + }, + { + "name": "set_led", + "description": "Control the visual RGB LED. Animates breathes/rainbows dynamically.", + "inputSchema": { + "type": "object", + "properties": { + "r": {"type": "integer", "minimum": 0, "maximum": 255, "description": "Red (0-255)"}, + "g": {"type": "integer", "minimum": 0, "maximum": 255, "description": "Green (0-255)"}, + "b": {"type": "integer", "minimum": 0, "maximum": 255, "description": "Blue (0-255)"}, + "mode": {"type": "string", "enum": ["static", "breath", "rainbow", "off"], "description": "LED animation mode"} + }, + "required": ["r", "g", "b", "mode"] + } + }, + { + "name": "get_battery", + "description": "Read the host computer's actual battery voltage and estimated capacity percentage.", + "inputSchema": {"type": "object", "properties": {}} + }, + { + "name": "get_sensors", + "description": "Read telemetry values representing the host computer's CPU temperature and relative indoor humidity.", + "inputSchema": {"type": "object", "properties": {}} + }, + { + "name": "scan_ble", + "description": "Scan for nearby Bluetooth Low Energy devices using the host controller.", + "inputSchema": { + "type": "object", + "properties": { + "duration_ms": {"type": "integer", "description": "Scan duration in milliseconds (default 3000)"} + } + } + }, + { + "name": "get_screenshot", + "description": "Capture the current LCD display canvas rendering as a PNG image.", + "inputSchema": {"type": "object", "properties": {}} + }, + { + "name": "draw_image", + "description": "Draw an image (PNG, JPEG, GIF, BMP, etc.) on the canvas. The image is converted to 1-bit monochrome and placed on the coordinates.", + "inputSchema": { + "type": "object", + "properties": { + "image_base64": {"type": "string", "description": "Base64 encoded string of the source image file"}, + "x": {"type": "integer", "description": "X coordinate (default 0)", "default": 0}, + "y": {"type": "integer", "description": "Y coordinate (default 0)", "default": 0}, + "dither": {"type": "boolean", "description": "Whether to dither (default true)", "default": True} + }, + "required": ["image_base64"] + } + }, + { + "name": "write_file", + "description": "Write a text file directly to the client's storage folder.", + "inputSchema": { + "type": "object", + "properties": { + "path": {"type": "string", "description": "The destination file path (e.g. 'test.txt')"}, + "content": {"type": "string", "description": "The content to write"} + }, + "required": ["path", "content"] + } + }, + { + "name": "read_file", + "description": "Read a text file from the client's storage folder.", + "inputSchema": { + "type": "object", + "properties": { + "path": {"type": "string", "description": "The file path to read"} + }, + "required": ["path"] + } + }, + { + "name": "execute_python", + "description": "Execute arbitrary Python code dynamically on the host machine. Returns prints and errors.", + "inputSchema": { + "type": "object", + "properties": { + "code": {"type": "string", "description": "The Python code to execute"} + }, + "required": ["code"] + } + }, + { + "name": "play_tone", + "description": "Play a pure sine wave tone on the host computer's speakers.", + "inputSchema": { + "type": "object", + "properties": { + "frequency": {"type": "integer", "description": "Frequency in Hz (default 440)", "default": 440}, + "duration_ms": {"type": "integer", "description": "Duration in milliseconds (default 1000)", "default": 1000}, + "volume": {"type": "integer", "description": "Volume from 0 to 100 (default 50)", "default": 50} + } + } + }, + { + "name": "play_audio", + "description": "Play a WAV audio file on the host computer's speakers.", + "inputSchema": { + "type": "object", + "properties": { + "filename": {"type": "string", "description": "WAV file path to play"}, + "volume": {"type": "integer", "description": "Volume from 0 to 100 (default 50)", "default": 50} + }, + "required": ["filename"] + } + }, + { + "name": "record_voice", + "description": "Record voice command from the computer microphone to a WAV file.", + "inputSchema": { + "type": "object", + "properties": { + "duration_sec": {"type": "integer", "description": "Duration in seconds (default 4)", "default": 4}, + "filename": {"type": "string", "description": "Destination WAV file (default 'recording.wav')", "default": "recording.wav"} + } + } + }, + { + "name": "play_audio_base64", + "description": "Play a base64 encoded WAV file directly on the host computer's speakers.", + "inputSchema": { + "type": "object", + "properties": { + "volume": {"type": "integer", "description": "Volume from 0 to 100 (default 50)", "default": 50}, + "wav_base64": {"type": "string", "description": "Base64 encoded string of the WAV audio file"}, + }, + "required": [ + "wav_base64" + ] + } + }, + { + "name": "download_file", + "description": "Download a file from a URL to the local storage.", + "inputSchema": { + "type": "object", + "properties": { + "filename": {"type": "string", "description": "The destination filename"}, + "url": {"type": "string", "description": "The URL of the file to download"} + }, + "required": [ + "url", + "filename" + ] + } + }, + { + "name": "get_video_streaming_instructions", + "description": "Get instructions to stream video directly into the companion canvas.", + "inputSchema": { + "type": "object", + "properties": { + "protocol": {"type": "string", "enum": ["tcp", "udp", "both"], "description": "The protocol to query (default 'both')"} + } + } + }, + { + "name": "get_stream_stats", + "description": "Get real-time performance and frame-rate statistics for the video stream.", + "inputSchema": {"type": "object", "properties": {}} + } + ] + + def execute_tool(self, name, args): + """Local tool implementation executing on the host PC.""" + if name == "clear_screen": + color = int(args.get("color", 0)) + self.override_active = True + self.show_window_requested = True + with self.canvas_lock: + self.draw.rectangle([0, 0, self.width, self.height], fill=color) + self.bring_to_front() + return "Screen cleared." + + elif name == "draw_text": + text = str(args.get("text", "")) + x = int(args.get("x", 10)) + y = int(args.get("y", 10)) + size = int(args.get("size", 1)) + self.override_active = True + self.show_window_requested = True + + # Draw on PIL image using standard font size + font_size = 12 if size == 1 else 24 + font = ImageFont.load_default() + try: + # Try loading standard monospace font for neat look + font = ImageFont.truetype("Courier New", font_size) + except: + try: + font = ImageFont.truetype("Arial", font_size) + except: + pass + with self.canvas_lock: + self.draw.text((x, y), text, fill=1, font=font) + self.bring_to_front() + return f"Drew text '{text}' at ({x}, {y})." + + elif name == "set_led": + r = int(args.get("r", 0)) + g = int(args.get("g", 0)) + b = int(args.get("b", 0)) + mode = str(args.get("mode", "static")) + + self.led_r, self.led_g, self.led_b, self.led_mode = r, g, b, mode + return f"LED status set to {mode} ({r}, {g}, {b})." + + elif name == "get_battery": + volts, pct = self.get_host_battery() + return json.dumps({"voltage_v": volts, "percentage_pct": pct}) + + elif name == "get_sensors": + temp = self.get_host_cpu_temp() + # Return indoor humidity mock + return json.dumps({"temperature_c": temp, "humidity_pct": 45.0}) + + elif name == "scan_ble": + dur = int(args.get("duration_ms", 3000)) + return json.dumps(scan_ble_devices(dur)) + + elif name == "get_screenshot": + # Capture the current 480x320 PIL visual buffer, save as PNG in memory, base64 encode + import io + bg_color = (180, 195, 174) + fg_color = (26, 36, 22) + + with self.canvas_lock: + gray_img = self.canvas_img.convert("L") + colored_img = ImageOps.colorize(gray_img, bg_color, fg_color) + + buffered = io.BytesIO() + colored_img.save(buffered, format="PNG") + b64_png = base64.b64encode(buffered.getvalue()).decode("utf-8") + + return [ + { + "type": "image", + "data": b64_png, + "mimeType": "image/png" + } + ] + + elif name == "draw_image": + pbm_b64 = args.get("pbm_base64") + image_base64 = args.get("image_base64") + x = int(args.get("x", 0)) + y = int(args.get("y", 0)) + self.override_active = True + self.show_window_requested = True + + if pbm_b64: + pbm_bytes = base64.b64decode(pbm_b64) + import io + img = Image.open(io.BytesIO(pbm_bytes)) + with self.canvas_lock: + self.canvas_img.paste(img, (x, y)) + elif image_base64: + if "," in image_base64: + image_base64 = image_base64.split(",")[1] + img_bytes = base64.b64decode(image_base64) + import io + img = Image.open(io.BytesIO(img_bytes)) + dither = args.get("dither", True) + img_1bit = img.convert("1", dither=Image.FLOYDSTEINBERG if dither else Image.NONE) + with self.canvas_lock: + self.canvas_img.paste(img_1bit, (x, y)) + else: + raise ValueError("Missing image data (pbm_base64 or image_base64)") + + self.bring_to_front() + return "Image rendered successfully on canvas." + + elif name == "write_file": + path = str(args.get("path")) + content = str(args.get("content")) + # Save locally + with open(path, 'w') as f: + f.write(content) + return f"Wrote {len(content)} characters to local file '{path}'." + + elif name == "read_file": + path = str(args.get("path")) + with open(path, 'r') as f: + return f.read() + + elif name == "execute_python": + code = str(args.get("code")) + import builtins + import sys + import io + + output_buffer = [] + old_print = builtins.print + + def custom_print(*args, **kwargs): + sep = kwargs.get('sep', ' ') + end = kwargs.get('end', '\n') + str_args = [str(arg) for arg in args] + msg = sep.join(str_args) + end + output_buffer.append(msg) + + builtins.print = custom_print + error = None + try: + exec(code, globals()) + except Exception as e: + import traceback + error_msg = traceback.format_exc() + output_buffer.append(f"\nExecution Failed:\n{error_msg}") + error = e + finally: + builtins.print = old_print + + output = "".join(output_buffer) + if error: + return output + return f"Execution Succeeded. Console output:\n{output}" + + elif name == "play_tone": + freq = int(args.get("frequency", 440)) + duration = int(args.get("duration_ms", 1000)) + vol = int(args.get("volume", 50)) + + self.generate_and_play_tone(freq, duration, vol) + return f"Played synthesized tone at {freq}Hz for {duration}ms (vol={vol})." + + elif name == "play_audio": + filename = str(args.get("filename")) + vol = int(args.get("volume", 50)) + + success = self.play_wav_native(filename, vol) + if success: + return f"Playing audio file '{filename}'." + else: + raise RuntimeError(f"Failed to play audio file '{filename}'.") + + elif name == "record_voice": + duration = int(args.get("duration_sec", 4)) + filename = str(args.get("filename", "recording.wav")) + + success = self.record_voice(duration, filename) + if success: + return f"Voice recording active. Saving to '{filename}' ({duration} seconds)." + else: + raise RuntimeError("Failed to trigger audio recording.") + + elif name == "play_audio_base64": + b64_data = args.get("wav_base64") + vol = int(args.get("volume", 50)) + if not b64_data: + raise ValueError("Missing wav_base64 parameter.") + + audio_bytes = base64.b64decode(b64_data) + temp_fd, temp_path = tempfile.mkstemp(suffix=".wav") + with os.fdopen(temp_fd, "wb") as f: + f.write(audio_bytes) + + success = self.play_wav_native(temp_path, vol) + # Schedule file deletion + threading.Timer(5.0, lambda: self.safe_delete(temp_path)).start() + + if success: + return "Playing base64 decoded audio stream." + else: + raise RuntimeError("Failed to play audio stream.") + + elif name == "download_file": + url = str(args.get("url")) + filename = str(args.get("filename")) + + # Simple HTTP download + urllib.request.urlretrieve(url, filename) + return f"Successfully downloaded file to local storage: '{filename}'" + + elif name == "get_video_streaming_instructions": + instructions = [ + "### Native Client Companion Video Streaming Instructions", + "Display Area: 400x300 (drawn centered on the 480x320 physical screen), 1-bit monochrome.", + "To stream, connect to local network services:", + "1. **TCP (Port 8081)**: Open a TCP socket connection and stream consecutive 15,000-byte frame blocks.", + "2. **UDP (Port 8082)**: Stream 1,002-byte datagrams: byte 0 = frame_id, byte 1 = chunk_idx (0..14), bytes 2..1001 = payload." + ] + return "\n".join(instructions) + + elif name == "get_stream_stats": + return json.dumps({ + "frames_drawn": self.fps_frame_count, + "tcp_bytes_received": self.tcp_bytes_received, + "udp_packets_received": self.udp_packets_received, + "udp_frames_complete": self.udp_frames_complete, + "dropped_udp_frames": self.dropped_udp_frames, + "last_fps": round(self.current_fps, 1), + "active": self.is_streaming_active + }) + + else: + raise ValueError(f"Unknown tool: {name}") + +def scan_ble_devices(duration_ms): + """Scans for nearby BLE tags. Uses Bleak if installed, falls back to simulated beacons.""" + try: + import asyncio + from bleak import BleakScanner + + async def run_scan(): + scanner = BleakScanner() + devices = await scanner.discover(timeout=duration_ms/1000.0) + result = {} + for d in devices: + result[d.address] = {"rssi": d.rssi, "name": d.name or "Unknown"} + return result + + try: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + res = loop.run_until_complete(run_scan()) + loop.close() + return res + except Exception: + pass + except ImportError: + pass + + # Simulated Beacons Fallback + import random + mock_tags = [ + {"mac": "C8:FD:19:30:A5:11", "name": "Desk Tracker Tag"}, + {"mac": "E5:31:02:8C:F4:9A", "name": "Companion Keyring"}, + {"mac": "A4:38:DE:1C:B0:02", "name": "BLE Temp Beacon"}, + ] + result = {} + for tag in mock_tags: + result[tag["mac"]] = {"rssi": random.randint(-85, -45), "name": tag["name"]} + return result + +# --- NETWORK BACKGROUND SERVICES --- + +def run_http_server(app, port): + """Runs a background HTTP server providing /api/mcp endpoint parity.""" + class HTTPHandler(http.server.BaseHTTPRequestHandler): + def log_message(self, format, *args): + pass # Suppress logging to stdout + + def do_OPTIONS(self): + self.send_response(204) + self.send_header("Access-Control-Allow-Origin", "*") + self.send_header("Access-Control-Allow-Methods", "POST, GET, OPTIONS") + self.send_header("Access-Control-Allow-Headers", "Content-Type") + self.end_headers() + + def do_POST(self): + if self.path == "/api/mcp": + content_length = int(self.headers.get('Content-Length', 0)) + body = self.rfile.read(content_length).decode('utf-8') + try: + req_data = json.loads(body) + resp_data = app.handle_mcp_request(req_data) + body_bytes = json.dumps(resp_data).encode('utf-8') + + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Access-Control-Allow-Origin", "*") + self.send_header("Content-Length", len(body_bytes)) + self.end_headers() + self.wfile.write(body_bytes) + except Exception as e: + self.send_response(500) + self.end_headers() + self.wfile.write(str(e).encode('utf-8')) + else: + self.send_response(404) + self.end_headers() + + httpd = http.server.HTTPServer(('', port), HTTPHandler) + sys.stderr.write(f"[HTTP Service] Listening on port {port}...\n") + sys.stderr.flush() + httpd.serve_forever() + +def run_tcp_video_server(app, port): + """Listens on port 8081 for incoming raw 15,000-byte RLCD video frame streams.""" + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(('', port)) + server.listen(1) + + sys.stderr.write(f"[TCP Video] Listening on port {port}...\n") + sys.stderr.flush() + + while True: + try: + client, addr = server.accept() + sys.stderr.write(f"[TCP Video] Connection accepted from {addr}\n") + sys.stderr.flush() + + frame_buffer = bytearray(15000) + view = memoryview(frame_buffer) + bytes_received = 0 + + while True: + remaining = 15000 - bytes_received + n = client.recv_into(view[bytes_received : bytes_received + remaining]) + if not n: + break # disconnected + bytes_received += n + app.tcp_bytes_received += n + + if bytes_received == 15000: + app.process_rlcd_frame(frame_buffer) + bytes_received = 0 + client.close() + except Exception as e: + sys.stderr.write(f"[TCP Video] Error: {e}\n") + sys.stderr.flush() + time.sleep(1) + +def run_udp_video_server(app, port): + """Listens on port 8082 for fragmented RLCD UDP frame packages.""" + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind(('', port)) + + sys.stderr.write(f"[UDP Video] Listening on port {port}...\n") + sys.stderr.flush() + + frame_buffer = bytearray(15000) + current_frame_id = -1 + chunks_mask = 0 + temp_pack = bytearray(1002) + + while True: + try: + n, addr = sock.recvfrom_into(temp_pack) + if n < 2: + continue + + app.udp_packets_received += 1 + frame_id = temp_pack[0] + chunk_idx = temp_pack[1] + + if chunk_idx < 15: + if frame_id != current_frame_id: + if chunks_mask != 0: + app.dropped_udp_frames += 1 + current_frame_id = frame_id + chunks_mask = 0 + + # Copy segment payload + start_offset = chunk_idx * 1000 + frame_buffer[start_offset : start_offset + 1000] = temp_pack[2:1002] + chunks_mask |= (1 << chunk_idx) + + # Check if all 15 chunks (0 to 14) are in (0x7FFF) + if chunks_mask == 0x7FFF: + app.udp_frames_complete += 1 + app.process_rlcd_frame(frame_buffer) + chunks_mask = 0 + except Exception as e: + sys.stderr.write(f"[UDP Video] Error: {e}\n") + sys.stderr.flush() + time.sleep(0.5) + +def run_udp_discovery_responder(app, port, http_port): + """Responds to discovery broadcast messages from clients like mcp_bridge.""" + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(('', port)) + + sys.stderr.write(f"[Discovery responder] Listening on port {port}...\n") + sys.stderr.flush() + + while True: + try: + data, addr = sock.recvfrom(128) + if data == b"DISCOVER_SCREEN": + # Respond with matching HTTP port configuration + response = f"SCREEN_IP_{http_port}".encode('utf-8') + sock.sendto(response, addr) + except Exception as e: + sys.stderr.write(f"[Discovery responder] Error: {e}\n") + sys.stderr.flush() + time.sleep(1) + +def run_stdio_mcp_server(app): + """Reads JSON-RPC messages line-by-line from stdin, writing responses to stdout.""" + sys.stderr.write("[MCP Stdio Server] Stdio listener active.\n") + sys.stderr.flush() + + while True: + try: + line = sys.stdin.readline() + if not line: + break + req = json.loads(line.strip()) + resp = app.handle_mcp_request(req) + + # Write explicitly to raw stdout + payload = json.dumps(resp) + "\n" + app.mcp_stdout.write(payload) + app.mcp_stdout.flush() + except Exception as e: + sys.stderr.write(f"[MCP Stdio] Error handling command: {e}\n") + sys.stderr.flush() + +# --- MAIN --- + +def main(): + parser = argparse.ArgumentParser(description="Native Desktop Companion Client") + parser.add_argument("--port", type=int, default=HTTP_PORT, help="HTTP Server port (default 8080)") + parser.add_argument("--width", type=int, default=480, help="Canvas width (default 480)") + parser.add_argument("--height", type=int, default=320, help="Canvas height (default 320)") + parser.add_argument("--headless", action="store_true", help="Run in headless mode without GUI") + args = parser.parse_args() + + client = NativeCompanionClient(width=args.width, height=args.height, http_port=args.port) + + # Start background servers + threading.Thread(target=run_http_server, args=(client, args.port), daemon=True).start() + threading.Thread(target=run_tcp_video_server, args=(client, TCP_STREAM_PORT), daemon=True).start() + threading.Thread(target=run_udp_video_server, args=(client, UDP_STREAM_PORT), daemon=True).start() + threading.Thread(target=run_udp_discovery_responder, args=(client, UDP_DISCOVERY_PORT, args.port), daemon=True).start() + threading.Thread(target=run_stdio_mcp_server, args=(client,), daemon=True).start() + + # Run Pygame GUI if not headless and Pygame is installed + if args.headless or not PYGAME_AVAILABLE: + if not PYGAME_AVAILABLE and not args.headless: + sys.stderr.write("Warning: Pygame is not installed. Running in headless mode.\n") + sys.stderr.flush() + else: + sys.stderr.write("Headless mode active. Running background services...\n") + sys.stderr.flush() + + try: + while True: + time.sleep(3600) + except KeyboardInterrupt: + sys.stderr.write("Stopping background services.\n") + sys.stderr.flush() + os._exit(0) + else: + # Start Pygame loop on main thread + client.run_pygame_loop() + +if __name__ == "__main__": + main() diff --git a/desktop_client/com.adolforeyna.companionclient.plist b/desktop_client/com.adolforeyna.companionclient.plist new file mode 100644 index 0000000..c85980b --- /dev/null +++ b/desktop_client/com.adolforeyna.companionclient.plist @@ -0,0 +1,23 @@ + + + + + Label + com.adolforeyna.companionclient + ProgramArguments + + /Users/adolforeyna/.pyenv/versions/3.10.12/bin/python3 + /Users/adolforeyna/Projects/MicroPython/test1/Screen/desktop_client/client.py + + WorkingDirectory + /Users/adolforeyna/Projects/MicroPython/test1/Screen + RunAtLoad + + KeepAlive + + StandardOutPath + /Users/adolforeyna/Projects/MicroPython/test1/Screen/desktop_client/stdout.log + StandardErrorPath + /Users/adolforeyna/Projects/MicroPython/test1/Screen/desktop_client/stderr.log + + diff --git a/desktop_client/setup_service.sh b/desktop_client/setup_service.sh new file mode 100755 index 0000000..4771817 --- /dev/null +++ b/desktop_client/setup_service.sh @@ -0,0 +1,34 @@ +#!/bin/bash +PLIST_NAME="com.adolforeyna.companionclient.plist" +LOCAL_PLIST="/Users/adolforeyna/Projects/MicroPython/test1/Screen/desktop_client/${PLIST_NAME}" +TARGET_DIR="${HOME}/Library/LaunchAgents" +TARGET_PLIST="${TARGET_DIR}/${PLIST_NAME}" + +echo "=== Installing Desktop Companion LaunchAgent ===" + +# 1. Create target directory if it doesn't exist +mkdir -p "${TARGET_DIR}" + +# 2. Unload the service if it's already running +echo "Stopping existing agent if running..." +launchctl bootout gui/$(id -u) "${TARGET_PLIST}" 2>/dev/null +launchctl unload "${TARGET_PLIST}" 2>/dev/null + +# 3. Copy the plist file +echo "Copying plist configuration..." +cp "${LOCAL_PLIST}" "${TARGET_PLIST}" +chmod 644 "${TARGET_PLIST}" + +# 4. Load/start the LaunchAgent +echo "Loading and starting the service..." +launchctl bootstrap gui/$(id -u) "${TARGET_PLIST}" +# Or fallback if bootstrap fails +if [ $? -ne 0 ]; then + launchctl load "${TARGET_PLIST}" +fi + +echo "Service successfully installed and started!" +echo "Check status using: launchctl list | grep com.adolforeyna.companionclient" +echo "Logs are available at:" +echo " STDOUT: /Users/adolforeyna/Projects/MicroPython/test1/Screen/desktop_client/stdout.log" +echo " STDERR: /Users/adolforeyna/Projects/MicroPython/test1/Screen/desktop_client/stderr.log" diff --git a/desktop_client/uninstall_service.sh b/desktop_client/uninstall_service.sh new file mode 100755 index 0000000..5181171 --- /dev/null +++ b/desktop_client/uninstall_service.sh @@ -0,0 +1,17 @@ +#!/bin/bash +PLIST_NAME="com.adolforeyna.companionclient.plist" +TARGET_PLIST="${HOME}/Library/LaunchAgents/${PLIST_NAME}" + +echo "=== Uninstalling Desktop Companion LaunchAgent ===" + +if [ -f "${TARGET_PLIST}" ]; then + echo "Stopping and unloading the service..." + launchctl bootout gui/$(id -u) "${TARGET_PLIST}" 2>/dev/null + launchctl unload "${TARGET_PLIST}" 2>/dev/null + + echo "Removing plist configuration..." + rm "${TARGET_PLIST}" + echo "Service successfully uninstalled!" +else + echo "Service is not installed." +fi diff --git a/ft6336u.py b/ft6336u.py index 9122bfd..9385e5c 100644 --- a/ft6336u.py +++ b/ft6336u.py @@ -78,8 +78,18 @@ class FT6336U: return True def is_touched(self): - """Returns True if screen is touched (the active-low INT pin is pulled LOW).""" - return self.int() == 0 + """Returns True if screen is touched by checking the TD_STATUS register and clearing coordinates.""" + if not self.initialized: + return False + td_status = self.read_reg(self.REG_TD_STATUS) + if td_status is None: + return False + touch_count = td_status[0] & 0x0F + if touch_count > 0 and touch_count < 3: + # Read P1 coordinates to clear register/interrupt state on the chip + self.read_reg(self.REG_P1_XH, 6) + return True + return False def read_touch(self): """Reads touch point coordinates. diff --git a/ili9341.py b/ili9341.py new file mode 100644 index 0000000..18e9f32 --- /dev/null +++ b/ili9341.py @@ -0,0 +1,261 @@ +import time +from machine import Pin, SPI +import framebuf +import micropython + +class ILI9341: + def __init__(self, spi, cs, dc, rst=None, bl=None, width=320, height=240, invert_color=True): + self.spi = spi + self.cs = cs + self.dc = dc + self.rst = rst + self.bl = bl + self.width = width + self.height = height + self.invert_color = invert_color + + # 1. 1-bit Canvas Buffer (Standard MONO_HLSB for drawing) + self.hw_len = (self.width * self.height) // 8 + self.canvas_buffer = bytearray(self.hw_len) + self.canvas = framebuf.FrameBuffer(self.canvas_buffer, self.width, self.height, framebuf.MONO_HLSB) + + # Pre-allocate chunk buffer for conversion (16 rows: 320 * 16 * 2 = 10240 bytes) + self.chunk_rows = 16 + self.row_buffer = bytearray(self.width * self.chunk_rows * 2) + + # Initialize pins + self.cs.init(self.cs.OUT, value=1) + self.dc.init(self.dc.OUT, value=0) + + if self.rst is not None: + self.rst.init(self.rst.OUT, value=1) + + if self.bl is not None: + self.bl.init(self.bl.OUT, value=1) + + self.reset() + self.init_display() + self.clear(0) + self.show() + + # --- DRAWING WRAPPERS --- + def pixel(self, x, y, c): self.canvas.pixel(x, y, c) + def line(self, x1, y1, x2, y2, c): self.canvas.line(x1, y1, x2, y2, c) + def rect(self, x, y, w, h, c): self.canvas.rect(x, y, w, h, c) + def fill_rect(self, x, y, w, h, c): self.canvas.fill_rect(x, y, w, h, c) + def text(self, msg, x, y, c=1): self.canvas.text(msg, x, y, c) + def clear(self, c=0): self.canvas.fill(c) + + # --- SCALABLE TEXT --- + def text_large(self, msg, x, y, scale=2, c=1): + char_w = 8; char_h = 8 + tmp_buf = bytearray(char_w * char_h // 8) + tmp_fb = framebuf.FrameBuffer(tmp_buf, char_w, char_h, framebuf.MONO_HLSB) + for char in msg: + tmp_fb.fill(0); tmp_fb.text(char, 0, 0, 1) + for py in range(8): + for px in range(8): + if tmp_fb.pixel(px, py): + self.canvas.fill_rect(x + (px * scale), y + (py * scale), scale, scale, c) + x += (8 * scale) + + # --- RAW BITMAPS (1:1 scale) --- + def bitmap(self, x, y, w, h, pixel_data): + img = framebuf.FrameBuffer(pixel_data, w, h, framebuf.MONO_HLSB) + self.canvas.blit(img, x, y) + + # --- PBM FILE LOADER WITH SCALING --- + def draw_pbm(self, filename, x, y, scale=1): + try: + with open(filename, 'rb') as f: + line1 = f.readline() + if not line1.startswith(b'P4'): print("Err: Not P4 PBM"); return + while True: + line = f.readline() + if not line.startswith(b'#'): break + dims = line.split(); w = int(dims[0]); h = int(dims[1]) + data = bytearray(f.read()) + + src_fb = framebuf.FrameBuffer(data, w, h, framebuf.MONO_HLSB) + + if scale == 1: + self.canvas.blit(src_fb, x, y) + else: + for sy in range(h): + for sx in range(w): + if src_fb.pixel(sx, sy): + self.canvas.fill_rect(x + (sx * scale), y + (sy * scale), scale, scale, 1) + print(f"Loaded {filename} (scale {scale})") + except OSError: + print(f"Error: Could not open {filename}") + + # --- SCREENSHOT --- + def save_screenshot(self, filename): + print(f"Saving screenshot to {filename}...") + try: + with open(filename, 'wb') as f: + f.write(b'P4\n') + f.write(f"{self.width} {self.height}\n".encode()) + f.write(self.canvas_buffer) + print("Saved!") + except Exception as e: + print(f"Error saving screenshot: {e}") + + # --- HARDWARE LOGIC --- + def reset(self): + if self.rst is not None: + self.rst(1); time.sleep_ms(5); self.rst(0); time.sleep_ms(15); self.rst(1); time.sleep_ms(15) + + def write_cmd(self, cmd): + self.dc(0); self.cs(0); self.spi.write(bytearray([cmd])); self.cs(1) + + def write_data(self, data): + self.dc(1); self.cs(0) + if isinstance(data, int): self.spi.write(bytearray([data])) + elif isinstance(data, list): self.spi.write(bytearray(data)) + else: self.spi.write(data) + self.cs(1) + + def init_display(self): + # ILI9341 Initialization Sequence + self.write_cmd(0x01) # SWRESET + time.sleep_ms(150) + + self.write_cmd(0xCF); self.write_data(b"\x00\xC1\x30") + self.write_cmd(0xED); self.write_data(b"\x64\x03\x12\x81") + self.write_cmd(0xE8); self.write_data(b"\x85\x00\x78") + self.write_cmd(0xCB); self.write_data(b"\x39\x2C\x00\x34\x02") + self.write_cmd(0xF7); self.write_data(b"\x20") + self.write_cmd(0xEA); self.write_data(b"\x00\x00") + + self.write_cmd(0xC0); self.write_data(b"\x13") # Power Control 1 + self.write_cmd(0xC1); self.write_data(b"\x13") # Power Control 2 + self.write_cmd(0xC5); self.write_data(b"\x22\x35") # VCOM Control 1 + self.write_cmd(0xC7); self.write_data(b"\xBD") # VCOM Control 2 + + # Memory Access Control (MADCTL) = 0x68 (Landscape: MV=1, MX=1, MY=0, BGR color filter) + self.write_cmd(0x36); self.write_data(b"\x68") + + self.write_cmd(0xB6); self.write_data(b"\x0A\xA2") # Display Function Control + self.write_cmd(0x3A); self.write_data(b"\x55") # Pixel Format (COLMOD) = 16-bit RGB565 + self.write_cmd(0xF6); self.write_data(b"\x01\x30") + self.write_cmd(0xB1); self.write_data(b"\x00\x1B") # Frame Rate Control + self.write_cmd(0xF2); self.write_data(b"\x00") + self.write_cmd(0x26); self.write_data(b"\x01") # Gamma Curve + + self.write_cmd(0xE0); self.write_data(b"\x0F\x35\x31\x0B\x0E\x06\x49\xA7\x33\x07\x0F\x03\x0C\x0A\x00") # Positive Gamma Correction + self.write_cmd(0xE1); self.write_data(b"\x00\x0A\x0F\x04\x11\x08\x36\x58\x4D\x07\x10\x0C\x32\x34\x0F") # Negative Gamma Correction + + if self.invert_color: + self.write_cmd(0x21) # INVON + else: + self.write_cmd(0x20) # INVOFF + + self.write_cmd(0x11) # SLPOUT (Exit sleep mode) + time.sleep_ms(120) + self.write_cmd(0x29) # DISPON (Display on) + time.sleep_ms(10) + + def invert(self, enable): + if enable: + self.write_cmd(0x21) # INVON + else: + self.write_cmd(0x20) # INVOFF + + def set_window(self, x0, y0, x1, y1): + # Column Address Set (CASET) + self.write_cmd(0x2A) + self.write_data(bytearray([x0 >> 8, x0 & 0xFF, x1 >> 8, x1 & 0xFF])) + + # Row Address Set (RASET) + self.write_cmd(0x2B) + self.write_data(bytearray([y0 >> 8, y0 & 0xFF, y1 >> 8, y1 & 0xFF])) + + # Memory Write (RAMWR) + self.write_cmd(0x2C) + + @micropython.native + def _convert_rows(self, start_row, num_rows, row_buf): + """Converts 1-bit monochrome row segment to 16-bit RGB565 format. + Compiles block-wise bitwise operations at native speed. + """ + width = self.width + canvas_buf = self.canvas_buffer + idx = 0 + + for y in range(start_row, start_row + num_rows): + byte_offset = y * (width // 8) + for x_byte_idx in range(width // 8): + val = canvas_buf[byte_offset + x_byte_idx] + + # Unroll 8 bits for speed + # Bit 7 + if val & 0x80: + row_buf[idx] = 0xFF; row_buf[idx+1] = 0xFF + else: + row_buf[idx] = 0x00; row_buf[idx+1] = 0x00 + idx += 2 + + # Bit 6 + if val & 0x40: + row_buf[idx] = 0xFF; row_buf[idx+1] = 0xFF + else: + row_buf[idx] = 0x00; row_buf[idx+1] = 0x00 + idx += 2 + + # Bit 5 + if val & 0x20: + row_buf[idx] = 0xFF; row_buf[idx+1] = 0xFF + else: + row_buf[idx] = 0x00; row_buf[idx+1] = 0x00 + idx += 2 + + # Bit 4 + if val & 0x10: + row_buf[idx] = 0xFF; row_buf[idx+1] = 0xFF + else: + row_buf[idx] = 0x00; row_buf[idx+1] = 0x00 + idx += 2 + + # Bit 3 + if val & 0x08: + row_buf[idx] = 0xFF; row_buf[idx+1] = 0xFF + else: + row_buf[idx] = 0x00; row_buf[idx+1] = 0x00 + idx += 2 + + # Bit 2 + if val & 0x04: + row_buf[idx] = 0xFF; row_buf[idx+1] = 0xFF + else: + row_buf[idx] = 0x00; row_buf[idx+1] = 0x00 + idx += 2 + + # Bit 1 + if val & 0x02: + row_buf[idx] = 0xFF; row_buf[idx+1] = 0xFF + else: + row_buf[idx] = 0x00; row_buf[idx+1] = 0x00 + idx += 2 + + # Bit 0 + if val & 0x01: + row_buf[idx] = 0xFF; row_buf[idx+1] = 0xFF + else: + row_buf[idx] = 0x00; row_buf[idx+1] = 0x00 + idx += 2 + + def show(self): + """Refreshes the screen by writing the frame buffer segment-by-segment.""" + self.set_window(0, 0, self.width - 1, self.height - 1) + + self.dc(1) + self.cs(0) + + num_chunks = self.height // self.chunk_rows + for chunk in range(num_chunks): + start_row = chunk * self.chunk_rows + self._convert_rows(start_row, self.chunk_rows, self.row_buffer) + self.spi.write(self.row_buffer) + + self.cs(1) diff --git a/main.py b/main.py index 1a3fcfc..0246991 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,7 @@ import time import sys import machine -from machine import Pin, SPI, I2C +from machine import Pin, SPI, I2C, I2S try: import network @@ -12,7 +12,7 @@ except ImportError: if sys.platform == 'rp2': import st7796 as display_module else: - import rlcd as display_module + import ili9341 as display_module class DummyMCP: def __init__(self): @@ -36,6 +36,29 @@ from rgb_led_util import BoardLED from ble_util import BLEUART from mcp_server import MCPServer from video_stream import VideoStreamServer +from audio_util import ES8311 +import struct +import urequests + +def create_wav_header(data_size): + # Generates a 44-byte WAV header for 16kHz, 16-bit mono PCM + riff = b'RIFF' + file_size = data_size + 36 + wave = b'WAVE' + fmt = b'fmt ' + chunk_size = 16 + audio_format = 1 # PCM + channels = 1 # Mono + sample_rate = 16000 + bits_per_sample = 16 + byte_rate = sample_rate * channels * (bits_per_sample // 8) + block_align = channels * (bits_per_sample // 8) + data_label = b'data' + + return struct.pack('<4sI4s4sIHHIIHH4sI', + riff, file_size, wave, fmt, chunk_size, + audio_format, channels, sample_rate, byte_rate, + block_align, bits_per_sample, data_label, data_size) # LED mode options for manual cycling led_modes = [ @@ -57,7 +80,7 @@ def main(): i2c = None if sys.platform != 'rp2': try: - i2c = I2C(0, sda=Pin(13), scl=Pin(14)) + i2c = I2C(0, sda=Pin(16), scl=Pin(15)) except Exception as e: print("Failed to initialize I2C0:", e) @@ -74,13 +97,18 @@ def main(): except Exception as te: print("Failed to initialize touch:", te) else: - spi = SPI(1, baudrate=20000000, polarity=0, phase=0, sck=Pin(11), mosi=Pin(12)) - display = display_module.RLCD(spi, cs=Pin(40), dc=Pin(5), rst=Pin(41)) + spi = SPI(1, baudrate=40000000, polarity=0, phase=0, sck=Pin(12), mosi=Pin(11), miso=Pin(13)) + display = display_module.ILI9341(spi, cs=Pin(10), dc=Pin(46), bl=Pin(45), rst=None) touch = None + try: + from ft6336u import FT6336U + touch = FT6336U(i2c, rst_pin=18, int_pin=17, width=320, height=240, swap_xy=True, invert_x=False, invert_y=True) + except Exception as te: + print("Failed to initialize touch:", te) if sys.platform != 'rp2': - # Configure Audio Amp control pin to save power - amp_pin = Pin(46, Pin.OUT, value=0) + # Configure Audio Amp control pin (GPIO 1, Active Low) to save power + amp_pin = Pin(1, Pin.OUT, value=1) # 2. Initialize utility objects sensor = None @@ -100,7 +128,7 @@ def main(): buttons = None if sys.platform != 'rp2': buttons = BoardButtons() - ble_uart = BLEUART(name="ESP32-S3-RLCD") + ble_uart = BLEUART(name="ESP32-S3-TFT") # Sync system clock from RTC chip if rtc_chip: @@ -126,7 +154,7 @@ def main(): # 4b. Start MCP Server from mcp_server import MCPServer - mcp = MCPServer(display, led, battery, sensor, rtc_chip, ble_uart, vstream=vstream) + mcp = MCPServer(display, led, battery, sensor, rtc_chip, ble_uart, vstream=vstream, touch=touch) mcp.start(port=80) except Exception as ne: print("Failed to start network services:", ne) @@ -169,6 +197,7 @@ def main(): last_dashboard_update = 0 dashboard_update_interval_ms = 5000 last_led_update = 0 + last_wifi_check = 0 print("ESP32 MCP loop running...") @@ -176,16 +205,227 @@ def main(): while True: now = time.ticks_ms() - # Check touch interaction if available - if touch and touch.is_touched(): - pt = touch.read_touch() - if pt: - tx, ty = pt - print(f"Touch detected at: ({tx}, {ty})") - last_action_str = f"Touch: ({tx}, {ty})" - mcp.override_active = False + # A. Check Wi-Fi status periodically (every 10 seconds) and auto-reconnect + if has_network and time.ticks_diff(now, last_wifi_check) >= 10000: + last_wifi_check = now + if not wlan.isconnected(): + print("Wi-Fi connection lost. Attempting reconnect...") + last_action_str = "Wi-Fi Disconnected" + if ip_addr != "Disconnected": + ip_addr = "Disconnected" + force_dashboard_redraw = True + try: + wlan.connect(wifi_config.WIFI_SSID, wifi_config.WIFI_PASS) + except Exception as e: + print("Wi-Fi reconnect trigger failed:", e) + elif ip_addr == "Disconnected" or ip_addr == "Offline (USB)": + ip_addr = wlan.ifconfig()[0] + print(f"Wi-Fi Connected! IP Address: {ip_addr}") + last_action_str = f"Wi-Fi Connected: {ip_addr}" force_dashboard_redraw = True + # Check touch interaction if available (Push-to-talk Voice Assistant) + if touch and touch.is_touched(): + print("Touch detected! Starting Hermes Voice Assistant...") + + def draw_status_bar(text): + display.line(0, 215, 320, 215, 1) + display.fill_rect(0, 216, 320, 24, 0) + display.text(text, 10, 222, 1) + display.show() + + def clear_status_bar(): + display.fill_rect(0, 215, 320, 25, 0) + display.show() + + # State 1: Wait for user to release their finger from the initial touch + draw_status_bar("Release finger...") + + release_start = time.ticks_ms() + while touch.is_touched(): + if time.ticks_diff(time.ticks_ms(), release_start) > 2000: + print("Release timeout occurred") + break + time.sleep_ms(30) + + # State 2: Wait for tap-to-start recording + draw_status_bar("Ready. Tap screen to record...") + + tap_started = False + start_wait = time.ticks_ms() + while time.ticks_diff(time.ticks_ms(), start_wait) < 10000: # 10s timeout to start recording + if touch.is_touched(): + tap_started = True + break + time.sleep_ms(30) + + if tap_started: + # User tapped the screen. Let's record! + draw_status_bar("Recording: 15s (Tap to stop)") + + # 1. Start MCLK PWM and configure ES8311 mic path + mclk_pin = Pin(4, Pin.OUT) + mclk_pwm = machine.PWM(mclk_pin) + mclk_pwm.freq(6144000) + mclk_pwm.duty_u16(32768) + + codec = ES8311(i2c) + if codec.init(sample_rate=16000): + codec.set_volume(80) # Set a balanced volume to prevent speaker saturation + try: + codec._write(0x14, 0x1A) # Enable analog mic input & PGA + codec._write(0x16, 0x01) # Enable +6dB gain boost for mic recording + codec._write(0x17, 0xC8) # Set ADC digital volume + except Exception as e: + print("Failed to set mic gain:", e) + + # 2. Configure I2S RX for recording (Mono 16kHz) + i2s_rx = I2S(1, + sck=Pin(5), + ws=Pin(7), + sd=Pin(6), + mode=I2S.RX, + ibuf=8000, + rate=16000, + bits=16, + format=I2S.MONO) + + audio_chunks = [] + total_data_bytes = 0 + buffer = bytearray(1024) + + # Track release of the start-tap first + start_tap_released = False + rec_start_time = time.ticks_ms() + max_rec_duration_ms = 15000 # 15 seconds max duration + max_rec_bytes = 480000 # 15 seconds of mono 16kHz 16-bit PCM (32KB/s) + last_sec_left = -1 + + try: + while True: + # Safety: check duration and byte size + elapsed = time.ticks_diff(time.ticks_ms(), rec_start_time) + if elapsed >= max_rec_duration_ms or total_data_bytes >= max_rec_bytes: + print("Recording stopped: maximum duration/size reached") + break + + # Update countdown + sec_left = max(0, 15 - (elapsed // 1000)) + if sec_left != last_sec_left: + last_sec_left = sec_left + draw_status_bar(f"Recording: {sec_left}s (Tap to stop)") + + # Read I2S chunk + bytes_read = i2s_rx.readinto(buffer) + if bytes_read > 0: + audio_chunks.append(bytes(buffer[:bytes_read])) + total_data_bytes += bytes_read + + # Check touch state + touched = touch.is_touched() + if not start_tap_released: + if not touched: + start_tap_released = True + print("Start-tap released. Listening for stop-tap...") + else: + if touched: + print("Stop-tap detected. Stopping recording...") + break + except Exception as e: + print("Error recording:", e) + finally: + i2s_rx.deinit() + try: + codec._write(0x16, 0x00) # Reset mic gain back to 0dB immediately to prevent playback saturation + except: + pass + + if total_data_bytes >= 1000: + draw_status_bar("Sending & processing...") + + # Prepend WAV header + raw_pcm = b"".join(audio_chunks) + wav_data = create_wav_header(len(raw_pcm)) + raw_pcm + + # Headers + headers = { + "Authorization": "Bearer mcT1YA1vOr9wXSiHpCYalweEGGZKX-PIfZv2drp8BSg", + "Content-Type": "audio/wav", + "X-Device-ID": "kitchen-button" + } + + print(f"Posting WAV ({len(wav_data)} bytes) to Hermes API...") + try: + res = urequests.post("http://192.168.68.126:8642/api/esp32/voice", headers=headers, data=wav_data) + print(f"HTTP Status: {res.status_code}") + + if res.status_code == 200: + response_data = res.content + print(f"Received response: {len(response_data)} bytes") + + if response_data[0:4] == b'RIFF' and response_data[8:12] == b'WAVE': + idx = response_data.find(b'fmt ') + if idx != -1: + fmt_chunk = response_data[idx:idx+24] + audio_format, channels, sample_rate, byte_rate, block_align, bits = struct.unpack(' 2000: + break + time.sleep_ms(30) + time.sleep_ms(200) + + last_action_str = "Voice query finished" + force_dashboard_redraw = True + # A. Handle non-blocking MCP client connection updates mcp.update() @@ -214,31 +454,46 @@ def main(): # Draw standard status dashboard layout display.clear(0) - title_text = "RP2350-TFT MCP SERVER" if sys.platform == 'rp2' else "ESP32-S3-RLCD MCP SERVER" - line_w = 470 if sys.platform == 'rp2' else 390 + title_text = "RP2350-TFT MCP SERVER" if sys.platform == 'rp2' else "Hosyond ESP32-S3 Server" + line_w = 470 if sys.platform == 'rp2' else 310 - display.text(title_text, 10, 10, 1) - display.line(10, 20, line_w, 20, 1) - - display.text_large("ENVIRONMENT", 15, 30, scale=2, c=1) - display.text(f"Temp : {t_str}", 25, 55, 1) - display.text(f"Humid : {h_str}", 25, 70, 1) - - display.line(10, 95, line_w, 95, 1) - - display.text_large("MCP NET CONNECTION", 15, 105, scale=2, c=1) - display.text(f"IP Address : {ip_addr}", 25, 130, 1) - display.text(f"Port / Path : 80 /api/mcp", 25, 145, 1) - display.text(f"BLE Name : ESP32-S3-RLCD", 25, 160, 1) - - display.line(10, 185, line_w, 185, 1) - - display.text_large("SYSTEM STATUS", 15, 195, scale=2, c=1) - display.text(f"Battery : {bat_str}", 25, 220, 1) - display.text(f"Time : {time_str}", 25, 235, 1) - - display.line(10, 255, line_w, 255, 1) - display.text(f"Status: {last_action_str}", 15, 265, 1) + if sys.platform == 'rp2': + display.text(title_text, 10, 10, 1) + display.line(10, 20, line_w, 20, 1) + display.text_large("ENVIRONMENT", 15, 30, scale=2, c=1) + display.text(f"Temp : {t_str}", 25, 55, 1) + display.text(f"Humid : {h_str}", 25, 70, 1) + display.line(10, 95, line_w, 95, 1) + display.text_large("MCP NET CONNECTION", 15, 105, scale=2, c=1) + display.text(f"IP Address : {ip_addr}", 25, 130, 1) + display.text(f"Port / Path : 80 /api/mcp", 25, 145, 1) + display.text(f"BLE Name : ESP32-S3-RLCD", 25, 160, 1) + display.line(10, 185, line_w, 185, 1) + display.text_large("SYSTEM STATUS", 15, 195, scale=2, c=1) + display.text(f"Battery : {bat_str}", 25, 220, 1) + display.text(f"Time : {time_str}", 25, 235, 1) + display.line(10, 255, line_w, 255, 1) + display.text(f"Status: {last_action_str}", 15, 265, 1) + else: + display.text(title_text, 10, 8, 1) + display.line(10, 18, line_w, 18, 1) + # Left Column (System & Environment) + display.text("SYSTEM & ENV", 10, 28, 1) + display.line(10, 38, 150, 38, 1) + display.text(f"Temp : {t_str}", 10, 46, 1) + display.text(f"Hum : {h_str}", 10, 58, 1) + display.text(f"Bat : {bat_str}", 10, 70, 1) + display.text(f"Time : {time_str[11:19]}", 10, 82, 1) + display.text(f"Date : {time_str[0:10]}", 10, 94, 1) + # Right Column (Network) + display.text("MCP NETWORK", 170, 28, 1) + display.line(170, 38, line_w, 38, 1) + display.text(f"IP : {ip_addr}", 170, 46, 1) + display.text("Port: 80/api/mcp", 170, 58, 1) + display.text("BLE : S3-TFT", 170, 70, 1) + # Bottom Status + display.line(10, 115, line_w, 115, 1) + display.text(f"Status: {last_action_str}", 10, 125, 1) display.show() # D. Update NeoPixel animation smoothly (runs every 50ms) diff --git a/mcp_server.py b/mcp_server.py index c852eee..22c03d1 100644 --- a/mcp_server.py +++ b/mcp_server.py @@ -5,7 +5,7 @@ import time class MCPServer: """A lightweight JSON-RPC HTTP server implementing Model Context Protocol (MCP) endpoints.""" - def __init__(self, display, led, battery, sensor, rtc, ble, vstream=None): + def __init__(self, display, led, battery, sensor, rtc, ble, vstream=None, touch=None): self.display = display self.led = led self.battery = battery @@ -13,6 +13,7 @@ class MCPServer: self.rtc = rtc self.ble = ble self.vstream = vstream + self.touch = touch self.sock = None self.active_led_mode = "off" # static, breath, rainbow, off @@ -20,6 +21,7 @@ class MCPServer: def start(self, port=80): """Starts the TCP server non-blockingly.""" + self.port = port self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.sock.bind(('', port)) @@ -39,6 +41,27 @@ class MCPServer: print("Failed to bind UDP Discovery socket: {}".format(ue)) self.udp_sock = None + def restart(self): + """Re-initializes the listening sockets after a fatal error.""" + print("Restarting MCP Server sockets...") + try: + if self.sock: + self.sock.close() + except: + pass + self.sock = None + try: + if hasattr(self, 'udp_sock') and self.udp_sock: + self.udp_sock.close() + except: + pass + self.udp_sock = None + + # Add a short delay to allow socket reuse to settle + time.sleep_ms(100) + port = getattr(self, 'port', 80) + self.start(port) + def update(self): """Check for incoming HTTP requests and handle them non-blockingly.""" # 1. Handle UDP Discovery queries @@ -56,8 +79,18 @@ class MCPServer: try: client, addr = self.sock.accept() - except OSError: - # No incoming connection + except OSError as e: + import errno + err = getattr(e, 'errno', None) + if err is None and e.args: + err = e.args[0] + ewouldblock = getattr(errno, 'EWOULDBLOCK', errno.EAGAIN) + if err in (errno.EAGAIN, ewouldblock) or err is None: + # No incoming connection + return + # Fatal socket error + print(f"Fatal socket error in MCP Server accept(): {e}. Re-initializing...") + self.restart() return try: @@ -100,14 +133,21 @@ class MCPServer: rpc_req = json.loads(body) rpc_resp = self._handle_rpc(rpc_req) - # Send HTTP Response resp_body = json.dumps(rpc_resp) resp = "HTTP/1.1 200 OK\r\n" resp += "Content-Type: application/json\r\n" resp += f"Content-Length: {len(resp_body)}\r\n" resp += "Connection: close\r\n\r\n" resp += resp_body - client.send(resp.encode('utf-8')) + + data_to_send = resp.encode('utf-8') + total_sent = 0 + while total_sent < len(data_to_send): + sent = client.write(data_to_send[total_sent:]) + if sent is None or sent == 0: + time.sleep_ms(10) + continue + total_sent += sent else: # Return 404 resp = "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" @@ -171,6 +211,11 @@ class MCPServer: "description": "Read the current battery voltage and estimated capacity percentage.", "inputSchema": {"type": "object", "properties": {}} }, + { + "name": "get_touch", + "description": "Read the current touch state and coordinates from the capacitive touchscreen.", + "inputSchema": {"type": "object", "properties": {}} + }, { "name": "get_sensors", "description": "Read onboard SHTC3 temperature and relative humidity.", @@ -404,6 +449,17 @@ class MCPServer: t, h = self.sensor.read_sensor() return json.dumps({"temperature_c": t, "humidity_pct": h}) + elif name == "get_touch": + if self.touch is None: + return json.dumps({"error": "Touchscreen not configured on this device."}) + is_t = self.touch.is_touched() + x, y = None, None + if is_t: + pt = self.touch.read_touch() + if pt: + x, y = pt + return json.dumps({"is_touched": is_t, "x": x, "y": y}) + elif name == "scan_ble": dur = int(args.get("duration_ms", 3000)) # Scan returns dictionary: {mac: {rssi: rssi, name: name}} diff --git a/rgb_led_util.py b/rgb_led_util.py index 95447e9..4b2bbc8 100644 --- a/rgb_led_util.py +++ b/rgb_led_util.py @@ -9,7 +9,7 @@ class BoardLED: def __init__(self, pin_num=None): import sys if pin_num is None: - pin_num = 14 if sys.platform == 'rp2' else 38 + pin_num = 14 if sys.platform == 'rp2' else 42 self.np = neopixel.NeoPixel(Pin(pin_num), 1) self.base_color = (0, 0, 0) self.off() diff --git a/scratch/recover.py b/scratch/recover.py new file mode 100644 index 0000000..07a534a --- /dev/null +++ b/scratch/recover.py @@ -0,0 +1,75 @@ +import json +import urllib.request +import time +import sys + +ESP32_IP = "192.168.68.123" +URL = f"http://{ESP32_IP}/api/mcp" + +def call_mcp_tool(tool_name, arguments): + payload = { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": tool_name, + "arguments": arguments + } + } + + req_data = json.dumps(payload).encode('utf-8') + req = urllib.request.Request( + URL, + data=req_data, + headers={"Content-Type": "application/json"}, + method="POST" + ) + + # Try multiple times for each call since the connection might reset + for attempt in range(1, 100): + try: + with urllib.request.urlopen(req, timeout=1.0) as response: + resp_body = response.read().decode('utf-8') + resp_json = json.loads(resp_body) + if "error" in resp_json: + return None + return resp_json.get("result", {}).get("content", [{}])[0].get("text", "") + except Exception as e: + time.sleep(0.01) + return None + +def upload_file(local_path, remote_path): + print(f"Reading local file {local_path}...") + with open(local_path, 'r') as f: + content = f.read() + + print(f"Uploading to ESP32: {remote_path}...") + for attempt in range(1, 20): + res = call_mcp_tool("write_file", {"path": remote_path, "content": content}) + if res: + print(f"Success uploading {remote_path} on attempt {attempt}!") + return True + print(f"Retry {attempt} uploading {remote_path}...") + time.sleep(0.1) + return False + +def main(): + print("=== ESP32 OTA Recovery Tool ===") + + # Upload mcp_server.py first + if not upload_file("mcp_server.py", "mcp_server.py"): + print("Failed to upload mcp_server.py. Exiting.") + sys.exit(1) + + # Upload video_stream.py + if not upload_file("video_stream.py", "video_stream.py"): + print("Failed to upload video_stream.py. Exiting.") + sys.exit(1) + + print("Rebooting the board...") + reboot_code = "import machine\nmachine.reset()" + call_mcp_tool("execute_python", {"code": reboot_code}) + print("Reboot command sent! The board should recover now.") + +if __name__ == "__main__": + main() diff --git a/test_stream.py b/test_stream.py index 0cf31d1..eb19660 100644 --- a/test_stream.py +++ b/test_stream.py @@ -80,6 +80,28 @@ def generate_animation_frame(frame_idx, mode_name): return img +def connect_socket(esp32_ip, port, protocol): + """Establishes connection to the ESP32 stream server, retrying on failure for TCP.""" + while True: + try: + if protocol == "tcp": + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((esp32_ip, port)) + print(f"Connected to stream server at {esp32_ip}:{port}") + return sock + else: # udp + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + if esp32_ip.endswith(".255") or esp32_ip == "255.255.255.255": + sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + return sock + except (socket.error, Exception) as e: + if protocol == "tcp": + print(f"Failed to connect to stream server: {e}. Retrying in 2 seconds...") + time.sleep(2.0) + else: + # UDP doesn't establish connection, so any error here is fatal + raise e + def stream_camera(esp32_ip, port, protocol): """Streams camera capture frames using OpenCV (requires opencv-python).""" try: @@ -94,16 +116,8 @@ def stream_camera(esp32_ip, port, protocol): print("Error: Could not open camera.") sys.exit(1) - sock = None - if protocol == "tcp": - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((esp32_ip, port)) - else: # udp - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - # Enable broadcast if destination is broadcast - if esp32_ip.endswith(".255") or esp32_ip == "255.255.255.255": - sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) - + sock = connect_socket(esp32_ip, port, protocol) + print(f"Streaming webcam to {esp32_ip}:{port} via {protocol.upper()}...") frame_count = 0 start_time = time.time() @@ -128,11 +142,19 @@ def stream_camera(esp32_ip, port, protocol): # Map pixels to hardware buffer raw_bytes = map_to_rlcd_hw_buffer(bg) - # Transmit - if protocol == "tcp": - sock.sendall(raw_bytes) - else: - send_udp_frame_chunked(sock, esp32_ip, port, frame_count, raw_bytes) + # Transmit with reconnect logic + try: + if protocol == "tcp": + sock.sendall(raw_bytes) + else: + send_udp_frame_chunked(sock, esp32_ip, port, frame_count, raw_bytes) + except (socket.error, Exception) as se: + print(f"Transmission failed: {se}. Reconnecting...") + try: + sock.close() + except: + pass + sock = connect_socket(esp32_ip, port, protocol) frame_count += 1 elapsed = time.time() - start_time @@ -151,14 +173,7 @@ def stream_camera(esp32_ip, port, protocol): def stream_animation(esp32_ip, port, protocol): """Streams a generated Pillow vector animation.""" - sock = None - if protocol == "tcp": - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((esp32_ip, port)) - else: # udp - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - if esp32_ip.endswith(".255") or esp32_ip == "255.255.255.255": - sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + sock = connect_socket(esp32_ip, port, protocol) print(f"Streaming vector animation to {esp32_ip}:{port} via {protocol.upper()}...") frame_idx = 0 @@ -172,11 +187,19 @@ def stream_animation(esp32_ip, port, protocol): # Map pixels to hardware buffer raw_bytes = map_to_rlcd_hw_buffer(pil_img) - # Transmit - if protocol == "tcp": - sock.sendall(raw_bytes) - else: - send_udp_frame_chunked(sock, esp32_ip, port, frame_idx, raw_bytes) + # Transmit with reconnect logic + try: + if protocol == "tcp": + sock.sendall(raw_bytes) + else: + send_udp_frame_chunked(sock, esp32_ip, port, frame_idx, raw_bytes) + except (socket.error, Exception) as se: + print(f"Transmission failed: {se}. Reconnecting...") + try: + sock.close() + except: + pass + sock = connect_socket(esp32_ip, port, protocol) frame_idx += 1 diff --git a/upload_files.py b/upload_files.py index 7b7c5ee..5254f64 100644 --- a/upload_files.py +++ b/upload_files.py @@ -3,7 +3,7 @@ import json import urllib.request import sys -ESP32_IP = "192.168.68.123" +ESP32_IP = "192.168.68.124" URL = f"http://{ESP32_IP}/api/mcp" def call_mcp_tool(tool_name, arguments): diff --git a/verify_streaming.py b/verify_streaming.py index 82762fc..1bd9aef 100644 --- a/verify_streaming.py +++ b/verify_streaming.py @@ -9,7 +9,7 @@ from PIL import Image ESP32_IP = "192.168.68.123" URL = f"http://{ESP32_IP}/api/mcp" -ARTIFACTS_DIR = "/Users/adolforeyna/.gemini/antigravity/brain/8b421e8b-10a8-4d06-a7d0-27bd82b42804" +ARTIFACTS_DIR = "/Users/adolforeyna/.gemini/antigravity/brain/da0e7ff3-d127-4865-bfea-f8ae9198bd40" def capture_screenshot(output_filename): print(f"Requesting screenshot from ESP32 ({ESP32_IP})...") diff --git a/video_stream.py b/video_stream.py index 49258dc..a78d441 100644 --- a/video_stream.py +++ b/video_stream.py @@ -136,6 +136,46 @@ class VideoStreamServer: except Exception as e: print(f"Failed to start UDP stream server: {e}") + def restart_tcp_server(self): + """Re-initializes the TCP stream server socket after a fatal error.""" + print("Restarting TCP Stream Server...") + self.close_tcp_client() + try: + if self.tcp_server: + self.tcp_server.close() + except: + pass + self.tcp_server = None + time.sleep_ms(100) + try: + self.tcp_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.tcp_server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.tcp_server.bind(('', self.tcp_port)) + self.tcp_server.listen(1) + self.tcp_server.setblocking(False) + print(f"TCP Stream Server restarted on port {self.tcp_port}") + except Exception as e: + print(f"Failed to restart TCP stream server: {e}") + + def restart_udp_sock(self): + """Re-initializes the UDP stream server socket after a fatal error.""" + print("Restarting UDP Stream Server...") + try: + if self.udp_sock: + self.udp_sock.close() + except: + pass + self.udp_sock = None + time.sleep_ms(100) + try: + self.udp_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.udp_sock.bind(('', self.udp_port)) + self.udp_sock.setblocking(False) + print(f"UDP Stream Server restarted on port {self.udp_port}") + except Exception as e: + print(f"Failed to restart UDP stream server: {e}") + def update(self): """Updates connection states and checks for incoming video data. Call this periodically in the main execution loop. @@ -195,7 +235,16 @@ class VideoStreamServer: self._draw_frame() self.chunks_received = 0 # Reset for next frame except OSError as e: - break # EWOULDBLOCK (no data available) + import errno + err = getattr(e, 'errno', None) + if err is None and e.args: + err = e.args[0] + ewouldblock = getattr(errno, 'EWOULDBLOCK', errno.EAGAIN) + if err in (errno.EAGAIN, ewouldblock) or err is None: + break # EAGAIN/EWOULDBLOCK (no data available) + print(f"Fatal UDP socket error in update(): {e}. Re-initializing UDP socket...") + self.restart_udp_sock() + break # 3. Handle TCP stream if self.tcp_server: @@ -208,8 +257,15 @@ class VideoStreamServer: self.active = True self.last_packet_time = now print(f"TCP Stream client connected from: {addr}") - except OSError: - pass + except OSError as e: + import errno + err = getattr(e, 'errno', None) + if err is None and e.args: + err = e.args[0] + ewouldblock = getattr(errno, 'EWOULDBLOCK', errno.EAGAIN) + if err not in (errno.EAGAIN, ewouldblock) and err is not None: + print(f"Fatal TCP stream server socket error in accept(): {e}. Re-initializing...") + self.restart_tcp_server() # If client is connected, read data if self.tcp_client is not None: