#!/usr/bin/env python3 import sys import os import json import socket import time import threading import math import subprocess import tempfile import struct import base64 import argparse import urllib.request import http.server import re from PIL import Image, ImageDraw, ImageFont, ImageOps # Try importing pygame, but allow headless run if missing try: import pygame PYGAME_AVAILABLE = True except ImportError: PYGAME_AVAILABLE = False # Port Configuration Defaults HTTP_PORT = 8080 TCP_STREAM_PORT = 8081 UDP_STREAM_PORT = 8082 UDP_DISCOVERY_PORT = 5000 # Constants for Reflective LCD (RLCD) Palette COLOR_BG_RGB = (180, 195, 174) # Reflective LCD light green-grey COLOR_FG_RGB = (26, 36, 22) # Dark green-black ink COLOR_FRAME_RGB = (15, 23, 42) # Dark bezel COLOR_WINDOW_RGB = (220, 220, 220) # Light grey window background class NativeCompanionClient: def __init__(self, width=480, height=320, http_port=HTTP_PORT): self.width = width self.height = height self.http_port = http_port # Thread lock for canvas buffer safety self.canvas_lock = threading.Lock() # Visual Buffer (1-bit monochrome PIL image: 0=bg/white, 1=fg/black) self.canvas_img = Image.new("1", (self.width, self.height), 0) self.draw = ImageDraw.Draw(self.canvas_img) # Video stream and performance stats self.fps_frame_count = 0 self.fps_start_time = time.time() self.current_fps = 0.0 self.tcp_bytes_received = 0 self.udp_packets_received = 0 self.udp_frames_complete = 0 self.dropped_udp_frames = 0 self.is_streaming_active = False self.last_stream_packet_time = 0 # LED state self.led_r = 0 self.led_g = 0 self.led_b = 0 self.led_mode = "off" # static, breath, rainbow, off # Audio / recording state self.audio_status = "Idle" self.mic_status = "Idle" # Start Time for Uptime self.start_time = time.time() self.override_active = False # Native stdout handler for stdio MCP self.mcp_stdout = sys.stdout # Redirect default prints to stderr so they don't break JSON-RPC over stdout sys.stdout = sys.stderr def get_screenshot_path(self, filename): """Returns a writeable path for saving screenshots, falling back to local folder if needed.""" mac_path = "/Users/adolforeyna/.gemini/antigravity/brain/4acf0d08-1256-4c7a-a2e4-3e8576b58c73" if os.path.exists(mac_path): return os.path.join(mac_path, filename) # Fallback for Linux / Pi 5 local_path = os.path.join(os.path.expanduser("~"), "Projects", "Screen") if os.path.exists(local_path): return os.path.join(local_path, filename) return os.path.join(os.getcwd(), filename) def bring_to_front(self): """Focuses/raises the Pygame window on macOS using System Events.""" if sys.platform == "darwin": try: cmd = """osascript -e 'tell application "System Events" to set frontmost of every process whose name is "Python" to true'""" subprocess.Popen(cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except Exception: pass def draw_local_dashboard_canvas(self, battery_text, temp_text, uptime_text): """Draws the companion dashboard onto the 1-bit PIL canvas image.""" self.draw.rectangle([0, 0, self.width, self.height], fill=0) font_large = ImageFont.load_default() font_normal = ImageFont.load_default() try: # Monospace look font_large = ImageFont.truetype("Courier New", 18) font_normal = ImageFont.truetype("Courier New", 12) except: pass # Draw header self.draw.text((15, 10), "DESKTOP MCP COMPANION", fill=1, font=font_large) self.draw.line((10, 32, 470, 32), fill=1) # Sections self.draw.text((15, 45), "HOST TELEMETRY", fill=1, font=font_large) t_val = temp_text.split(": ")[1] if ": " in temp_text else temp_text self.draw.text((25, 70), f"CPU Temp : {t_val}", fill=1, font=font_normal) u_val = uptime_text.split(": ")[1] if ": " in uptime_text else uptime_text self.draw.text((25, 85), f"Uptime : {u_val}", fill=1, font=font_normal) self.draw.line((10, 110, 470, 110), fill=1) self.draw.text((15, 120), "MCP ENDPOINTS", fill=1, font=font_large) local_ip = self.get_local_ip() self.draw.text((25, 145), f"IP Address : {local_ip}", fill=1, font=font_normal) self.draw.text((25, 160), f"HTTP API : {self.http_port} /api/mcp", fill=1, font=font_normal) self.draw.text((25, 175), f"Video Ports: 8081 (TCP) / 8082 (UDP)", fill=1, font=font_normal) self.draw.line((10, 200, 470, 200), fill=1) self.draw.text((15, 210), "SYSTEM STATUS", fill=1, font=font_large) b_val = battery_text.split(": ")[1] if ": " in battery_text else battery_text self.draw.text((25, 235), f"Battery : {b_val}", fill=1, font=font_normal) time_str = time.strftime("%Y-%m-%d %H:%M:%S") self.draw.text((25, 250), f"Local Time : {time_str}", fill=1, font=font_normal) self.draw.line((10, 275, 470, 275), fill=1) status_msg = f"Speaker: {self.audio_status} | Mic: {self.mic_status}" self.draw.text((15, 290), f"Status : {status_msg}", fill=1, font=font_normal) def run_pygame_loop(self): """Starts and runs the Pygame GUI main thread loop.""" pygame.init() pygame.font.init() # Display resolution (960x640 canvas with bezel = 1000x680 window) screen_w = 1000 screen_h = 680 # Start hidden initially as requested self.window_visible = False self.show_window_requested = False screen = pygame.display.set_mode((screen_w, screen_h), pygame.HIDDEN) pygame.display.set_caption("Native Agent Companion Client") # Use default system font or built-in font font = pygame.font.Font(None, 22) title_font = pygame.font.Font(None, 28) clock = pygame.time.Clock() # Internal stats query timers last_telemetry_time = 0 battery_text = "Battery: Querying..." temp_text = "CPU Temperature: Querying..." uptime_text = "Uptime: 00:00:00" volts = 4.20 pct = 100 temp = 38.5 auto_screenshot_done = False running = True while running: # Check if a message was received and we need to make the window visible if getattr(self, "show_window_requested", False): self.show_window_requested = False if not self.window_visible: # Switch display to SHOWN mode screen = pygame.display.set_mode((screen_w, screen_h), pygame.SHOWN) self.window_visible = True self.bring_to_front() # 1. Process Pygame Window Events for event in pygame.event.get(): if event.type == pygame.QUIT: running = False elif event.type == pygame.KEYDOWN: if event.key == pygame.K_d: self.override_active = False # Force immediately uptime_sec = int(time.time() - self.start_time) hrs = uptime_sec // 3600 mins = (uptime_sec % 3600) // 60 secs = uptime_sec % 60 uptime_text = f"Uptime: {hrs:02d}:{mins:02d}:{secs:02d}" volts, pct = self.get_host_battery() battery_text = f"Battery: {volts:.2f}V ({pct}%)" temp = self.get_host_cpu_temp() temp_text = f"CPU Temperature: {temp:.1f}°C" with self.canvas_lock: self.draw_local_dashboard_canvas(battery_text, temp_text, uptime_text) elif event.key == pygame.K_s: try: save_path = self.get_screenshot_path("pygame_screen.png") pygame.image.save(screen, save_path) sys.stderr.write(f"Manual screen capture saved to {save_path}.\n") sys.stderr.flush() except Exception as ex: sys.stderr.write(f"Manual screen capture failed: {ex}\n") sys.stderr.flush() now = time.time() # 2. Update telemetry stats once per second if now - last_telemetry_time >= 1.0: last_telemetry_time = now uptime_sec = int(now - self.start_time) hrs = uptime_sec // 3600 mins = (uptime_sec % 3600) // 60 secs = uptime_sec % 60 uptime_text = f"Uptime: {hrs:02d}:{mins:02d}:{secs:02d}" volts, pct = self.get_host_battery() battery_text = f"Battery: {volts:.2f}V ({pct}%)" temp = self.get_host_cpu_temp() temp_text = f"CPU Temperature: {temp:.1f}°C" # Check video streaming inactivity if self.is_streaming_active and (now - self.last_stream_packet_time) > 3.0: self.is_streaming_active = False self.current_fps = 0.0 # Update dashboard on canvas if not overridden if not self.override_active and not self.is_streaming_active: with self.canvas_lock: self.draw_local_dashboard_canvas(battery_text, temp_text, uptime_text) # 3. Clear window with dark background screen.fill(COLOR_WINDOW_RGB) # 4. Draw Reflective LCD Screen Bezel Frame # Left side bezel box (width 964, height 644) at (15, 15) pygame.draw.rect(screen, COLOR_FRAME_RGB, (15, 15, 970, 650), border_radius=12) # 5. Render LCD Canvas Buffer try: # Thread-safe copy of canvas pixels using fast colorization with self.canvas_lock: gray_img = self.canvas_img.convert("L") colored_img = ImageOps.colorize(gray_img, COLOR_BG_RGB, COLOR_FG_RGB) # Scale up to 2x (960x640) scaled_img = colored_img.resize((960, 640), Image.NEAREST) img_bytes = scaled_img.tobytes("raw", "RGB") # Blit to screen lcd_surface = pygame.image.frombytes(img_bytes, (960, 640), "RGB") screen.blit(lcd_surface, (20, 20)) except Exception as e: # Print rendering failures to stderr sys.stderr.write(f"GUI Canvas Render Fail: {e}\n") sys.stderr.flush() # Auto screenshot at 3 seconds after window becomes visible if self.window_visible and not auto_screenshot_done: if not hasattr(self, "visible_start_time"): self.visible_start_time = time.time() elif time.time() - self.visible_start_time >= 3.0: try: save_path = self.get_screenshot_path("pygame_screen_auto.png") pygame.image.save(screen, save_path) sys.stderr.write(f"Auto screen capture saved to {save_path}.\n") sys.stderr.flush() except Exception as ex: sys.stderr.write(f"Auto screen capture failed: {ex}\n") sys.stderr.flush() auto_screenshot_done = True # 6. Draw LED Indicator Ring (on the window border) led_color = (71, 85, 105) # Off (dark grey) if self.led_mode == "static": led_color = (self.led_r, self.led_g, self.led_b) elif self.led_mode == "breath": brightness = 0.25 + 0.75 * (math.sin(now * 4.0) + 1.0) / 2.0 led_color = (int(self.led_r * brightness), int(self.led_g * brightness), int(self.led_b * brightness)) elif self.led_mode == "rainbow": hue = (now * 0.5) % 1.0 led_color = self.hsl_to_rgb(hue, 1.0, 0.5) # Draw tiny status LED on the window frame border pygame.draw.circle(screen, led_color, (990, 30), 5) # 8. Render updates pygame.display.flip() clock.tick(30) # Lock to 30 FPS for rendering updates pygame.quit() os._exit(0) def hsl_to_rgb(self, h, s, l): def hue_2_rgb(p, q, t): if t < 0: t += 1 if t > 1: t -= 1 if t < 1/6: return p + (q - p) * 6 * t if t < 1/2: return q if t < 2/3: return p + (q - p) * (2/3 - t) * 6 return p if s == 0: r = g = b = l else: q = l * (1 + s) if l < 0.5 else l + s - l * s p = 2 * l - q r = hue_2_rgb(p, q, h + 1/3) g = hue_2_rgb(p, q, h) b = hue_2_rgb(p, q, h - 1/3) return int(r * 255), int(g * 255), int(b * 255) # --- HOST HARDWARE QUERIES --- def get_host_battery(self): """Native battery status extraction for macOS, Linux, and Windows.""" if sys.platform == "darwin": # macOS try: output = subprocess.check_output(["pmset", "-g", "batt"]).decode("utf-8") pct_match = re.search(r"(\d+)%", output) pct = int(pct_match.group(1)) if pct_match else 100 volts = round(3.5 + (pct / 100.0) * 0.7, 3) return volts, pct except Exception: pass elif sys.platform.startswith("linux"): # Linux try: pct = 100 # Check typical battery path cap_path = "/sys/class/power_supply/BAT0/capacity" if not os.path.exists(cap_path): cap_path = "/sys/class/power_supply/BAT1/capacity" if os.path.exists(cap_path): with open(cap_path, "r") as f: pct = int(f.read().strip()) vol_path = "/sys/class/power_supply/BAT0/voltage_now" if not os.path.exists(vol_path): vol_path = "/sys/class/power_supply/BAT1/voltage_now" volts = None if os.path.exists(vol_path): with open(vol_path, "r") as f: volts = int(f.read().strip()) / 1_000_000.0 # microvolts to volts if volts is None: volts = round(3.5 + (pct / 100.0) * 0.7, 3) return volts, pct except Exception: pass elif sys.platform == "win32": # Windows try: output = subprocess.check_output("wmic Path Win32_Battery Get EstimatedChargeRemaining", shell=True).decode("utf-8") pct = 100 numbers = re.findall(r"\d+", output) if numbers: pct = int(numbers[0]) volts = round(3.5 + (pct / 100.0) * 0.7, 3) return volts, pct except Exception: pass # Fallback return 4.20, 100 def get_host_cpu_temp(self): """Dynamic temperature estimate based on thermal zone or CPU load.""" if sys.platform.startswith("linux"): try: for path in ["/sys/class/thermal/thermal_zone0/temp", "/sys/class/thermal/thermal_zone1/temp"]: if os.path.exists(path): with open(path, "r") as f: return round(int(f.read().strip()) / 1000.0, 1) except Exception: pass # Dynamic fallback matching CPU load average try: if hasattr(os, "getloadavg"): load = os.getloadavg()[0] else: load = 0.15 temp = 36.0 + min(30.0, load * 8.0) return round(temp, 1) except Exception: return 38.5 def get_local_ip(self): s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) try: s.connect(('10.255.255.255', 1)) IP = s.getsockname()[0] except Exception: IP = '127.0.0.1' finally: s.close() return IP # --- AUDIO INPUT/OUTPUT ENGINES --- def play_wav_native(self, filepath, volume=50): """Launches native player asynchronously.""" self.audio_status = f"Playing WAV ({volume}%)" # Native Windows Sound module if sys.platform == "win32": try: import winsound def run_winsound(): try: winsound.PlaySound(filepath, winsound.SND_FILENAME) finally: self.audio_status = "Idle" threading.Thread(target=run_winsound, daemon=True).start() return True except Exception as e: sys.stderr.write(f"winsound failed: {e}\n") sys.stderr.flush() self.audio_status = "Error" return False # MacOS / Linux CLI launchers cmd = None if sys.platform == "darwin": # afplay supports volume scaling (0.0 to 1.0) vol_factor = volume / 100.0 cmd = ["afplay", "-v", str(vol_factor), filepath] elif sys.platform.startswith("linux"): vol_factor = volume / 100.0 for player in ["paplay", "pw-play", "aplay"]: try: subprocess.run(["which", player], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) if player == "paplay": pa_vol = int((volume / 100.0) * 65536) cmd = ["paplay", "--volume", str(pa_vol), filepath] elif player == "pw-play": cmd = ["pw-play", "--volume", str(vol_factor), filepath] else: cmd = ["aplay", filepath] break except subprocess.CalledProcessError: continue if cmd: def run_player(): try: subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except Exception as ex: sys.stderr.write(f"Audio command failed: {ex}\n") sys.stderr.flush() finally: self.audio_status = "Idle" threading.Thread(target=run_player, daemon=True).start() return True else: sys.stderr.write("No supported command line audio player found.\n") sys.stderr.flush() self.audio_status = "Idle" return False def generate_and_play_tone(self, freq, duration_ms, volume): self.audio_status = f"Playing {freq}Hz Tone" sample_rate = 16000 num_samples = int(sample_rate * (duration_ms / 1000.0)) volume_scale = int((volume / 100.0) * 32767) # Generate raw 16-bit PCM mono sine wave pcm_data = bytearray() for i in range(num_samples): val = int(volume_scale * math.sin(2 * math.pi * freq * i / sample_rate)) pcm_data.extend(struct.pack("> 3)] |= (1 << (7 - (cx & 7))) # bit 6 if val & 0x40: cx, cy = x_base + 1, y_base canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7))) # bit 5 if val & 0x20: cx, cy = x_base, y_base - 1 canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7))) # bit 4 if val & 0x10: cx, cy = x_base + 1, y_base - 1 canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7))) # bit 3 if val & 0x08: cx, cy = x_base, y_base - 2 canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7))) # bit 2 if val & 0x04: cx, cy = x_base + 1, y_base - 2 canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7))) # bit 1 if val & 0x02: cx, cy = x_base, y_base - 3 canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7))) # bit 0 if val & 0x01: cx, cy = x_base + 1, y_base - 3 canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7))) return canvas_buf def process_rlcd_frame(self, rlcd_frame_bytes): """Converts incoming raw RLCD bytes to PIL and triggers GUI refresh.""" self.override_active = True self.show_window_requested = True canvas_bytes = self.rlcd_to_mono(rlcd_frame_bytes) # Load directly into our Visual Buffer (thread-safe lock) with self.canvas_lock: img = Image.frombytes("1", (self.width, self.height), bytes(canvas_bytes)) self.canvas_img.paste(img) # Calculate dynamic FPS now = time.time() self.fps_frame_count += 1 elapsed = now - self.fps_start_time if elapsed >= 1.0: self.current_fps = self.fps_frame_count / elapsed self.fps_frame_count = 0 self.fps_start_time = now if not self.is_streaming_active: self.bring_to_front() self.is_streaming_active = True self.last_stream_packet_time = now # --- MCP SERVER LOGIC --- def handle_mcp_request(self, req): """Processes an incoming JSON-RPC dictionary and executes the matching tool.""" rpc_id = req.get("id") method = req.get("method") params = req.get("params", {}) if method == "initialize": return { "jsonrpc": "2.0", "id": rpc_id, "result": { "protocolVersion": "2024-11-05", "capabilities": {"tools": {}}, "serverInfo": { "name": "desktop-companion-client", "version": "1.0.0" } } } elif method == "tools/list": return { "jsonrpc": "2.0", "id": rpc_id, "result": { "tools": self.get_mcp_tools_list() } } elif method == "tools/call": tool_name = params.get("name") arguments = params.get("arguments", {}) try: res_content = self.execute_tool(tool_name, arguments) if isinstance(res_content, list): content = res_content else: content = [{"type": "text", "text": str(res_content)}] return { "jsonrpc": "2.0", "id": rpc_id, "result": {"content": content} } except Exception as e: return { "jsonrpc": "2.0", "id": rpc_id, "error": { "code": -32000, "message": str(e) } } return { "jsonrpc": "2.0", "id": rpc_id, "error": { "code": -32601, "message": f"Method {method} not found" } } def get_mcp_tools_list(self): return [ { "name": "clear_screen", "description": "Clear the 480x320 canvas screen to white (0) or black (1).", "inputSchema": { "type": "object", "properties": { "color": {"type": "integer", "enum": [0, 1], "description": "0 = White, 1 = Black"} }, "required": ["color"] } }, { "name": "draw_text", "description": "Draw text on the canvas screen at specified (x,y) coordinates.", "inputSchema": { "type": "object", "properties": { "text": {"type": "string", "description": "The message to display"}, "x": {"type": "integer", "description": "X coordinate (0-470)"}, "y": {"type": "integer", "description": "Y coordinate (0-310)"}, "size": {"type": "integer", "enum": [1, 2], "description": "Text scale (1=normal, 2=large)"} }, "required": ["text", "x", "y"] } }, { "name": "set_led", "description": "Control the visual RGB LED. Animates breathes/rainbows dynamically.", "inputSchema": { "type": "object", "properties": { "r": {"type": "integer", "minimum": 0, "maximum": 255, "description": "Red (0-255)"}, "g": {"type": "integer", "minimum": 0, "maximum": 255, "description": "Green (0-255)"}, "b": {"type": "integer", "minimum": 0, "maximum": 255, "description": "Blue (0-255)"}, "mode": {"type": "string", "enum": ["static", "breath", "rainbow", "off"], "description": "LED animation mode"} }, "required": ["r", "g", "b", "mode"] } }, { "name": "get_battery", "description": "Read the host computer's actual battery voltage and estimated capacity percentage.", "inputSchema": {"type": "object", "properties": {}} }, { "name": "get_sensors", "description": "Read telemetry values representing the host computer's CPU temperature and relative indoor humidity.", "inputSchema": {"type": "object", "properties": {}} }, { "name": "scan_ble", "description": "Scan for nearby Bluetooth Low Energy devices using the host controller.", "inputSchema": { "type": "object", "properties": { "duration_ms": {"type": "integer", "description": "Scan duration in milliseconds (default 3000)"} } } }, { "name": "get_screenshot", "description": "Capture the current LCD display canvas rendering as a PNG image.", "inputSchema": {"type": "object", "properties": {}} }, { "name": "draw_image", "description": "Draw an image (PNG, JPEG, GIF, BMP, etc.) on the canvas. The image is converted to 1-bit monochrome and placed on the coordinates.", "inputSchema": { "type": "object", "properties": { "image_base64": {"type": "string", "description": "Base64 encoded string of the source image file"}, "x": {"type": "integer", "description": "X coordinate (default 0)", "default": 0}, "y": {"type": "integer", "description": "Y coordinate (default 0)", "default": 0}, "dither": {"type": "boolean", "description": "Whether to dither (default true)", "default": True} }, "required": ["image_base64"] } }, { "name": "write_file", "description": "Write a text file directly to the client's storage folder.", "inputSchema": { "type": "object", "properties": { "path": {"type": "string", "description": "The destination file path (e.g. 'test.txt')"}, "content": {"type": "string", "description": "The content to write"} }, "required": ["path", "content"] } }, { "name": "read_file", "description": "Read a text file from the client's storage folder.", "inputSchema": { "type": "object", "properties": { "path": {"type": "string", "description": "The file path to read"} }, "required": ["path"] } }, { "name": "execute_python", "description": "Execute arbitrary Python code dynamically on the host machine. Returns prints and errors.", "inputSchema": { "type": "object", "properties": { "code": {"type": "string", "description": "The Python code to execute"} }, "required": ["code"] } }, { "name": "play_tone", "description": "Play a pure sine wave tone on the host computer's speakers.", "inputSchema": { "type": "object", "properties": { "frequency": {"type": "integer", "description": "Frequency in Hz (default 440)", "default": 440}, "duration_ms": {"type": "integer", "description": "Duration in milliseconds (default 1000)", "default": 1000}, "volume": {"type": "integer", "description": "Volume from 0 to 100 (default 50)", "default": 50} } } }, { "name": "play_audio", "description": "Play a WAV audio file on the host computer's speakers.", "inputSchema": { "type": "object", "properties": { "filename": {"type": "string", "description": "WAV file path to play"}, "volume": {"type": "integer", "description": "Volume from 0 to 100 (default 50)", "default": 50} }, "required": ["filename"] } }, { "name": "record_voice", "description": "Record voice command from the computer microphone to a WAV file.", "inputSchema": { "type": "object", "properties": { "duration_sec": {"type": "integer", "description": "Duration in seconds (default 4)", "default": 4}, "filename": {"type": "string", "description": "Destination WAV file (default 'recording.wav')", "default": "recording.wav"} } } }, { "name": "play_audio_base64", "description": "Play a base64 encoded WAV file directly on the host computer's speakers.", "inputSchema": { "type": "object", "properties": { "volume": {"type": "integer", "description": "Volume from 0 to 100 (default 50)", "default": 50}, "wav_base64": {"type": "string", "description": "Base64 encoded string of the WAV audio file"}, }, "required": [ "wav_base64" ] } }, { "name": "download_file", "description": "Download a file from a URL to the local storage.", "inputSchema": { "type": "object", "properties": { "filename": {"type": "string", "description": "The destination filename"}, "url": {"type": "string", "description": "The URL of the file to download"} }, "required": [ "url", "filename" ] } }, { "name": "get_video_streaming_instructions", "description": "Get instructions to stream video directly into the companion canvas.", "inputSchema": { "type": "object", "properties": { "protocol": {"type": "string", "enum": ["tcp", "udp", "both"], "description": "The protocol to query (default 'both')"} } } }, { "name": "get_stream_stats", "description": "Get real-time performance and frame-rate statistics for the video stream.", "inputSchema": {"type": "object", "properties": {}} } ] def execute_tool(self, name, args): """Local tool implementation executing on the host PC.""" if name == "clear_screen": color = int(args.get("color", 0)) self.override_active = True self.show_window_requested = True with self.canvas_lock: self.draw.rectangle([0, 0, self.width, self.height], fill=color) self.bring_to_front() return "Screen cleared." elif name == "draw_text": text = str(args.get("text", "")) x = int(args.get("x", 10)) y = int(args.get("y", 10)) size = int(args.get("size", 1)) self.override_active = True self.show_window_requested = True # Draw on PIL image using standard font size font_size = 12 if size == 1 else 24 font = ImageFont.load_default() try: # Try loading standard monospace font for neat look font = ImageFont.truetype("Courier New", font_size) except: try: font = ImageFont.truetype("Arial", font_size) except: pass with self.canvas_lock: self.draw.text((x, y), text, fill=1, font=font) self.bring_to_front() return f"Drew text '{text}' at ({x}, {y})." elif name == "set_led": r = int(args.get("r", 0)) g = int(args.get("g", 0)) b = int(args.get("b", 0)) mode = str(args.get("mode", "static")) self.led_r, self.led_g, self.led_b, self.led_mode = r, g, b, mode return f"LED status set to {mode} ({r}, {g}, {b})." elif name == "get_battery": volts, pct = self.get_host_battery() return json.dumps({"voltage_v": volts, "percentage_pct": pct}) elif name == "get_sensors": temp = self.get_host_cpu_temp() # Return indoor humidity mock return json.dumps({"temperature_c": temp, "humidity_pct": 45.0}) elif name == "scan_ble": dur = int(args.get("duration_ms", 3000)) return json.dumps(scan_ble_devices(dur)) elif name == "get_screenshot": # Capture the current 480x320 PIL visual buffer, save as PNG in memory, base64 encode import io bg_color = (180, 195, 174) fg_color = (26, 36, 22) with self.canvas_lock: gray_img = self.canvas_img.convert("L") colored_img = ImageOps.colorize(gray_img, bg_color, fg_color) buffered = io.BytesIO() colored_img.save(buffered, format="PNG") b64_png = base64.b64encode(buffered.getvalue()).decode("utf-8") return [ { "type": "image", "data": b64_png, "mimeType": "image/png" } ] elif name == "draw_image": pbm_b64 = args.get("pbm_base64") image_base64 = args.get("image_base64") x = int(args.get("x", 0)) y = int(args.get("y", 0)) self.override_active = True self.show_window_requested = True if pbm_b64: pbm_bytes = base64.b64decode(pbm_b64) import io img = Image.open(io.BytesIO(pbm_bytes)) with self.canvas_lock: self.canvas_img.paste(img, (x, y)) elif image_base64: if "," in image_base64: image_base64 = image_base64.split(",")[1] img_bytes = base64.b64decode(image_base64) import io img = Image.open(io.BytesIO(img_bytes)) dither = args.get("dither", True) img_1bit = img.convert("1", dither=Image.FLOYDSTEINBERG if dither else Image.NONE) with self.canvas_lock: self.canvas_img.paste(img_1bit, (x, y)) else: raise ValueError("Missing image data (pbm_base64 or image_base64)") self.bring_to_front() return "Image rendered successfully on canvas." elif name == "write_file": path = str(args.get("path")) content = str(args.get("content")) # Save locally with open(path, 'w') as f: f.write(content) return f"Wrote {len(content)} characters to local file '{path}'." elif name == "read_file": path = str(args.get("path")) with open(path, 'r') as f: return f.read() elif name == "execute_python": code = str(args.get("code")) import builtins import sys import io output_buffer = [] old_print = builtins.print def custom_print(*args, **kwargs): sep = kwargs.get('sep', ' ') end = kwargs.get('end', '\n') str_args = [str(arg) for arg in args] msg = sep.join(str_args) + end output_buffer.append(msg) builtins.print = custom_print error = None try: exec(code, globals()) except Exception as e: import traceback error_msg = traceback.format_exc() output_buffer.append(f"\nExecution Failed:\n{error_msg}") error = e finally: builtins.print = old_print output = "".join(output_buffer) if error: return output return f"Execution Succeeded. Console output:\n{output}" elif name == "play_tone": freq = int(args.get("frequency", 440)) duration = int(args.get("duration_ms", 1000)) vol = int(args.get("volume", 50)) self.generate_and_play_tone(freq, duration, vol) return f"Played synthesized tone at {freq}Hz for {duration}ms (vol={vol})." elif name == "play_audio": filename = str(args.get("filename")) vol = int(args.get("volume", 50)) success = self.play_wav_native(filename, vol) if success: return f"Playing audio file '{filename}'." else: raise RuntimeError(f"Failed to play audio file '{filename}'.") elif name == "record_voice": duration = int(args.get("duration_sec", 4)) filename = str(args.get("filename", "recording.wav")) success = self.record_voice(duration, filename) if success: return f"Voice recording active. Saving to '{filename}' ({duration} seconds)." else: raise RuntimeError("Failed to trigger audio recording.") elif name == "play_audio_base64": b64_data = args.get("wav_base64") vol = int(args.get("volume", 50)) if not b64_data: raise ValueError("Missing wav_base64 parameter.") audio_bytes = base64.b64decode(b64_data) temp_fd, temp_path = tempfile.mkstemp(suffix=".wav") with os.fdopen(temp_fd, "wb") as f: f.write(audio_bytes) success = self.play_wav_native(temp_path, vol) # Schedule file deletion threading.Timer(5.0, lambda: self.safe_delete(temp_path)).start() if success: return "Playing base64 decoded audio stream." else: raise RuntimeError("Failed to play audio stream.") elif name == "download_file": url = str(args.get("url")) filename = str(args.get("filename")) # Simple HTTP download urllib.request.urlretrieve(url, filename) return f"Successfully downloaded file to local storage: '{filename}'" elif name == "get_video_streaming_instructions": instructions = [ "### Native Client Companion Video Streaming Instructions", "Display Area: 400x300 (drawn centered on the 480x320 physical screen), 1-bit monochrome.", "To stream, connect to local network services:", "1. **TCP (Port 8081)**: Open a TCP socket connection and stream consecutive 15,000-byte frame blocks.", "2. **UDP (Port 8082)**: Stream 1,002-byte datagrams: byte 0 = frame_id, byte 1 = chunk_idx (0..14), bytes 2..1001 = payload." ] return "\n".join(instructions) elif name == "get_stream_stats": return json.dumps({ "frames_drawn": self.fps_frame_count, "tcp_bytes_received": self.tcp_bytes_received, "udp_packets_received": self.udp_packets_received, "udp_frames_complete": self.udp_frames_complete, "dropped_udp_frames": self.dropped_udp_frames, "last_fps": round(self.current_fps, 1), "active": self.is_streaming_active }) else: raise ValueError(f"Unknown tool: {name}") def scan_ble_devices(duration_ms): """Scans for nearby BLE tags. Uses Bleak if installed, falls back to simulated beacons.""" try: import asyncio from bleak import BleakScanner async def run_scan(): scanner = BleakScanner() devices = await scanner.discover(timeout=duration_ms/1000.0) result = {} for d in devices: result[d.address] = {"rssi": d.rssi, "name": d.name or "Unknown"} return result try: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) res = loop.run_until_complete(run_scan()) loop.close() return res except Exception: pass except ImportError: pass # Simulated Beacons Fallback import random mock_tags = [ {"mac": "C8:FD:19:30:A5:11", "name": "Desk Tracker Tag"}, {"mac": "E5:31:02:8C:F4:9A", "name": "Companion Keyring"}, {"mac": "A4:38:DE:1C:B0:02", "name": "BLE Temp Beacon"}, ] result = {} for tag in mock_tags: result[tag["mac"]] = {"rssi": random.randint(-85, -45), "name": tag["name"]} return result # --- NETWORK BACKGROUND SERVICES --- def run_http_server(app, port): """Runs a background HTTP server providing /api/mcp endpoint parity.""" class HTTPHandler(http.server.BaseHTTPRequestHandler): def log_message(self, format, *args): pass # Suppress logging to stdout def do_OPTIONS(self): self.send_response(204) self.send_header("Access-Control-Allow-Origin", "*") self.send_header("Access-Control-Allow-Methods", "POST, GET, OPTIONS") self.send_header("Access-Control-Allow-Headers", "Content-Type") self.end_headers() def do_POST(self): if self.path == "/api/mcp": content_length = int(self.headers.get('Content-Length', 0)) body = self.rfile.read(content_length).decode('utf-8') try: req_data = json.loads(body) resp_data = app.handle_mcp_request(req_data) body_bytes = json.dumps(resp_data).encode('utf-8') self.send_response(200) self.send_header("Content-Type", "application/json") self.send_header("Access-Control-Allow-Origin", "*") self.send_header("Content-Length", len(body_bytes)) self.end_headers() self.wfile.write(body_bytes) except Exception as e: self.send_response(500) self.end_headers() self.wfile.write(str(e).encode('utf-8')) else: self.send_response(404) self.end_headers() httpd = http.server.HTTPServer(('', port), HTTPHandler) sys.stderr.write(f"[HTTP Service] Listening on port {port}...\n") sys.stderr.flush() httpd.serve_forever() def run_tcp_video_server(app, port): """Listens on port 8081 for incoming raw 15,000-byte RLCD video frame streams.""" server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server.bind(('', port)) server.listen(1) sys.stderr.write(f"[TCP Video] Listening on port {port}...\n") sys.stderr.flush() while True: try: client, addr = server.accept() sys.stderr.write(f"[TCP Video] Connection accepted from {addr}\n") sys.stderr.flush() frame_buffer = bytearray(15000) view = memoryview(frame_buffer) bytes_received = 0 while True: remaining = 15000 - bytes_received n = client.recv_into(view[bytes_received : bytes_received + remaining]) if not n: break # disconnected bytes_received += n app.tcp_bytes_received += n if bytes_received == 15000: app.process_rlcd_frame(frame_buffer) bytes_received = 0 client.close() except Exception as e: sys.stderr.write(f"[TCP Video] Error: {e}\n") sys.stderr.flush() time.sleep(1) def run_udp_video_server(app, port): """Listens on port 8082 for fragmented RLCD UDP frame packages.""" sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.bind(('', port)) sys.stderr.write(f"[UDP Video] Listening on port {port}...\n") sys.stderr.flush() frame_buffer = bytearray(15000) current_frame_id = -1 chunks_mask = 0 temp_pack = bytearray(1002) while True: try: n, addr = sock.recvfrom_into(temp_pack) if n < 2: continue app.udp_packets_received += 1 frame_id = temp_pack[0] chunk_idx = temp_pack[1] if chunk_idx < 15: if frame_id != current_frame_id: if chunks_mask != 0: app.dropped_udp_frames += 1 current_frame_id = frame_id chunks_mask = 0 # Copy segment payload start_offset = chunk_idx * 1000 frame_buffer[start_offset : start_offset + 1000] = temp_pack[2:1002] chunks_mask |= (1 << chunk_idx) # Check if all 15 chunks (0 to 14) are in (0x7FFF) if chunks_mask == 0x7FFF: app.udp_frames_complete += 1 app.process_rlcd_frame(frame_buffer) chunks_mask = 0 except Exception as e: sys.stderr.write(f"[UDP Video] Error: {e}\n") sys.stderr.flush() time.sleep(0.5) def run_udp_discovery_responder(app, port, http_port): """Responds to discovery broadcast messages from clients like mcp_bridge.""" sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind(('', port)) sys.stderr.write(f"[Discovery responder] Listening on port {port}...\n") sys.stderr.flush() while True: try: data, addr = sock.recvfrom(128) if data == b"DISCOVER_SCREEN": # Respond with matching HTTP port configuration response = f"SCREEN_IP_{http_port}".encode('utf-8') sock.sendto(response, addr) except Exception as e: sys.stderr.write(f"[Discovery responder] Error: {e}\n") sys.stderr.flush() time.sleep(1) def run_stdio_mcp_server(app): """Reads JSON-RPC messages line-by-line from stdin, writing responses to stdout.""" sys.stderr.write("[MCP Stdio Server] Stdio listener active.\n") sys.stderr.flush() while True: try: line = sys.stdin.readline() if not line: break req = json.loads(line.strip()) resp = app.handle_mcp_request(req) # Write explicitly to raw stdout payload = json.dumps(resp) + "\n" app.mcp_stdout.write(payload) app.mcp_stdout.flush() except Exception as e: sys.stderr.write(f"[MCP Stdio] Error handling command: {e}\n") sys.stderr.flush() # --- MAIN --- def main(): parser = argparse.ArgumentParser(description="Native Desktop Companion Client") parser.add_argument("--port", type=int, default=HTTP_PORT, help="HTTP Server port (default 8080)") parser.add_argument("--width", type=int, default=480, help="Canvas width (default 480)") parser.add_argument("--height", type=int, default=320, help="Canvas height (default 320)") parser.add_argument("--headless", action="store_true", help="Run in headless mode without GUI") args = parser.parse_args() client = NativeCompanionClient(width=args.width, height=args.height, http_port=args.port) # Start background servers threading.Thread(target=run_http_server, args=(client, args.port), daemon=True).start() threading.Thread(target=run_tcp_video_server, args=(client, TCP_STREAM_PORT), daemon=True).start() threading.Thread(target=run_udp_video_server, args=(client, UDP_STREAM_PORT), daemon=True).start() threading.Thread(target=run_udp_discovery_responder, args=(client, UDP_DISCOVERY_PORT, args.port), daemon=True).start() threading.Thread(target=run_stdio_mcp_server, args=(client,), daemon=True).start() # Run Pygame GUI if not headless and Pygame is installed if args.headless or not PYGAME_AVAILABLE: if not PYGAME_AVAILABLE and not args.headless: sys.stderr.write("Warning: Pygame is not installed. Running in headless mode.\n") sys.stderr.flush() else: sys.stderr.write("Headless mode active. Running background services...\n") sys.stderr.flush() try: while True: time.sleep(3600) except KeyboardInterrupt: sys.stderr.write("Stopping background services.\n") sys.stderr.flush() os._exit(0) else: # Start Pygame loop on main thread client.run_pygame_loop() if __name__ == "__main__": main()