diff --git a/boot.py b/boot.py index 01bd078..0c89e31 100644 --- a/boot.py +++ b/boot.py @@ -21,6 +21,13 @@ def connect_wifi(): print("Display init failed in boot.py:", e) # Initialize Wi-Fi Station + try: + import network + network.hostname('esp32screen') + print("Hostname set to: {}".format(network.hostname())) + except Exception as he: + print("Failed to set hostname: {}".format(he)) + wlan = network.WLAN(network.STA_IF) wlan.active(True) diff --git a/mcp_bridge.py b/mcp_bridge.py index 1fd8648..7cf9b9f 100755 --- a/mcp_bridge.py +++ b/mcp_bridge.py @@ -4,13 +4,125 @@ import json import urllib.request import argparse +def discover_screen(): + import socket + sys.stderr.write("Searching for ESP32 Screen via UDP broadcast on port 5000...\n") + sys.stderr.flush() + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + s.settimeout(1.5) + + broadcast_ips = ['255.255.255.255', ''] + try: + hostname = socket.gethostname() + ip_list = socket.gethostbyname_ex(hostname)[2] + for ip in ip_list: + if not ip.startswith('127.'): + parts = ip.split('.') + if len(parts) == 4: + subnet_bcast = f"{parts[0]}.{parts[1]}.{parts[2]}.255" + if subnet_bcast not in broadcast_ips: + broadcast_ips.append(subnet_bcast) + except: + pass + + for target_ip in broadcast_ips: + try: + s.sendto(b"DISCOVER_SCREEN", (target_ip, 5000)) + except: + pass + + try: + data, addr = s.recvfrom(128) + if data == b"SCREEN_IP_80": + sys.stderr.write(f"Auto-discovered ESP32 Screen at IP: {addr[0]} via UDP\n") + sys.stderr.flush() + return addr[0] + except: + pass + finally: + s.close() + return None + +def scan_subnet(): + import socket + import concurrent.futures + + try: + hostname = socket.gethostname() + local_ips = [ip for ip in socket.gethostbyname_ex(hostname)[2] if not ip.startswith('127.')] + if not local_ips: + return None + local_ip = local_ips[0] + except Exception as e: + sys.stderr.write(f"Subnet lookup error: {e}\n") + sys.stderr.flush() + return None + + parts = local_ip.split('.') + if len(parts) == 4: + base_ip = f"{parts[0]}.{parts[1]}.{parts[2]}." + else: + return None + + sys.stderr.write(f"Scanning local subnet {base_ip}1 to {base_ip}254 on port 80...\n") + sys.stderr.flush() + + def check_ip(ip_suffix): + target_ip = f"{base_ip}{ip_suffix}" + url = f"http://{target_ip}:80/api/mcp" + payload = { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/list" + } + req_data = json.dumps(payload).encode('utf-8') + req = urllib.request.Request( + url, + data=req_data, + headers={"Content-Type": "application/json"}, + method="POST" + ) + try: + with urllib.request.urlopen(req, timeout=2.0) as response: + resp_data = json.loads(response.read().decode('utf-8')) + tools = resp_data.get("result", {}).get("tools", []) + if any(t.get("name") == "clear_screen" for t in tools): + return target_ip + except: + pass + return None + + with concurrent.futures.ThreadPoolExecutor(max_workers=50) as executor: + futures = [executor.submit(check_ip, i) for i in range(1, 255)] + for fut in concurrent.futures.as_completed(futures): + res = fut.result() + if res: + executor.shutdown(wait=False) + sys.stderr.write(f"Auto-discovered ESP32 Screen at IP: {res} via subnet scan\n") + sys.stderr.flush() + return res + return None + def main(): parser = argparse.ArgumentParser(description="MCP Stdio-to-HTTP Bridge for ESP32-S3-RLCD-4.2") - parser.add_argument("--ip", required=True, help="IP address of the ESP32 board (e.g. 192.168.1.123)") + parser.add_argument("--ip", help="IP address of the ESP32 board. If omitted, auto-discovery is performed.") parser.add_argument("--port", type=int, default=80, help="Port the MCP server is listening on (default 80)") args = parser.parse_args() - url = f"http://{args.ip}:{args.port}/api/mcp" + ip = args.ip + if not ip: + ip = discover_screen() + if not ip: + sys.stderr.write("UDP broadcast discovery timed out. Initializing subnet sweep...\n") + sys.stderr.flush() + ip = scan_subnet() + if not ip: + sys.stderr.write("Subnet sweep failed. Falling back to hostname: 'esp32screen.local'\n") + sys.stderr.flush() + ip = "esp32screen.local" + + url = f"http://{ip}:{args.port}/api/mcp" sys.stderr.write(f"ESP32 MCP Stdio-to-HTTP Bridge started. Routing stdio to {url}\n") sys.stderr.flush() diff --git a/mcp_server.py b/mcp_server.py index 69d70c7..4878c78 100644 --- a/mcp_server.py +++ b/mcp_server.py @@ -26,9 +26,30 @@ class MCPServer: # Set socket to non-blocking so the main loop can run animations concurrently self.sock.setblocking(False) print(f"MCP server listening on port {port}...") + + # Setup UDP socket for auto-discovery on port 5000 + try: + self.udp_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.udp_sock.bind(('', 5000)) + self.udp_sock.setblocking(False) + print("UDP Discovery responder listening on port 5000...") + except Exception as ue: + print("Failed to bind UDP Discovery socket: {}".format(ue)) + self.udp_sock = None def update(self): """Check for incoming HTTP requests and handle them non-blockingly.""" + # 1. Handle UDP Discovery queries + if hasattr(self, 'udp_sock') and self.udp_sock is not None: + try: + data, addr = self.udp_sock.recvfrom(128) + if data == b"DISCOVER_SCREEN": + self.udp_sock.sendto(b"SCREEN_IP_80", addr) + except OSError: + pass + + # 2. Check for incoming HTTP TCP connections if self.sock is None: return diff --git a/virtual_screen_mcp/README.md b/virtual_screen_mcp/README.md new file mode 100644 index 0000000..a25a037 --- /dev/null +++ b/virtual_screen_mcp/README.md @@ -0,0 +1,101 @@ +# Virtual Screen & Speakers MCP Companion + +This directory implements a python-based Model Context Protocol (MCP) server that hosts a premium, futuristic web dashboard. You can open this web interface on **any local network device** (like a phone, tablet, or secondary monitor) to act as a virtual screen and speakers for the LLM. + +``` +┌─────────────────────────────────┐ +│ Local LLM Client / Cursor │ +└────────────────┬────────────────┘ + │ (JSON-RPC over Stdio) + ▼ +┌─────────────────────────────────┐ +│ server.py │ <── (Maintains PIL canvas state in memory) +└────────────────┬────────────────┘ + │ (JSON-RPC over WebSockets) + ▼ +┌─────────────────────────────────┐ +│ Browser Web Client │ (Render canvas, play Audio API tones/WAV, +│ (index.html UI) │ capture mic and upload WAV) +└─────────────────────────────────┘ +``` + +--- + +## 1. Prerequisites + +Ensure you have Python 3.10+ installed along with the required libraries: +```bash +pip install pillow tornado +``` + +--- + +## 2. Launching the Server + +Start the companion server from this folder: +```bash +python3 server.py --port 8080 +``` +Upon launching, it will print out the connection URLs, such as: +```text +-------------------------------------------------- +Virtual Screen & Speaker MCP Server Initialized. +Connect local devices in your browser to: +==> http://192.168.1.15:8080/ +==> http://localhost:8080/ +-------------------------------------------------- +``` +Open either URL on your laptop or enter the local IP version (`http://192.168.1.15:8080/`) into the browser of your phone or tablet. + +--- + +## 3. Claude Desktop Integration + +To register this server in Claude Desktop, open your configuration file: +* **macOS**: `/Users//Library/Application Support/Claude/claude_desktop_config.json` +* **Windows**: `%APPDATA%\Claude\claude_desktop_config.json` + +Add the server to the list: +```json +{ + "mcpServers": { + "virtual-companion": { + "command": "python3", + "args": [ + "/Users/adolforeyna/Projects/MicroPython/test1/Screen/virtual_screen_mcp/server.py", + "--port", + "8080" + ] + } + } +} +``` +Restart Claude Desktop to load the companion tools. + +--- + +## 4. Features & Tools + +The server exposes the following MCP tools to the LLM: + +| Tool Name | Parameters | Description | +|---|---|---| +| `clear_screen` | `color` (optional CSS color, e.g., `#1e1e2e` or legacy 0/1) | Clears the web canvas. | +| `draw_text` | `text`, `x`, `y`, `size`, `color` | Draws text with custom fonts and colors on the display. | +| `draw_shape` | `shape` (rect/circle/line), `x`, `y`, `width`, `height`, `color`, `fill` | Draws vector shapes on the canvas. | +| `draw_image` | `image_base64`, `x`, `y`, `width`, `height` | Draws base64 encoded images. | +| `get_screenshot` | *(None)* | Captures the virtual display buffer as a PNG image for the LLM. | +| `set_led` | `r`, `g`, `b`, `mode` (static/breath/rainbow/off) | Controls the CSS WS2812 NeoPixel ring. | +| `get_sensors` | *(None)* | Reads telemetry values adjusted by the dashboard sliders. | +| `play_tone` | `frequency`, `duration_ms`, `volume` | Synthesizes pure tones on the browser speakers. | +| `play_audio` | `audio_base64_or_url`, `volume` | Plays an audio track on the browser speakers. | +| `record_voice` | `duration_sec` | Captures microphone input from the browser, encodes it as a mono 16-bit WAV, and saves it. | + +--- + +## 5. Web Client Telemetry + +The dashboard provides interactive elements: +* **Virtual Sensor Sliders**: Move the sliders for Temperature, Humidity, and Light to feed custom telemetry data to the LLM. When the LLM calls `get_sensors`, it receives these live values. +* **Microphone Recorder**: Shows active microphone status, handles media constraints, and encodes 16-bit WAV files locally on the browser side. +* **LED NeoPixel ring**: Visualizes breath, rainbow, and static LED modes in full glowing CSS. diff --git a/virtual_screen_mcp/index.html b/virtual_screen_mcp/index.html new file mode 100644 index 0000000..4301368 --- /dev/null +++ b/virtual_screen_mcp/index.html @@ -0,0 +1,1082 @@ + + + + + + Virtual Screen & Speakers - MCP Companion + + + + + + + + +
+ +
+ + Disconnected +
+
+ +
+ +
+
+

Virtual Sensors

+
+
Adjust sliders to simulate local device telemetry
+ +
+
+
+ Temperature + 22.5 °C +
+ +
+ +
+
+ Humidity + 45 % +
+ +
+ +
+
+ Ambient Light + 350 lx +
+ +
+
+ +
+ +
+

System telemetry

+
+
+
Virtual Client
+
Browser Tab
+ +
WebSocket URL
+
ws://...
+ +
Latency
+
0 ms
+
+
+ + +
+
+
+
+

Device Display

+
800x600 High-Res Virtual Frame Buffer
+
+
+ +
+ +
+
+
+
+ + +
+
+

NeoPixel LED Ring

+
+
Mode: Off
+ +
+ +
+ +
+ +
+

Audio & Microphone

+
+ +
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+ + + +
+ Microphone Idle +
+
+
+ + +
+ +
Establishing WebSocket Connection...
+
+ + + + diff --git a/virtual_screen_mcp/mcp_config.json b/virtual_screen_mcp/mcp_config.json new file mode 100644 index 0000000..68a382b --- /dev/null +++ b/virtual_screen_mcp/mcp_config.json @@ -0,0 +1,12 @@ +{ + "mcpServers": { + "virtual-screen-companion": { + "command": "python3", + "args": [ + "/Users/adolforeyna/Projects/MicroPython/test1/Screen/virtual_screen_mcp/server.py", + "--port", + "8080" + ] + } + } +} diff --git a/virtual_screen_mcp/server.py b/virtual_screen_mcp/server.py new file mode 100644 index 0000000..a07bd13 --- /dev/null +++ b/virtual_screen_mcp/server.py @@ -0,0 +1,677 @@ +#!/usr/bin/env python3 +import sys +import json +import socket +import asyncio +import base64 +import uuid +import os +import argparse +from io import BytesIO +from PIL import Image, ImageDraw, ImageFont, ImageColor + +# Third-party dependencies +import tornado.web +import tornado.websocket +import tornado.ioloop + +# Global stdout reference for MCP protocol responses +mcp_stdout = None + +# Helper to parse color values +def parse_color(color_val, default=(255, 255, 255, 255)): + if isinstance(color_val, (int, float)): + # Handle Waveshare legacy screen logic: 0 = White, 1 = Black + if color_val == 0: + return (255, 255, 255, 255) + else: + return (0, 0, 0, 255) + + if not color_val: + return default + + try: + rgb = ImageColor.getrgb(str(color_val)) + if len(rgb) == 3: + return rgb + (255,) + return rgb + except Exception: + return default + +# Class to manage the state of the virtual companion +class VirtualDeviceState: + def __init__(self): + # 800x600 virtual screen buffer + self.canvas = Image.new("RGBA", (800, 600), (15, 23, 42, 255)) # slate-900 initial color + self.draw = ImageDraw.Draw(self.canvas) + + # LED Ring parameters + self.led_r = 0 + self.led_g = 0 + self.led_b = 0 + self.led_mode = "off" + + # Virtual sensors telemetry + self.sensor_temp = 22.5 + self.sensor_hum = 45.0 + self.sensor_light = 350.0 + + # Pending microphone recording futures + self.active_recordings = {} + + def clear_screen(self, color): + fill_color = parse_color(color, (15, 23, 42, 255)) + self.draw.rectangle([0, 0, 800, 600], fill=fill_color) + + def draw_text(self, text, x, y, size=24, color="white"): + fill_color = parse_color(color, (255, 255, 255, 255)) + # Load system fonts safely + font = None + for font_name in ["Arial.ttf", "Arial.ttc", "Helvetica.ttf", "Helvetica.ttc", "Courier.ttf"]: + try: + # Check typical paths or load by name + font = ImageFont.truetype(font_name, size) + break + except IOError: + continue + + if font is None: + # Try macOS system fonts location + macos_font_path = f"/System/Library/Fonts/Supplemental/Arial.ttf" + if os.path.exists(macos_font_path): + try: + font = ImageFont.truetype(macos_font_path, size) + except IOError: + pass + + if font is None: + font = ImageFont.load_default() + + self.draw.text((x, y), text, fill=fill_color, font=font) + + def draw_shape(self, shape, x, y, w, h, color="white", fill=False): + draw_color = parse_color(color, (255, 255, 255, 255)) + + if shape == "rect": + if fill: + self.draw.rectangle([x, y, x + w, y + h], fill=draw_color) + else: + self.draw.rectangle([x, y, x + w, y + h], outline=draw_color, width=2) + elif shape == "circle": + if fill: + self.draw.ellipse([x, y, x + w, y + h], fill=draw_color) + else: + self.draw.ellipse([x, y, x + w, y + h], outline=draw_color, width=2) + elif shape == "line": + self.draw.line([x, y, x + w, y + h], fill=draw_color, width=2) + + def draw_image(self, image_base64, x=0, y=0, w=None, h=None): + if "," in image_base64: + image_base64 = image_base64.split(",")[1] + + img_data = base64.b64decode(image_base64) + img = Image.open(BytesIO(img_data)) + + # Handle scaling compatibility + try: + resample_method = Image.Resampling.LANCZOS + except AttributeError: + resample_method = Image.ANTIALIAS + + if w and h: + img = img.resize((w, h), resample_method) + elif w or h: + orig_w, orig_h = img.size + if w: + ratio = w / orig_w + img = img.resize((w, int(orig_h * ratio)), resample_method) + else: + ratio = h / orig_h + img = img.resize((int(orig_w * ratio), h), resample_method) + + # Paste image on canvas + self.canvas.paste(img, (x, y), img if img.mode in ('RGBA', 'LA') else None) + + def get_screenshot_base64(self): + buffered = BytesIO() + self.canvas.save(buffered, format="PNG") + return base64.b64encode(buffered.getvalue()).decode("utf-8") + +# Instantiate device state +state = VirtualDeviceState() + +# Tornado Web Application Route Handlers +class MainHandler(tornado.web.RequestHandler): + def get(self): + # Serve the index.html from the same directory + self.render("index.html") + +class ClientWebSocketHandler(tornado.websocket.WebSocketHandler): + clients = set() + + def check_origin(self, origin): + return True # Enable local network browser connections + + def open(self): + ClientWebSocketHandler.clients.add(self) + sys.stderr.write(f"[Web Server] Client connected from: {self.request.remote_ip}\n") + sys.stderr.flush() + + # Send current screen content and active LED state to client on join + self.send_initial_state() + + def send_initial_state(self): + try: + b64_png = state.get_screenshot_base64() + # Draw the current canvas image to the canvas of the newly joined client + self.write_message(json.dumps({ + "action": "draw_image", + "image_base64": b64_png, + "x": 0, + "y": 0, + "w": 800, + "h": 600 + })) + # Send current LED settings + self.write_message(json.dumps({ + "action": "set_led", + "r": state.led_r, + "g": state.led_g, + "b": state.led_b, + "mode": state.led_mode + })) + except Exception as e: + sys.stderr.write(f"[Web Server] Error syncing state: {e}\n") + sys.stderr.flush() + + def on_close(self): + ClientWebSocketHandler.clients.remove(self) + sys.stderr.write("[Web Server] Client disconnected\n") + sys.stderr.flush() + + def on_message(self, message): + try: + data = json.loads(message) + event = data.get("event") + + if event == "sensor_update": + state.sensor_temp = data.get("temperature", 22.5) + state.sensor_hum = data.get("humidity", 45.0) + state.sensor_light = data.get("light", 350.0) + + elif event == "voice_recording": + rec_id = data.get("recording_id") + if rec_id in state.active_recordings: + fut = state.active_recordings[rec_id] + if not fut.done(): + error_msg = data.get("error") + if error_msg: + fut.set_exception(RuntimeError(error_msg)) + else: + fut.set_result(data.get("audio_base64")) + + elif event == "ping": + # Respond to browser ping with pong latency calculation + self.write_message(json.dumps({"event": "pong", "t": data.get("t")})) + + except Exception as e: + sys.stderr.write(f"[Web Server] Error processing message: {e}\n") + sys.stderr.flush() + + @classmethod + def broadcast(cls, payload): + msg = json.dumps(payload) + for client in cls.clients: + try: + client.write_message(msg) + except Exception: + pass + +# Send responses back to the LLM Client via stdio +def send_rpc_response(resp): + payload = json.dumps(resp) + "\n" + mcp_stdout.write(payload) + mcp_stdout.flush() + +# Get local LAN IP address +def get_local_ip(): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + s.connect(('10.255.255.255', 1)) + IP = s.getsockname()[0] + except Exception: + IP = '127.0.0.1' + finally: + s.close() + return IP + +# Define the list of MCP tools +def get_mcp_tools_list(): + return [ + { + "name": "clear_screen", + "description": "Clear the virtual screen. Optionally specify color (e.g. standard hex '#1e1e2e', name 'white', or 0 for white / 1 for black).", + "inputSchema": { + "type": "object", + "properties": { + "color": {"type": "string", "description": "Color hex, CSS name, or integer (0=white, 1=black)"} + } + } + }, + { + "name": "draw_text", + "description": "Draw text at specified (x,y) coordinates on the virtual display.", + "inputSchema": { + "type": "object", + "properties": { + "text": {"type": "string", "description": "The text message to draw"}, + "x": {"type": "integer", "description": "X coordinate (0-780)"}, + "y": {"type": "integer", "description": "Y coordinate (0-580)"}, + "size": {"type": "integer", "description": "Font size in pixels (default 24)"}, + "color": {"type": "string", "description": "Text color (default 'white')"} + }, + "required": ["text", "x", "y"] + } + }, + { + "name": "draw_shape", + "description": "Draw a geometric shape (rectangle, circle, or line) on the virtual display.", + "inputSchema": { + "type": "object", + "properties": { + "shape": {"type": "string", "enum": ["rect", "circle", "line"], "description": "Type of shape"}, + "x": {"type": "integer", "description": "Start X coordinate"}, + "y": {"type": "integer", "description": "Start Y coordinate"}, + "width": {"type": "integer", "description": "Width (or delta X for line)"}, + "height": {"type": "integer", "description": "Height (or delta Y for line)"}, + "color": {"type": "string", "description": "Color of the outline or fill (default 'white')"}, + "fill": {"type": "boolean", "description": "Whether to fill the shape (default false)"} + }, + "required": ["shape", "x", "y", "width", "height"] + } + }, + { + "name": "draw_image", + "description": "Draw a base64 encoded PNG or JPEG image on the virtual screen at (x,y).", + "inputSchema": { + "type": "object", + "properties": { + "image_base64": {"type": "string", "description": "Base64 encoded string of the image file"}, + "x": {"type": "integer", "description": "X coordinate (default 0)", "default": 0}, + "y": {"type": "integer", "description": "Y coordinate (default 0)", "default": 0}, + "width": {"type": "integer", "description": "Width to scale the image (optional)"}, + "height": {"type": "integer", "description": "Height to scale the image (optional)"} + }, + "required": ["image_base64"] + } + }, + { + "name": "get_screenshot", + "description": "Capture the current virtual display buffer and return it as a PNG image, allowing you to see what is currently rendered on the screen.", + "inputSchema": {"type": "object", "properties": {}} + }, + { + "name": "set_led", + "description": "Control the virtual RGB LED ring lights.", + "inputSchema": { + "type": "object", + "properties": { + "r": {"type": "integer", "minimum": 0, "maximum": 255, "description": "Red channel (0-255)"}, + "g": {"type": "integer", "minimum": 0, "maximum": 255, "description": "Green channel (0-255)"}, + "b": {"type": "integer", "minimum": 0, "maximum": 255, "description": "Blue channel (0-255)"}, + "mode": {"type": "string", "enum": ["static", "breath", "rainbow", "off"], "description": "LED animation mode"} + }, + "required": ["r", "g", "b", "mode"] + } + }, + { + "name": "get_sensors", + "description": "Read telemetry values from virtual temperature, humidity, and ambient light sensors.", + "inputSchema": {"type": "object", "properties": {}} + }, + { + "name": "play_tone", + "description": "Play a sound tone of a specific frequency and duration on the virtual speaker.", + "inputSchema": { + "type": "object", + "properties": { + "frequency": {"type": "integer", "description": "Frequency in Hz (default 440)"}, + "duration_ms": {"type": "integer", "description": "Duration in milliseconds (default 1000)"}, + "volume": {"type": "integer", "minimum": 0, "maximum": 100, "description": "Volume percent from 0 to 100 (default 50)"} + } + } + }, + { + "name": "play_audio", + "description": "Play a base64 encoded audio track (like WAV or MP3) or an audio URL on the virtual speaker.", + "inputSchema": { + "type": "object", + "properties": { + "audio_base64_or_url": {"type": "string", "description": "Base64 audio bytes or accessible audio file URL"}, + "volume": {"type": "integer", "minimum": 0, "maximum": 100, "description": "Volume percent from 0 to 100 (default 50)"} + }, + "required": ["audio_base64_or_url"] + } + }, + { + "name": "record_voice", + "description": "Record voice commands/input from the connected client's microphone for a specified duration.", + "inputSchema": { + "type": "object", + "properties": { + "duration_sec": {"type": "integer", "minimum": 1, "maximum": 20, "description": "Duration to record in seconds (default 4)", "default": 4} + } + } + } + ] + +# Execute tool operations +async def execute_mcp_tool(name, args): + if name == "clear_screen": + color = args.get("color", 0) + state.clear_screen(color) + ClientWebSocketHandler.broadcast({"action": "clear", "color": color}) + return "Screen cleared successfully." + + elif name == "draw_text": + text = str(args.get("text", "")) + x = int(args.get("x", 10)) + y = int(args.get("y", 10)) + size = int(args.get("size", 24)) + color = str(args.get("color", "white")) + + state.draw_text(text, x, y, size, color) + ClientWebSocketHandler.broadcast({ + "action": "draw_text", + "text": text, + "x": x, + "y": y, + "size": size, + "color": color + }) + return f"Successfully drew text at ({x}, {y})." + + elif name == "draw_shape": + shape = str(args.get("shape")) + x = int(args.get("x")) + y = int(args.get("y")) + w = int(args.get("width")) + h = int(args.get("height")) + color = str(args.get("color", "white")) + fill = bool(args.get("fill", False)) + + state.draw_shape(shape, x, y, w, h, color, fill) + ClientWebSocketHandler.broadcast({ + "action": "draw_shape", + "shape": shape, + "x": x, + "y": y, + "w": w, + "h": h, + "color": color, + "fill": fill + }) + return f"Successfully drew {shape} shape." + + elif name == "draw_image": + img_b64 = str(args.get("image_base64")) + x = int(args.get("x", 0)) + y = int(args.get("y", 0)) + w = args.get("width") + h = args.get("height") + + if w is not None: w = int(w) + if h is not None: h = int(h) + + state.draw_image(img_b64, x, y, w, h) + ClientWebSocketHandler.broadcast({ + "action": "draw_image", + "image_base64": img_b64, + "x": x, + "y": y, + "w": w, + "h": h + }) + return "Successfully rendered image on display." + + elif name == "get_screenshot": + b64_png = state.get_screenshot_base64() + return [ + { + "type": "image", + "data": b64_png, + "mimeType": "image/png" + } + ] + + elif name == "set_led": + r = int(args.get("r", 0)) + g = int(args.get("g", 0)) + b = int(args.get("b", 0)) + mode = str(args.get("mode", "static")) + + state.led_r, state.led_g, state.led_b, state.led_mode = r, g, b, mode + ClientWebSocketHandler.broadcast({ + "action": "set_led", + "r": r, + "g": g, + "b": b, + "mode": mode + }) + return f"NeoPixel LED ring set to '{mode}' mode with color ({r}, {g}, {b})." + + elif name == "get_sensors": + return json.dumps({ + "temperature_c": state.sensor_temp, + "humidity_pct": state.sensor_hum, + "light_lux": state.sensor_light + }) + + elif name == "play_tone": + freq = int(args.get("frequency", 440)) + dur = int(args.get("duration_ms", 1000)) + vol = int(args.get("volume", 50)) + + ClientWebSocketHandler.broadcast({ + "action": "play_tone", + "frequency": freq, + "duration": dur, + "volume": vol / 100.0 + }) + return f"Played synthesized tone at {freq}Hz for {dur}ms." + + elif name == "play_audio": + audio_payload = str(args.get("audio_base64_or_url")) + vol = int(args.get("volume", 50)) + + ClientWebSocketHandler.broadcast({ + "action": "play_audio", + "audio": audio_payload, + "volume": vol / 100.0 + }) + return "Successfully transmitted audio buffer." + + elif name == "record_voice": + dur = int(args.get("duration_sec", 4)) + + if not ClientWebSocketHandler.clients: + return "No active browser sessions to capture microphone input. Recording aborted." + + rec_id = str(uuid.uuid4()) + loop = asyncio.get_running_loop() + fut = loop.create_future() + state.active_recordings[rec_id] = fut + + ClientWebSocketHandler.broadcast({ + "action": "record_voice", + "duration_sec": dur, + "recording_id": rec_id + }) + + try: + # Wait for client recording with a safety buffer + audio_b64 = await asyncio.wait_for(fut, timeout=dur + 5.0) + audio_bytes = base64.b64decode(audio_b64) + + # Save the WAV file to local disk + filename = f"recording_{rec_id[:8]}.wav" + with open(filename, "wb") as f: + f.write(audio_bytes) + + return f"Successfully recorded {dur} seconds of audio. File saved to '{filename}'." + except asyncio.TimeoutError: + return "Microphone recording timed out. Make sure the connected web client has granted mic permissions." + except Exception as e: + return f"Microphone recording failed: {str(e)}" + finally: + state.active_recordings.pop(rec_id, None) + + else: + raise ValueError(f"Unknown MCP tool: {name}") + +# Handles parsed MCP stdin request objects +async def handle_mcp_request(req): + method = req.get("method") + rpc_id = req.get("id") + + is_notification = rpc_id is None + + if method == "initialize": + resp = { + "jsonrpc": "2.0", + "id": rpc_id, + "result": { + "protocolVersion": "2024-11-05", + "capabilities": { + "tools": {} + }, + "serverInfo": { + "name": "virtual-screen-mcp", + "version": "1.0.0" + } + } + } + send_rpc_response(resp) + return + + elif method == "tools/list": + resp = { + "jsonrpc": "2.0", + "id": rpc_id, + "result": { + "tools": get_mcp_tools_list() + } + } + send_rpc_response(resp) + return + + elif method == "tools/call": + params = req.get("params", {}) + tool_name = params.get("name") + arguments = params.get("arguments", {}) + + try: + result = await execute_mcp_tool(tool_name, arguments) + if isinstance(result, list): + content = result + else: + content = [{"type": "text", "text": str(result)}] + + resp = { + "jsonrpc": "2.0", + "id": rpc_id, + "result": { + "content": content + } + } + except Exception as e: + resp = { + "jsonrpc": "2.0", + "id": rpc_id, + "error": { + "code": -32000, + "message": str(e) + } + } + send_rpc_response(resp) + return + + if not is_notification: + resp = { + "jsonrpc": "2.0", + "id": rpc_id, + "error": { + "code": -32601, + "message": f"Method {method} not found" + } + } + send_rpc_response(resp) + +# Read stdin line-by-line asynchronously +async def stdio_reader_loop(): + loop = asyncio.get_running_loop() + reader = asyncio.StreamReader() + protocol = asyncio.StreamReaderProtocol(reader) + await loop.connect_read_pipe(lambda: protocol, sys.stdin) + + while True: + line = await reader.readline() + if not line: + break + + try: + req = json.loads(line.decode('utf-8').strip()) + await handle_mcp_request(req) + except Exception as e: + sys.stderr.write(f"[MCP Stdio] Error handling line input: {e}\n") + sys.stderr.flush() + +# Strong references to prevent garbage collection of background tasks +background_tasks = set() + +# Async main runtime loop +async def main_async(port): + app = tornado.web.Application([ + (r"/", MainHandler), + (r"/ws", ClientWebSocketHandler), + ], template_path=os.path.dirname(os.path.abspath(__file__))) + + app.listen(port) + + local_ip = get_local_ip() + sys.stderr.write(f"--------------------------------------------------\n") + sys.stderr.write(f"Virtual Screen & Speaker MCP Server Initialized.\n") + sys.stderr.write(f"Connect local devices in your browser to:\n") + sys.stderr.write(f"==> http://{local_ip}:{port}/\n") + sys.stderr.write(f"==> http://localhost:{port}/\n") + sys.stderr.write(f"--------------------------------------------------\n") + sys.stderr.flush() + + # Spawn the stdin line reader task and store a strong reference + task = asyncio.create_task(stdio_reader_loop()) + background_tasks.add(task) + task.add_done_callback(background_tasks.discard) + + # Run indefinitely + while True: + await asyncio.sleep(3600) + +def main(): + parser = argparse.ArgumentParser(description="Virtual Screen & Speakers MCP Server") + parser.add_argument("--port", type=int, default=8080, help="Web port (default 8080)") + args = parser.parse_args() + + # Set the MCP stdio channel + global mcp_stdout + mcp_stdout = sys.stdout + # Route default stdout to stderr so prints don't interrupt standard JSON-RPC communications + sys.stdout = sys.stderr + + try: + asyncio.run(main_async(args.port)) + except KeyboardInterrupt: + sys.stderr.write("[Server] Stopped by KeyboardInterrupt.\n") + sys.stderr.flush() + +if __name__ == "__main__": + main() diff --git a/virtual_screen_mcp/test_client.py b/virtual_screen_mcp/test_client.py new file mode 100644 index 0000000..2870135 --- /dev/null +++ b/virtual_screen_mcp/test_client.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +import subprocess +import json +import base64 +import time +import os + +def run_test(): + print("Starting server.py in a subprocess for stdio test...") + # Start server in subprocess on port 8089 to avoid port conflicts + proc = subprocess.Popen( + ["python3", "server.py", "--port", "8089"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, # ignore logs on stderr + text=True, + bufsize=1 + ) + + # Wait a moment for server to bind + time.sleep(1.5) + + def send_cmd(cmd_dict): + proc.stdin.write(json.dumps(cmd_dict) + "\n") + proc.stdin.flush() + line = proc.stdout.readline() + if not line: + return None + return json.loads(line.strip()) + + print("\n[1/4] Sending initialize request...") + init_resp = send_cmd({ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": {} + }) + print("Response:", json.dumps(init_resp, indent=2)) + + print("\n[2/4] Sending clear_screen tool call...") + resp = send_cmd({ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": { + "name": "clear_screen", + "arguments": {"color": "#1e1e2e"} + } + }) + print("Response:", json.dumps(resp, indent=2)) + + print("\n[3/4] Sending draw_text tool call...") + resp = send_cmd({ + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": { + "name": "draw_text", + "arguments": { + "text": "TEST SUCCEEDED", + "x": 200, + "y": 280, + "size": 48, + "color": "#10b981" + } + } + }) + print("Response:", json.dumps(resp, indent=2)) + + print("\n[4/4] Sending get_screenshot tool call...") + resp = send_cmd({ + "jsonrpc": "2.0", + "id": 4, + "method": "tools/call", + "params": { + "name": "get_screenshot", + "arguments": {} + } + }) + + try: + content = resp["result"]["content"][0] + if content["type"] == "image": + img_b64 = content["data"] + output_file = "test_screenshot.png" + with open(output_file, "wb") as f: + f.write(base64.b64decode(img_b64)) + print(f"Success! Captured screenshot saved to: {os.path.abspath(output_file)}") + else: + print("Failed: content is not an image", content) + except Exception as e: + print("Error reading screenshot response:", e) + print("Raw response:", resp) + + # Stop subprocess + proc.terminate() + proc.wait() + print("\nVerification script finished.") + +if __name__ == "__main__": + run_test()