Add auto-discovery support (UDP discovery responder & subnet scanner) and implement virtual screen MCP server
This commit is contained in:
Adolfo Reyna
@@ -21,6 +21,13 @@ def connect_wifi():
|
||||
print("Display init failed in boot.py:", e)
|
||||
|
||||
# Initialize Wi-Fi Station
|
||||
try:
|
||||
import network
|
||||
network.hostname('esp32screen')
|
||||
print("Hostname set to: {}".format(network.hostname()))
|
||||
except Exception as he:
|
||||
print("Failed to set hostname: {}".format(he))
|
||||
|
||||
wlan = network.WLAN(network.STA_IF)
|
||||
wlan.active(True)
|
||||
|
||||
|
||||
+114
-2
@@ -4,13 +4,125 @@ import json
|
||||
import urllib.request
|
||||
import argparse
|
||||
|
||||
def discover_screen():
|
||||
import socket
|
||||
sys.stderr.write("Searching for ESP32 Screen via UDP broadcast on port 5000...\n")
|
||||
sys.stderr.flush()
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
|
||||
s.settimeout(1.5)
|
||||
|
||||
broadcast_ips = ['255.255.255.255', '<broadcast>']
|
||||
try:
|
||||
hostname = socket.gethostname()
|
||||
ip_list = socket.gethostbyname_ex(hostname)[2]
|
||||
for ip in ip_list:
|
||||
if not ip.startswith('127.'):
|
||||
parts = ip.split('.')
|
||||
if len(parts) == 4:
|
||||
subnet_bcast = f"{parts[0]}.{parts[1]}.{parts[2]}.255"
|
||||
if subnet_bcast not in broadcast_ips:
|
||||
broadcast_ips.append(subnet_bcast)
|
||||
except:
|
||||
pass
|
||||
|
||||
for target_ip in broadcast_ips:
|
||||
try:
|
||||
s.sendto(b"DISCOVER_SCREEN", (target_ip, 5000))
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
data, addr = s.recvfrom(128)
|
||||
if data == b"SCREEN_IP_80":
|
||||
sys.stderr.write(f"Auto-discovered ESP32 Screen at IP: {addr[0]} via UDP\n")
|
||||
sys.stderr.flush()
|
||||
return addr[0]
|
||||
except:
|
||||
pass
|
||||
finally:
|
||||
s.close()
|
||||
return None
|
||||
|
||||
def scan_subnet():
|
||||
import socket
|
||||
import concurrent.futures
|
||||
|
||||
try:
|
||||
hostname = socket.gethostname()
|
||||
local_ips = [ip for ip in socket.gethostbyname_ex(hostname)[2] if not ip.startswith('127.')]
|
||||
if not local_ips:
|
||||
return None
|
||||
local_ip = local_ips[0]
|
||||
except Exception as e:
|
||||
sys.stderr.write(f"Subnet lookup error: {e}\n")
|
||||
sys.stderr.flush()
|
||||
return None
|
||||
|
||||
parts = local_ip.split('.')
|
||||
if len(parts) == 4:
|
||||
base_ip = f"{parts[0]}.{parts[1]}.{parts[2]}."
|
||||
else:
|
||||
return None
|
||||
|
||||
sys.stderr.write(f"Scanning local subnet {base_ip}1 to {base_ip}254 on port 80...\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
def check_ip(ip_suffix):
|
||||
target_ip = f"{base_ip}{ip_suffix}"
|
||||
url = f"http://{target_ip}:80/api/mcp"
|
||||
payload = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "tools/list"
|
||||
}
|
||||
req_data = json.dumps(payload).encode('utf-8')
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
data=req_data,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST"
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=2.0) as response:
|
||||
resp_data = json.loads(response.read().decode('utf-8'))
|
||||
tools = resp_data.get("result", {}).get("tools", [])
|
||||
if any(t.get("name") == "clear_screen" for t in tools):
|
||||
return target_ip
|
||||
except:
|
||||
pass
|
||||
return None
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=50) as executor:
|
||||
futures = [executor.submit(check_ip, i) for i in range(1, 255)]
|
||||
for fut in concurrent.futures.as_completed(futures):
|
||||
res = fut.result()
|
||||
if res:
|
||||
executor.shutdown(wait=False)
|
||||
sys.stderr.write(f"Auto-discovered ESP32 Screen at IP: {res} via subnet scan\n")
|
||||
sys.stderr.flush()
|
||||
return res
|
||||
return None
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="MCP Stdio-to-HTTP Bridge for ESP32-S3-RLCD-4.2")
|
||||
parser.add_argument("--ip", required=True, help="IP address of the ESP32 board (e.g. 192.168.1.123)")
|
||||
parser.add_argument("--ip", help="IP address of the ESP32 board. If omitted, auto-discovery is performed.")
|
||||
parser.add_argument("--port", type=int, default=80, help="Port the MCP server is listening on (default 80)")
|
||||
args = parser.parse_args()
|
||||
|
||||
url = f"http://{args.ip}:{args.port}/api/mcp"
|
||||
ip = args.ip
|
||||
if not ip:
|
||||
ip = discover_screen()
|
||||
if not ip:
|
||||
sys.stderr.write("UDP broadcast discovery timed out. Initializing subnet sweep...\n")
|
||||
sys.stderr.flush()
|
||||
ip = scan_subnet()
|
||||
if not ip:
|
||||
sys.stderr.write("Subnet sweep failed. Falling back to hostname: 'esp32screen.local'\n")
|
||||
sys.stderr.flush()
|
||||
ip = "esp32screen.local"
|
||||
|
||||
url = f"http://{ip}:{args.port}/api/mcp"
|
||||
sys.stderr.write(f"ESP32 MCP Stdio-to-HTTP Bridge started. Routing stdio to {url}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
|
||||
@@ -26,9 +26,30 @@ class MCPServer:
|
||||
# Set socket to non-blocking so the main loop can run animations concurrently
|
||||
self.sock.setblocking(False)
|
||||
print(f"MCP server listening on port {port}...")
|
||||
|
||||
# Setup UDP socket for auto-discovery on port 5000
|
||||
try:
|
||||
self.udp_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
self.udp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
self.udp_sock.bind(('', 5000))
|
||||
self.udp_sock.setblocking(False)
|
||||
print("UDP Discovery responder listening on port 5000...")
|
||||
except Exception as ue:
|
||||
print("Failed to bind UDP Discovery socket: {}".format(ue))
|
||||
self.udp_sock = None
|
||||
|
||||
def update(self):
|
||||
"""Check for incoming HTTP requests and handle them non-blockingly."""
|
||||
# 1. Handle UDP Discovery queries
|
||||
if hasattr(self, 'udp_sock') and self.udp_sock is not None:
|
||||
try:
|
||||
data, addr = self.udp_sock.recvfrom(128)
|
||||
if data == b"DISCOVER_SCREEN":
|
||||
self.udp_sock.sendto(b"SCREEN_IP_80", addr)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# 2. Check for incoming HTTP TCP connections
|
||||
if self.sock is None:
|
||||
return
|
||||
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
# Virtual Screen & Speakers MCP Companion
|
||||
|
||||
This directory implements a python-based Model Context Protocol (MCP) server that hosts a premium, futuristic web dashboard. You can open this web interface on **any local network device** (like a phone, tablet, or secondary monitor) to act as a virtual screen and speakers for the LLM.
|
||||
|
||||
```
|
||||
┌─────────────────────────────────┐
|
||||
│ Local LLM Client / Cursor │
|
||||
└────────────────┬────────────────┘
|
||||
│ (JSON-RPC over Stdio)
|
||||
▼
|
||||
┌─────────────────────────────────┐
|
||||
│ server.py │ <── (Maintains PIL canvas state in memory)
|
||||
└────────────────┬────────────────┘
|
||||
│ (JSON-RPC over WebSockets)
|
||||
▼
|
||||
┌─────────────────────────────────┐
|
||||
│ Browser Web Client │ (Render canvas, play Audio API tones/WAV,
|
||||
│ (index.html UI) │ capture mic and upload WAV)
|
||||
└─────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 1. Prerequisites
|
||||
|
||||
Ensure you have Python 3.10+ installed along with the required libraries:
|
||||
```bash
|
||||
pip install pillow tornado
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. Launching the Server
|
||||
|
||||
Start the companion server from this folder:
|
||||
```bash
|
||||
python3 server.py --port 8080
|
||||
```
|
||||
Upon launching, it will print out the connection URLs, such as:
|
||||
```text
|
||||
--------------------------------------------------
|
||||
Virtual Screen & Speaker MCP Server Initialized.
|
||||
Connect local devices in your browser to:
|
||||
==> http://192.168.1.15:8080/
|
||||
==> http://localhost:8080/
|
||||
--------------------------------------------------
|
||||
```
|
||||
Open either URL on your laptop or enter the local IP version (`http://192.168.1.15:8080/`) into the browser of your phone or tablet.
|
||||
|
||||
---
|
||||
|
||||
## 3. Claude Desktop Integration
|
||||
|
||||
To register this server in Claude Desktop, open your configuration file:
|
||||
* **macOS**: `/Users/<username>/Library/Application Support/Claude/claude_desktop_config.json`
|
||||
* **Windows**: `%APPDATA%\Claude\claude_desktop_config.json`
|
||||
|
||||
Add the server to the list:
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"virtual-companion": {
|
||||
"command": "python3",
|
||||
"args": [
|
||||
"/Users/adolforeyna/Projects/MicroPython/test1/Screen/virtual_screen_mcp/server.py",
|
||||
"--port",
|
||||
"8080"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
Restart Claude Desktop to load the companion tools.
|
||||
|
||||
---
|
||||
|
||||
## 4. Features & Tools
|
||||
|
||||
The server exposes the following MCP tools to the LLM:
|
||||
|
||||
| Tool Name | Parameters | Description |
|
||||
|---|---|---|
|
||||
| `clear_screen` | `color` (optional CSS color, e.g., `#1e1e2e` or legacy 0/1) | Clears the web canvas. |
|
||||
| `draw_text` | `text`, `x`, `y`, `size`, `color` | Draws text with custom fonts and colors on the display. |
|
||||
| `draw_shape` | `shape` (rect/circle/line), `x`, `y`, `width`, `height`, `color`, `fill` | Draws vector shapes on the canvas. |
|
||||
| `draw_image` | `image_base64`, `x`, `y`, `width`, `height` | Draws base64 encoded images. |
|
||||
| `get_screenshot` | *(None)* | Captures the virtual display buffer as a PNG image for the LLM. |
|
||||
| `set_led` | `r`, `g`, `b`, `mode` (static/breath/rainbow/off) | Controls the CSS WS2812 NeoPixel ring. |
|
||||
| `get_sensors` | *(None)* | Reads telemetry values adjusted by the dashboard sliders. |
|
||||
| `play_tone` | `frequency`, `duration_ms`, `volume` | Synthesizes pure tones on the browser speakers. |
|
||||
| `play_audio` | `audio_base64_or_url`, `volume` | Plays an audio track on the browser speakers. |
|
||||
| `record_voice` | `duration_sec` | Captures microphone input from the browser, encodes it as a mono 16-bit WAV, and saves it. |
|
||||
|
||||
---
|
||||
|
||||
## 5. Web Client Telemetry
|
||||
|
||||
The dashboard provides interactive elements:
|
||||
* **Virtual Sensor Sliders**: Move the sliders for Temperature, Humidity, and Light to feed custom telemetry data to the LLM. When the LLM calls `get_sensors`, it receives these live values.
|
||||
* **Microphone Recorder**: Shows active microphone status, handles media constraints, and encodes 16-bit WAV files locally on the browser side.
|
||||
* **LED NeoPixel ring**: Visualizes breath, rainbow, and static LED modes in full glowing CSS.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"mcpServers": {
|
||||
"virtual-screen-companion": {
|
||||
"command": "python3",
|
||||
"args": [
|
||||
"/Users/adolforeyna/Projects/MicroPython/test1/Screen/virtual_screen_mcp/server.py",
|
||||
"--port",
|
||||
"8080"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,677 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
import json
|
||||
import socket
|
||||
import asyncio
|
||||
import base64
|
||||
import uuid
|
||||
import os
|
||||
import argparse
|
||||
from io import BytesIO
|
||||
from PIL import Image, ImageDraw, ImageFont, ImageColor
|
||||
|
||||
# Third-party dependencies
|
||||
import tornado.web
|
||||
import tornado.websocket
|
||||
import tornado.ioloop
|
||||
|
||||
# Global stdout reference for MCP protocol responses
|
||||
mcp_stdout = None
|
||||
|
||||
# Helper to parse color values
|
||||
def parse_color(color_val, default=(255, 255, 255, 255)):
|
||||
if isinstance(color_val, (int, float)):
|
||||
# Handle Waveshare legacy screen logic: 0 = White, 1 = Black
|
||||
if color_val == 0:
|
||||
return (255, 255, 255, 255)
|
||||
else:
|
||||
return (0, 0, 0, 255)
|
||||
|
||||
if not color_val:
|
||||
return default
|
||||
|
||||
try:
|
||||
rgb = ImageColor.getrgb(str(color_val))
|
||||
if len(rgb) == 3:
|
||||
return rgb + (255,)
|
||||
return rgb
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
# Class to manage the state of the virtual companion
|
||||
class VirtualDeviceState:
|
||||
def __init__(self):
|
||||
# 800x600 virtual screen buffer
|
||||
self.canvas = Image.new("RGBA", (800, 600), (15, 23, 42, 255)) # slate-900 initial color
|
||||
self.draw = ImageDraw.Draw(self.canvas)
|
||||
|
||||
# LED Ring parameters
|
||||
self.led_r = 0
|
||||
self.led_g = 0
|
||||
self.led_b = 0
|
||||
self.led_mode = "off"
|
||||
|
||||
# Virtual sensors telemetry
|
||||
self.sensor_temp = 22.5
|
||||
self.sensor_hum = 45.0
|
||||
self.sensor_light = 350.0
|
||||
|
||||
# Pending microphone recording futures
|
||||
self.active_recordings = {}
|
||||
|
||||
def clear_screen(self, color):
|
||||
fill_color = parse_color(color, (15, 23, 42, 255))
|
||||
self.draw.rectangle([0, 0, 800, 600], fill=fill_color)
|
||||
|
||||
def draw_text(self, text, x, y, size=24, color="white"):
|
||||
fill_color = parse_color(color, (255, 255, 255, 255))
|
||||
# Load system fonts safely
|
||||
font = None
|
||||
for font_name in ["Arial.ttf", "Arial.ttc", "Helvetica.ttf", "Helvetica.ttc", "Courier.ttf"]:
|
||||
try:
|
||||
# Check typical paths or load by name
|
||||
font = ImageFont.truetype(font_name, size)
|
||||
break
|
||||
except IOError:
|
||||
continue
|
||||
|
||||
if font is None:
|
||||
# Try macOS system fonts location
|
||||
macos_font_path = f"/System/Library/Fonts/Supplemental/Arial.ttf"
|
||||
if os.path.exists(macos_font_path):
|
||||
try:
|
||||
font = ImageFont.truetype(macos_font_path, size)
|
||||
except IOError:
|
||||
pass
|
||||
|
||||
if font is None:
|
||||
font = ImageFont.load_default()
|
||||
|
||||
self.draw.text((x, y), text, fill=fill_color, font=font)
|
||||
|
||||
def draw_shape(self, shape, x, y, w, h, color="white", fill=False):
|
||||
draw_color = parse_color(color, (255, 255, 255, 255))
|
||||
|
||||
if shape == "rect":
|
||||
if fill:
|
||||
self.draw.rectangle([x, y, x + w, y + h], fill=draw_color)
|
||||
else:
|
||||
self.draw.rectangle([x, y, x + w, y + h], outline=draw_color, width=2)
|
||||
elif shape == "circle":
|
||||
if fill:
|
||||
self.draw.ellipse([x, y, x + w, y + h], fill=draw_color)
|
||||
else:
|
||||
self.draw.ellipse([x, y, x + w, y + h], outline=draw_color, width=2)
|
||||
elif shape == "line":
|
||||
self.draw.line([x, y, x + w, y + h], fill=draw_color, width=2)
|
||||
|
||||
def draw_image(self, image_base64, x=0, y=0, w=None, h=None):
|
||||
if "," in image_base64:
|
||||
image_base64 = image_base64.split(",")[1]
|
||||
|
||||
img_data = base64.b64decode(image_base64)
|
||||
img = Image.open(BytesIO(img_data))
|
||||
|
||||
# Handle scaling compatibility
|
||||
try:
|
||||
resample_method = Image.Resampling.LANCZOS
|
||||
except AttributeError:
|
||||
resample_method = Image.ANTIALIAS
|
||||
|
||||
if w and h:
|
||||
img = img.resize((w, h), resample_method)
|
||||
elif w or h:
|
||||
orig_w, orig_h = img.size
|
||||
if w:
|
||||
ratio = w / orig_w
|
||||
img = img.resize((w, int(orig_h * ratio)), resample_method)
|
||||
else:
|
||||
ratio = h / orig_h
|
||||
img = img.resize((int(orig_w * ratio), h), resample_method)
|
||||
|
||||
# Paste image on canvas
|
||||
self.canvas.paste(img, (x, y), img if img.mode in ('RGBA', 'LA') else None)
|
||||
|
||||
def get_screenshot_base64(self):
|
||||
buffered = BytesIO()
|
||||
self.canvas.save(buffered, format="PNG")
|
||||
return base64.b64encode(buffered.getvalue()).decode("utf-8")
|
||||
|
||||
# Instantiate device state
|
||||
state = VirtualDeviceState()
|
||||
|
||||
# Tornado Web Application Route Handlers
|
||||
class MainHandler(tornado.web.RequestHandler):
|
||||
def get(self):
|
||||
# Serve the index.html from the same directory
|
||||
self.render("index.html")
|
||||
|
||||
class ClientWebSocketHandler(tornado.websocket.WebSocketHandler):
|
||||
clients = set()
|
||||
|
||||
def check_origin(self, origin):
|
||||
return True # Enable local network browser connections
|
||||
|
||||
def open(self):
|
||||
ClientWebSocketHandler.clients.add(self)
|
||||
sys.stderr.write(f"[Web Server] Client connected from: {self.request.remote_ip}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
# Send current screen content and active LED state to client on join
|
||||
self.send_initial_state()
|
||||
|
||||
def send_initial_state(self):
|
||||
try:
|
||||
b64_png = state.get_screenshot_base64()
|
||||
# Draw the current canvas image to the canvas of the newly joined client
|
||||
self.write_message(json.dumps({
|
||||
"action": "draw_image",
|
||||
"image_base64": b64_png,
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"w": 800,
|
||||
"h": 600
|
||||
}))
|
||||
# Send current LED settings
|
||||
self.write_message(json.dumps({
|
||||
"action": "set_led",
|
||||
"r": state.led_r,
|
||||
"g": state.led_g,
|
||||
"b": state.led_b,
|
||||
"mode": state.led_mode
|
||||
}))
|
||||
except Exception as e:
|
||||
sys.stderr.write(f"[Web Server] Error syncing state: {e}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
def on_close(self):
|
||||
ClientWebSocketHandler.clients.remove(self)
|
||||
sys.stderr.write("[Web Server] Client disconnected\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
def on_message(self, message):
|
||||
try:
|
||||
data = json.loads(message)
|
||||
event = data.get("event")
|
||||
|
||||
if event == "sensor_update":
|
||||
state.sensor_temp = data.get("temperature", 22.5)
|
||||
state.sensor_hum = data.get("humidity", 45.0)
|
||||
state.sensor_light = data.get("light", 350.0)
|
||||
|
||||
elif event == "voice_recording":
|
||||
rec_id = data.get("recording_id")
|
||||
if rec_id in state.active_recordings:
|
||||
fut = state.active_recordings[rec_id]
|
||||
if not fut.done():
|
||||
error_msg = data.get("error")
|
||||
if error_msg:
|
||||
fut.set_exception(RuntimeError(error_msg))
|
||||
else:
|
||||
fut.set_result(data.get("audio_base64"))
|
||||
|
||||
elif event == "ping":
|
||||
# Respond to browser ping with pong latency calculation
|
||||
self.write_message(json.dumps({"event": "pong", "t": data.get("t")}))
|
||||
|
||||
except Exception as e:
|
||||
sys.stderr.write(f"[Web Server] Error processing message: {e}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
@classmethod
|
||||
def broadcast(cls, payload):
|
||||
msg = json.dumps(payload)
|
||||
for client in cls.clients:
|
||||
try:
|
||||
client.write_message(msg)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Send responses back to the LLM Client via stdio
|
||||
def send_rpc_response(resp):
|
||||
payload = json.dumps(resp) + "\n"
|
||||
mcp_stdout.write(payload)
|
||||
mcp_stdout.flush()
|
||||
|
||||
# Get local LAN IP address
|
||||
def get_local_ip():
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
try:
|
||||
s.connect(('10.255.255.255', 1))
|
||||
IP = s.getsockname()[0]
|
||||
except Exception:
|
||||
IP = '127.0.0.1'
|
||||
finally:
|
||||
s.close()
|
||||
return IP
|
||||
|
||||
# Define the list of MCP tools
|
||||
def get_mcp_tools_list():
|
||||
return [
|
||||
{
|
||||
"name": "clear_screen",
|
||||
"description": "Clear the virtual screen. Optionally specify color (e.g. standard hex '#1e1e2e', name 'white', or 0 for white / 1 for black).",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"color": {"type": "string", "description": "Color hex, CSS name, or integer (0=white, 1=black)"}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "draw_text",
|
||||
"description": "Draw text at specified (x,y) coordinates on the virtual display.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"text": {"type": "string", "description": "The text message to draw"},
|
||||
"x": {"type": "integer", "description": "X coordinate (0-780)"},
|
||||
"y": {"type": "integer", "description": "Y coordinate (0-580)"},
|
||||
"size": {"type": "integer", "description": "Font size in pixels (default 24)"},
|
||||
"color": {"type": "string", "description": "Text color (default 'white')"}
|
||||
},
|
||||
"required": ["text", "x", "y"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "draw_shape",
|
||||
"description": "Draw a geometric shape (rectangle, circle, or line) on the virtual display.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"shape": {"type": "string", "enum": ["rect", "circle", "line"], "description": "Type of shape"},
|
||||
"x": {"type": "integer", "description": "Start X coordinate"},
|
||||
"y": {"type": "integer", "description": "Start Y coordinate"},
|
||||
"width": {"type": "integer", "description": "Width (or delta X for line)"},
|
||||
"height": {"type": "integer", "description": "Height (or delta Y for line)"},
|
||||
"color": {"type": "string", "description": "Color of the outline or fill (default 'white')"},
|
||||
"fill": {"type": "boolean", "description": "Whether to fill the shape (default false)"}
|
||||
},
|
||||
"required": ["shape", "x", "y", "width", "height"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "draw_image",
|
||||
"description": "Draw a base64 encoded PNG or JPEG image on the virtual screen at (x,y).",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"image_base64": {"type": "string", "description": "Base64 encoded string of the image file"},
|
||||
"x": {"type": "integer", "description": "X coordinate (default 0)", "default": 0},
|
||||
"y": {"type": "integer", "description": "Y coordinate (default 0)", "default": 0},
|
||||
"width": {"type": "integer", "description": "Width to scale the image (optional)"},
|
||||
"height": {"type": "integer", "description": "Height to scale the image (optional)"}
|
||||
},
|
||||
"required": ["image_base64"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "get_screenshot",
|
||||
"description": "Capture the current virtual display buffer and return it as a PNG image, allowing you to see what is currently rendered on the screen.",
|
||||
"inputSchema": {"type": "object", "properties": {}}
|
||||
},
|
||||
{
|
||||
"name": "set_led",
|
||||
"description": "Control the virtual RGB LED ring lights.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"r": {"type": "integer", "minimum": 0, "maximum": 255, "description": "Red channel (0-255)"},
|
||||
"g": {"type": "integer", "minimum": 0, "maximum": 255, "description": "Green channel (0-255)"},
|
||||
"b": {"type": "integer", "minimum": 0, "maximum": 255, "description": "Blue channel (0-255)"},
|
||||
"mode": {"type": "string", "enum": ["static", "breath", "rainbow", "off"], "description": "LED animation mode"}
|
||||
},
|
||||
"required": ["r", "g", "b", "mode"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "get_sensors",
|
||||
"description": "Read telemetry values from virtual temperature, humidity, and ambient light sensors.",
|
||||
"inputSchema": {"type": "object", "properties": {}}
|
||||
},
|
||||
{
|
||||
"name": "play_tone",
|
||||
"description": "Play a sound tone of a specific frequency and duration on the virtual speaker.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"frequency": {"type": "integer", "description": "Frequency in Hz (default 440)"},
|
||||
"duration_ms": {"type": "integer", "description": "Duration in milliseconds (default 1000)"},
|
||||
"volume": {"type": "integer", "minimum": 0, "maximum": 100, "description": "Volume percent from 0 to 100 (default 50)"}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "play_audio",
|
||||
"description": "Play a base64 encoded audio track (like WAV or MP3) or an audio URL on the virtual speaker.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"audio_base64_or_url": {"type": "string", "description": "Base64 audio bytes or accessible audio file URL"},
|
||||
"volume": {"type": "integer", "minimum": 0, "maximum": 100, "description": "Volume percent from 0 to 100 (default 50)"}
|
||||
},
|
||||
"required": ["audio_base64_or_url"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "record_voice",
|
||||
"description": "Record voice commands/input from the connected client's microphone for a specified duration.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"duration_sec": {"type": "integer", "minimum": 1, "maximum": 20, "description": "Duration to record in seconds (default 4)", "default": 4}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
# Execute tool operations
|
||||
async def execute_mcp_tool(name, args):
|
||||
if name == "clear_screen":
|
||||
color = args.get("color", 0)
|
||||
state.clear_screen(color)
|
||||
ClientWebSocketHandler.broadcast({"action": "clear", "color": color})
|
||||
return "Screen cleared successfully."
|
||||
|
||||
elif name == "draw_text":
|
||||
text = str(args.get("text", ""))
|
||||
x = int(args.get("x", 10))
|
||||
y = int(args.get("y", 10))
|
||||
size = int(args.get("size", 24))
|
||||
color = str(args.get("color", "white"))
|
||||
|
||||
state.draw_text(text, x, y, size, color)
|
||||
ClientWebSocketHandler.broadcast({
|
||||
"action": "draw_text",
|
||||
"text": text,
|
||||
"x": x,
|
||||
"y": y,
|
||||
"size": size,
|
||||
"color": color
|
||||
})
|
||||
return f"Successfully drew text at ({x}, {y})."
|
||||
|
||||
elif name == "draw_shape":
|
||||
shape = str(args.get("shape"))
|
||||
x = int(args.get("x"))
|
||||
y = int(args.get("y"))
|
||||
w = int(args.get("width"))
|
||||
h = int(args.get("height"))
|
||||
color = str(args.get("color", "white"))
|
||||
fill = bool(args.get("fill", False))
|
||||
|
||||
state.draw_shape(shape, x, y, w, h, color, fill)
|
||||
ClientWebSocketHandler.broadcast({
|
||||
"action": "draw_shape",
|
||||
"shape": shape,
|
||||
"x": x,
|
||||
"y": y,
|
||||
"w": w,
|
||||
"h": h,
|
||||
"color": color,
|
||||
"fill": fill
|
||||
})
|
||||
return f"Successfully drew {shape} shape."
|
||||
|
||||
elif name == "draw_image":
|
||||
img_b64 = str(args.get("image_base64"))
|
||||
x = int(args.get("x", 0))
|
||||
y = int(args.get("y", 0))
|
||||
w = args.get("width")
|
||||
h = args.get("height")
|
||||
|
||||
if w is not None: w = int(w)
|
||||
if h is not None: h = int(h)
|
||||
|
||||
state.draw_image(img_b64, x, y, w, h)
|
||||
ClientWebSocketHandler.broadcast({
|
||||
"action": "draw_image",
|
||||
"image_base64": img_b64,
|
||||
"x": x,
|
||||
"y": y,
|
||||
"w": w,
|
||||
"h": h
|
||||
})
|
||||
return "Successfully rendered image on display."
|
||||
|
||||
elif name == "get_screenshot":
|
||||
b64_png = state.get_screenshot_base64()
|
||||
return [
|
||||
{
|
||||
"type": "image",
|
||||
"data": b64_png,
|
||||
"mimeType": "image/png"
|
||||
}
|
||||
]
|
||||
|
||||
elif name == "set_led":
|
||||
r = int(args.get("r", 0))
|
||||
g = int(args.get("g", 0))
|
||||
b = int(args.get("b", 0))
|
||||
mode = str(args.get("mode", "static"))
|
||||
|
||||
state.led_r, state.led_g, state.led_b, state.led_mode = r, g, b, mode
|
||||
ClientWebSocketHandler.broadcast({
|
||||
"action": "set_led",
|
||||
"r": r,
|
||||
"g": g,
|
||||
"b": b,
|
||||
"mode": mode
|
||||
})
|
||||
return f"NeoPixel LED ring set to '{mode}' mode with color ({r}, {g}, {b})."
|
||||
|
||||
elif name == "get_sensors":
|
||||
return json.dumps({
|
||||
"temperature_c": state.sensor_temp,
|
||||
"humidity_pct": state.sensor_hum,
|
||||
"light_lux": state.sensor_light
|
||||
})
|
||||
|
||||
elif name == "play_tone":
|
||||
freq = int(args.get("frequency", 440))
|
||||
dur = int(args.get("duration_ms", 1000))
|
||||
vol = int(args.get("volume", 50))
|
||||
|
||||
ClientWebSocketHandler.broadcast({
|
||||
"action": "play_tone",
|
||||
"frequency": freq,
|
||||
"duration": dur,
|
||||
"volume": vol / 100.0
|
||||
})
|
||||
return f"Played synthesized tone at {freq}Hz for {dur}ms."
|
||||
|
||||
elif name == "play_audio":
|
||||
audio_payload = str(args.get("audio_base64_or_url"))
|
||||
vol = int(args.get("volume", 50))
|
||||
|
||||
ClientWebSocketHandler.broadcast({
|
||||
"action": "play_audio",
|
||||
"audio": audio_payload,
|
||||
"volume": vol / 100.0
|
||||
})
|
||||
return "Successfully transmitted audio buffer."
|
||||
|
||||
elif name == "record_voice":
|
||||
dur = int(args.get("duration_sec", 4))
|
||||
|
||||
if not ClientWebSocketHandler.clients:
|
||||
return "No active browser sessions to capture microphone input. Recording aborted."
|
||||
|
||||
rec_id = str(uuid.uuid4())
|
||||
loop = asyncio.get_running_loop()
|
||||
fut = loop.create_future()
|
||||
state.active_recordings[rec_id] = fut
|
||||
|
||||
ClientWebSocketHandler.broadcast({
|
||||
"action": "record_voice",
|
||||
"duration_sec": dur,
|
||||
"recording_id": rec_id
|
||||
})
|
||||
|
||||
try:
|
||||
# Wait for client recording with a safety buffer
|
||||
audio_b64 = await asyncio.wait_for(fut, timeout=dur + 5.0)
|
||||
audio_bytes = base64.b64decode(audio_b64)
|
||||
|
||||
# Save the WAV file to local disk
|
||||
filename = f"recording_{rec_id[:8]}.wav"
|
||||
with open(filename, "wb") as f:
|
||||
f.write(audio_bytes)
|
||||
|
||||
return f"Successfully recorded {dur} seconds of audio. File saved to '{filename}'."
|
||||
except asyncio.TimeoutError:
|
||||
return "Microphone recording timed out. Make sure the connected web client has granted mic permissions."
|
||||
except Exception as e:
|
||||
return f"Microphone recording failed: {str(e)}"
|
||||
finally:
|
||||
state.active_recordings.pop(rec_id, None)
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unknown MCP tool: {name}")
|
||||
|
||||
# Handles parsed MCP stdin request objects
|
||||
async def handle_mcp_request(req):
|
||||
method = req.get("method")
|
||||
rpc_id = req.get("id")
|
||||
|
||||
is_notification = rpc_id is None
|
||||
|
||||
if method == "initialize":
|
||||
resp = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": rpc_id,
|
||||
"result": {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": {
|
||||
"tools": {}
|
||||
},
|
||||
"serverInfo": {
|
||||
"name": "virtual-screen-mcp",
|
||||
"version": "1.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
send_rpc_response(resp)
|
||||
return
|
||||
|
||||
elif method == "tools/list":
|
||||
resp = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": rpc_id,
|
||||
"result": {
|
||||
"tools": get_mcp_tools_list()
|
||||
}
|
||||
}
|
||||
send_rpc_response(resp)
|
||||
return
|
||||
|
||||
elif method == "tools/call":
|
||||
params = req.get("params", {})
|
||||
tool_name = params.get("name")
|
||||
arguments = params.get("arguments", {})
|
||||
|
||||
try:
|
||||
result = await execute_mcp_tool(tool_name, arguments)
|
||||
if isinstance(result, list):
|
||||
content = result
|
||||
else:
|
||||
content = [{"type": "text", "text": str(result)}]
|
||||
|
||||
resp = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": rpc_id,
|
||||
"result": {
|
||||
"content": content
|
||||
}
|
||||
}
|
||||
except Exception as e:
|
||||
resp = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": rpc_id,
|
||||
"error": {
|
||||
"code": -32000,
|
||||
"message": str(e)
|
||||
}
|
||||
}
|
||||
send_rpc_response(resp)
|
||||
return
|
||||
|
||||
if not is_notification:
|
||||
resp = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": rpc_id,
|
||||
"error": {
|
||||
"code": -32601,
|
||||
"message": f"Method {method} not found"
|
||||
}
|
||||
}
|
||||
send_rpc_response(resp)
|
||||
|
||||
# Read stdin line-by-line asynchronously
|
||||
async def stdio_reader_loop():
|
||||
loop = asyncio.get_running_loop()
|
||||
reader = asyncio.StreamReader()
|
||||
protocol = asyncio.StreamReaderProtocol(reader)
|
||||
await loop.connect_read_pipe(lambda: protocol, sys.stdin)
|
||||
|
||||
while True:
|
||||
line = await reader.readline()
|
||||
if not line:
|
||||
break
|
||||
|
||||
try:
|
||||
req = json.loads(line.decode('utf-8').strip())
|
||||
await handle_mcp_request(req)
|
||||
except Exception as e:
|
||||
sys.stderr.write(f"[MCP Stdio] Error handling line input: {e}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
# Strong references to prevent garbage collection of background tasks
|
||||
background_tasks = set()
|
||||
|
||||
# Async main runtime loop
|
||||
async def main_async(port):
|
||||
app = tornado.web.Application([
|
||||
(r"/", MainHandler),
|
||||
(r"/ws", ClientWebSocketHandler),
|
||||
], template_path=os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
app.listen(port)
|
||||
|
||||
local_ip = get_local_ip()
|
||||
sys.stderr.write(f"--------------------------------------------------\n")
|
||||
sys.stderr.write(f"Virtual Screen & Speaker MCP Server Initialized.\n")
|
||||
sys.stderr.write(f"Connect local devices in your browser to:\n")
|
||||
sys.stderr.write(f"==> http://{local_ip}:{port}/\n")
|
||||
sys.stderr.write(f"==> http://localhost:{port}/\n")
|
||||
sys.stderr.write(f"--------------------------------------------------\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
# Spawn the stdin line reader task and store a strong reference
|
||||
task = asyncio.create_task(stdio_reader_loop())
|
||||
background_tasks.add(task)
|
||||
task.add_done_callback(background_tasks.discard)
|
||||
|
||||
# Run indefinitely
|
||||
while True:
|
||||
await asyncio.sleep(3600)
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Virtual Screen & Speakers MCP Server")
|
||||
parser.add_argument("--port", type=int, default=8080, help="Web port (default 8080)")
|
||||
args = parser.parse_args()
|
||||
|
||||
# Set the MCP stdio channel
|
||||
global mcp_stdout
|
||||
mcp_stdout = sys.stdout
|
||||
# Route default stdout to stderr so prints don't interrupt standard JSON-RPC communications
|
||||
sys.stdout = sys.stderr
|
||||
|
||||
try:
|
||||
asyncio.run(main_async(args.port))
|
||||
except KeyboardInterrupt:
|
||||
sys.stderr.write("[Server] Stopped by KeyboardInterrupt.\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env python3
|
||||
import subprocess
|
||||
import json
|
||||
import base64
|
||||
import time
|
||||
import os
|
||||
|
||||
def run_test():
|
||||
print("Starting server.py in a subprocess for stdio test...")
|
||||
# Start server in subprocess on port 8089 to avoid port conflicts
|
||||
proc = subprocess.Popen(
|
||||
["python3", "server.py", "--port", "8089"],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL, # ignore logs on stderr
|
||||
text=True,
|
||||
bufsize=1
|
||||
)
|
||||
|
||||
# Wait a moment for server to bind
|
||||
time.sleep(1.5)
|
||||
|
||||
def send_cmd(cmd_dict):
|
||||
proc.stdin.write(json.dumps(cmd_dict) + "\n")
|
||||
proc.stdin.flush()
|
||||
line = proc.stdout.readline()
|
||||
if not line:
|
||||
return None
|
||||
return json.loads(line.strip())
|
||||
|
||||
print("\n[1/4] Sending initialize request...")
|
||||
init_resp = send_cmd({
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "initialize",
|
||||
"params": {}
|
||||
})
|
||||
print("Response:", json.dumps(init_resp, indent=2))
|
||||
|
||||
print("\n[2/4] Sending clear_screen tool call...")
|
||||
resp = send_cmd({
|
||||
"jsonrpc": "2.0",
|
||||
"id": 2,
|
||||
"method": "tools/call",
|
||||
"params": {
|
||||
"name": "clear_screen",
|
||||
"arguments": {"color": "#1e1e2e"}
|
||||
}
|
||||
})
|
||||
print("Response:", json.dumps(resp, indent=2))
|
||||
|
||||
print("\n[3/4] Sending draw_text tool call...")
|
||||
resp = send_cmd({
|
||||
"jsonrpc": "2.0",
|
||||
"id": 3,
|
||||
"method": "tools/call",
|
||||
"params": {
|
||||
"name": "draw_text",
|
||||
"arguments": {
|
||||
"text": "TEST SUCCEEDED",
|
||||
"x": 200,
|
||||
"y": 280,
|
||||
"size": 48,
|
||||
"color": "#10b981"
|
||||
}
|
||||
}
|
||||
})
|
||||
print("Response:", json.dumps(resp, indent=2))
|
||||
|
||||
print("\n[4/4] Sending get_screenshot tool call...")
|
||||
resp = send_cmd({
|
||||
"jsonrpc": "2.0",
|
||||
"id": 4,
|
||||
"method": "tools/call",
|
||||
"params": {
|
||||
"name": "get_screenshot",
|
||||
"arguments": {}
|
||||
}
|
||||
})
|
||||
|
||||
try:
|
||||
content = resp["result"]["content"][0]
|
||||
if content["type"] == "image":
|
||||
img_b64 = content["data"]
|
||||
output_file = "test_screenshot.png"
|
||||
with open(output_file, "wb") as f:
|
||||
f.write(base64.b64decode(img_b64))
|
||||
print(f"Success! Captured screenshot saved to: {os.path.abspath(output_file)}")
|
||||
else:
|
||||
print("Failed: content is not an image", content)
|
||||
except Exception as e:
|
||||
print("Error reading screenshot response:", e)
|
||||
print("Raw response:", resp)
|
||||
|
||||
# Stop subprocess
|
||||
proc.terminate()
|
||||
proc.wait()
|
||||
print("\nVerification script finished.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_test()
|
||||
Reference in New Issue
Block a user