Adolfo Reyna
1481 lines
61 KiB
Python
1481 lines
61 KiB
Python
#!/usr/bin/env python3
|
|
import sys
|
|
import os
|
|
import json
|
|
import socket
|
|
import time
|
|
import threading
|
|
import math
|
|
import subprocess
|
|
import tempfile
|
|
import struct
|
|
import base64
|
|
import argparse
|
|
import urllib.request
|
|
import http.server
|
|
import re
|
|
from PIL import Image, ImageDraw, ImageFont, ImageOps
|
|
|
|
# Try importing pygame, but allow headless run if missing
|
|
try:
|
|
import pygame
|
|
PYGAME_AVAILABLE = True
|
|
except ImportError:
|
|
PYGAME_AVAILABLE = False
|
|
|
|
# Port Configuration Defaults
|
|
HTTP_PORT = 8080
|
|
TCP_STREAM_PORT = 8081
|
|
UDP_STREAM_PORT = 8082
|
|
UDP_DISCOVERY_PORT = 5000
|
|
|
|
# Constants for Reflective LCD (RLCD) Palette
|
|
COLOR_BG_RGB = (180, 195, 174) # Reflective LCD light green-grey
|
|
COLOR_FG_RGB = (26, 36, 22) # Dark green-black ink
|
|
COLOR_FRAME_RGB = (15, 23, 42) # Dark bezel
|
|
COLOR_WINDOW_RGB = (220, 220, 220) # Light grey window background
|
|
|
|
class NativeCompanionClient:
|
|
def __init__(self, width=480, height=320, http_port=HTTP_PORT):
|
|
self.width = width
|
|
self.height = height
|
|
self.http_port = http_port
|
|
|
|
# Thread lock for canvas buffer safety
|
|
self.canvas_lock = threading.Lock()
|
|
|
|
# Visual Buffer (1-bit monochrome PIL image: 0=bg/white, 1=fg/black)
|
|
self.canvas_img = Image.new("1", (self.width, self.height), 0)
|
|
self.draw = ImageDraw.Draw(self.canvas_img)
|
|
|
|
# Video stream and performance stats
|
|
self.fps_frame_count = 0
|
|
self.fps_start_time = time.time()
|
|
self.current_fps = 0.0
|
|
self.tcp_bytes_received = 0
|
|
self.udp_packets_received = 0
|
|
self.udp_frames_complete = 0
|
|
self.dropped_udp_frames = 0
|
|
self.is_streaming_active = False
|
|
self.last_stream_packet_time = 0
|
|
|
|
# LED state
|
|
self.led_r = 0
|
|
self.led_g = 0
|
|
self.led_b = 0
|
|
self.led_mode = "off" # static, breath, rainbow, off
|
|
|
|
# Audio / recording state
|
|
self.audio_status = "Idle"
|
|
self.mic_status = "Idle"
|
|
|
|
# Start Time for Uptime
|
|
self.start_time = time.time()
|
|
self.override_active = False
|
|
|
|
# Native stdout handler for stdio MCP
|
|
self.mcp_stdout = sys.stdout
|
|
# Redirect default prints to stderr so they don't break JSON-RPC over stdout
|
|
sys.stdout = sys.stderr
|
|
|
|
def get_screenshot_path(self, filename):
|
|
"""Returns a writeable path for saving screenshots, falling back to local folder if needed."""
|
|
mac_path = "/Users/adolforeyna/.gemini/antigravity/brain/4acf0d08-1256-4c7a-a2e4-3e8576b58c73"
|
|
if os.path.exists(mac_path):
|
|
return os.path.join(mac_path, filename)
|
|
# Fallback for Linux / Pi 5
|
|
local_path = os.path.join(os.path.expanduser("~"), "Projects", "Screen")
|
|
if os.path.exists(local_path):
|
|
return os.path.join(local_path, filename)
|
|
return os.path.join(os.getcwd(), filename)
|
|
|
|
def bring_to_front(self):
|
|
"""Focuses/raises the Pygame window on macOS using System Events."""
|
|
if sys.platform == "darwin":
|
|
try:
|
|
cmd = """osascript -e 'tell application "System Events" to set frontmost of every process whose name is "Python" to true'"""
|
|
subprocess.Popen(cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
except Exception:
|
|
pass
|
|
|
|
def draw_local_dashboard_canvas(self, battery_text, temp_text, uptime_text):
|
|
"""Draws the companion dashboard onto the 1-bit PIL canvas image."""
|
|
self.draw.rectangle([0, 0, self.width, self.height], fill=0)
|
|
|
|
font_large = ImageFont.load_default()
|
|
font_normal = ImageFont.load_default()
|
|
try:
|
|
# Monospace look
|
|
font_large = ImageFont.truetype("Courier New", 18)
|
|
font_normal = ImageFont.truetype("Courier New", 12)
|
|
except:
|
|
pass
|
|
|
|
# Draw header
|
|
self.draw.text((15, 10), "DESKTOP MCP COMPANION", fill=1, font=font_large)
|
|
self.draw.line((10, 32, 470, 32), fill=1)
|
|
|
|
# Sections
|
|
self.draw.text((15, 45), "HOST TELEMETRY", fill=1, font=font_large)
|
|
t_val = temp_text.split(": ")[1] if ": " in temp_text else temp_text
|
|
self.draw.text((25, 70), f"CPU Temp : {t_val}", fill=1, font=font_normal)
|
|
u_val = uptime_text.split(": ")[1] if ": " in uptime_text else uptime_text
|
|
self.draw.text((25, 85), f"Uptime : {u_val}", fill=1, font=font_normal)
|
|
|
|
self.draw.line((10, 110, 470, 110), fill=1)
|
|
|
|
self.draw.text((15, 120), "MCP ENDPOINTS", fill=1, font=font_large)
|
|
local_ip = self.get_local_ip()
|
|
self.draw.text((25, 145), f"IP Address : {local_ip}", fill=1, font=font_normal)
|
|
self.draw.text((25, 160), f"HTTP API : {self.http_port} /api/mcp", fill=1, font=font_normal)
|
|
self.draw.text((25, 175), f"Video Ports: 8081 (TCP) / 8082 (UDP)", fill=1, font=font_normal)
|
|
|
|
self.draw.line((10, 200, 470, 200), fill=1)
|
|
|
|
self.draw.text((15, 210), "SYSTEM STATUS", fill=1, font=font_large)
|
|
b_val = battery_text.split(": ")[1] if ": " in battery_text else battery_text
|
|
self.draw.text((25, 235), f"Battery : {b_val}", fill=1, font=font_normal)
|
|
time_str = time.strftime("%Y-%m-%d %H:%M:%S")
|
|
self.draw.text((25, 250), f"Local Time : {time_str}", fill=1, font=font_normal)
|
|
|
|
self.draw.line((10, 275, 470, 275), fill=1)
|
|
|
|
status_msg = f"Speaker: {self.audio_status} | Mic: {self.mic_status}"
|
|
self.draw.text((15, 290), f"Status : {status_msg}", fill=1, font=font_normal)
|
|
|
|
def run_pygame_loop(self):
|
|
"""Starts and runs the Pygame GUI main thread loop."""
|
|
pygame.init()
|
|
pygame.font.init()
|
|
|
|
# Display resolution (960x640 canvas with bezel = 1000x680 window)
|
|
screen_w = 1000
|
|
screen_h = 680
|
|
|
|
# Start hidden initially as requested
|
|
self.window_visible = False
|
|
self.show_window_requested = False
|
|
screen = pygame.display.set_mode((screen_w, screen_h), pygame.HIDDEN)
|
|
pygame.display.set_caption("Native Agent Companion Client")
|
|
|
|
# Use default system font or built-in font
|
|
font = pygame.font.Font(None, 22)
|
|
title_font = pygame.font.Font(None, 28)
|
|
|
|
clock = pygame.time.Clock()
|
|
|
|
# Internal stats query timers
|
|
last_telemetry_time = 0
|
|
battery_text = "Battery: Querying..."
|
|
temp_text = "CPU Temperature: Querying..."
|
|
uptime_text = "Uptime: 00:00:00"
|
|
volts = 4.20
|
|
pct = 100
|
|
temp = 38.5
|
|
|
|
auto_screenshot_done = False
|
|
running = True
|
|
while running:
|
|
# Check if a message was received and we need to make the window visible
|
|
if getattr(self, "show_window_requested", False):
|
|
self.show_window_requested = False
|
|
if not self.window_visible:
|
|
# Switch display to SHOWN mode
|
|
screen = pygame.display.set_mode((screen_w, screen_h), pygame.SHOWN)
|
|
self.window_visible = True
|
|
self.bring_to_front()
|
|
|
|
# 1. Process Pygame Window Events
|
|
for event in pygame.event.get():
|
|
if event.type == pygame.QUIT:
|
|
running = False
|
|
elif event.type == pygame.KEYDOWN:
|
|
if event.key == pygame.K_d:
|
|
self.override_active = False
|
|
# Force immediately
|
|
uptime_sec = int(time.time() - self.start_time)
|
|
hrs = uptime_sec // 3600
|
|
mins = (uptime_sec % 3600) // 60
|
|
secs = uptime_sec % 60
|
|
uptime_text = f"Uptime: {hrs:02d}:{mins:02d}:{secs:02d}"
|
|
volts, pct = self.get_host_battery()
|
|
battery_text = f"Battery: {volts:.2f}V ({pct}%)"
|
|
temp = self.get_host_cpu_temp()
|
|
temp_text = f"CPU Temperature: {temp:.1f}°C"
|
|
with self.canvas_lock:
|
|
self.draw_local_dashboard_canvas(battery_text, temp_text, uptime_text)
|
|
elif event.key == pygame.K_s:
|
|
try:
|
|
save_path = self.get_screenshot_path("pygame_screen.png")
|
|
pygame.image.save(screen, save_path)
|
|
sys.stderr.write(f"Manual screen capture saved to {save_path}.\n")
|
|
sys.stderr.flush()
|
|
except Exception as ex:
|
|
sys.stderr.write(f"Manual screen capture failed: {ex}\n")
|
|
sys.stderr.flush()
|
|
|
|
now = time.time()
|
|
|
|
# 2. Update telemetry stats once per second
|
|
if now - last_telemetry_time >= 1.0:
|
|
last_telemetry_time = now
|
|
uptime_sec = int(now - self.start_time)
|
|
hrs = uptime_sec // 3600
|
|
mins = (uptime_sec % 3600) // 60
|
|
secs = uptime_sec % 60
|
|
uptime_text = f"Uptime: {hrs:02d}:{mins:02d}:{secs:02d}"
|
|
|
|
volts, pct = self.get_host_battery()
|
|
battery_text = f"Battery: {volts:.2f}V ({pct}%)"
|
|
|
|
temp = self.get_host_cpu_temp()
|
|
temp_text = f"CPU Temperature: {temp:.1f}°C"
|
|
|
|
# Check video streaming inactivity
|
|
if self.is_streaming_active and (now - self.last_stream_packet_time) > 3.0:
|
|
self.is_streaming_active = False
|
|
self.current_fps = 0.0
|
|
|
|
# Update dashboard on canvas if not overridden
|
|
if not self.override_active and not self.is_streaming_active:
|
|
with self.canvas_lock:
|
|
self.draw_local_dashboard_canvas(battery_text, temp_text, uptime_text)
|
|
|
|
# 3. Clear window with dark background
|
|
screen.fill(COLOR_WINDOW_RGB)
|
|
|
|
# 4. Draw Reflective LCD Screen Bezel Frame
|
|
# Left side bezel box (width 964, height 644) at (15, 15)
|
|
pygame.draw.rect(screen, COLOR_FRAME_RGB, (15, 15, 970, 650), border_radius=12)
|
|
|
|
# 5. Render LCD Canvas Buffer
|
|
try:
|
|
# Thread-safe copy of canvas pixels using fast colorization
|
|
with self.canvas_lock:
|
|
gray_img = self.canvas_img.convert("L")
|
|
colored_img = ImageOps.colorize(gray_img, COLOR_BG_RGB, COLOR_FG_RGB)
|
|
|
|
# Scale up to 2x (960x640)
|
|
scaled_img = colored_img.resize((960, 640), Image.NEAREST)
|
|
img_bytes = scaled_img.tobytes("raw", "RGB")
|
|
|
|
# Blit to screen
|
|
lcd_surface = pygame.image.frombytes(img_bytes, (960, 640), "RGB")
|
|
screen.blit(lcd_surface, (20, 20))
|
|
except Exception as e:
|
|
# Print rendering failures to stderr
|
|
sys.stderr.write(f"GUI Canvas Render Fail: {e}\n")
|
|
sys.stderr.flush()
|
|
|
|
# Auto screenshot at 3 seconds after window becomes visible
|
|
if self.window_visible and not auto_screenshot_done:
|
|
if not hasattr(self, "visible_start_time"):
|
|
self.visible_start_time = time.time()
|
|
elif time.time() - self.visible_start_time >= 3.0:
|
|
try:
|
|
save_path = self.get_screenshot_path("pygame_screen_auto.png")
|
|
pygame.image.save(screen, save_path)
|
|
sys.stderr.write(f"Auto screen capture saved to {save_path}.\n")
|
|
sys.stderr.flush()
|
|
except Exception as ex:
|
|
sys.stderr.write(f"Auto screen capture failed: {ex}\n")
|
|
sys.stderr.flush()
|
|
auto_screenshot_done = True
|
|
|
|
# 6. Draw LED Indicator Ring (on the window border)
|
|
led_color = (71, 85, 105) # Off (dark grey)
|
|
if self.led_mode == "static":
|
|
led_color = (self.led_r, self.led_g, self.led_b)
|
|
elif self.led_mode == "breath":
|
|
brightness = 0.25 + 0.75 * (math.sin(now * 4.0) + 1.0) / 2.0
|
|
led_color = (int(self.led_r * brightness), int(self.led_g * brightness), int(self.led_b * brightness))
|
|
elif self.led_mode == "rainbow":
|
|
hue = (now * 0.5) % 1.0
|
|
led_color = self.hsl_to_rgb(hue, 1.0, 0.5)
|
|
|
|
# Draw tiny status LED on the window frame border
|
|
pygame.draw.circle(screen, led_color, (990, 30), 5)
|
|
|
|
# 8. Render updates
|
|
pygame.display.flip()
|
|
clock.tick(30) # Lock to 30 FPS for rendering updates
|
|
|
|
pygame.quit()
|
|
os._exit(0)
|
|
|
|
def hsl_to_rgb(self, h, s, l):
|
|
def hue_2_rgb(p, q, t):
|
|
if t < 0: t += 1
|
|
if t > 1: t -= 1
|
|
if t < 1/6: return p + (q - p) * 6 * t
|
|
if t < 1/2: return q
|
|
if t < 2/3: return p + (q - p) * (2/3 - t) * 6
|
|
return p
|
|
if s == 0:
|
|
r = g = b = l
|
|
else:
|
|
q = l * (1 + s) if l < 0.5 else l + s - l * s
|
|
p = 2 * l - q
|
|
r = hue_2_rgb(p, q, h + 1/3)
|
|
g = hue_2_rgb(p, q, h)
|
|
b = hue_2_rgb(p, q, h - 1/3)
|
|
return int(r * 255), int(g * 255), int(b * 255)
|
|
|
|
# --- HOST HARDWARE QUERIES ---
|
|
|
|
def get_host_battery(self):
|
|
"""Native battery status extraction for macOS, Linux, and Windows."""
|
|
if sys.platform == "darwin": # macOS
|
|
try:
|
|
output = subprocess.check_output(["pmset", "-g", "batt"]).decode("utf-8")
|
|
pct_match = re.search(r"(\d+)%", output)
|
|
pct = int(pct_match.group(1)) if pct_match else 100
|
|
volts = round(3.5 + (pct / 100.0) * 0.7, 3)
|
|
return volts, pct
|
|
except Exception:
|
|
pass
|
|
elif sys.platform.startswith("linux"): # Linux
|
|
try:
|
|
pct = 100
|
|
# Check typical battery path
|
|
cap_path = "/sys/class/power_supply/BAT0/capacity"
|
|
if not os.path.exists(cap_path):
|
|
cap_path = "/sys/class/power_supply/BAT1/capacity"
|
|
if os.path.exists(cap_path):
|
|
with open(cap_path, "r") as f:
|
|
pct = int(f.read().strip())
|
|
|
|
vol_path = "/sys/class/power_supply/BAT0/voltage_now"
|
|
if not os.path.exists(vol_path):
|
|
vol_path = "/sys/class/power_supply/BAT1/voltage_now"
|
|
volts = None
|
|
if os.path.exists(vol_path):
|
|
with open(vol_path, "r") as f:
|
|
volts = int(f.read().strip()) / 1_000_000.0 # microvolts to volts
|
|
if volts is None:
|
|
volts = round(3.5 + (pct / 100.0) * 0.7, 3)
|
|
return volts, pct
|
|
except Exception:
|
|
pass
|
|
elif sys.platform == "win32": # Windows
|
|
try:
|
|
output = subprocess.check_output("wmic Path Win32_Battery Get EstimatedChargeRemaining", shell=True).decode("utf-8")
|
|
pct = 100
|
|
numbers = re.findall(r"\d+", output)
|
|
if numbers:
|
|
pct = int(numbers[0])
|
|
volts = round(3.5 + (pct / 100.0) * 0.7, 3)
|
|
return volts, pct
|
|
except Exception:
|
|
pass
|
|
# Fallback
|
|
return 4.20, 100
|
|
|
|
def get_host_cpu_temp(self):
|
|
"""Dynamic temperature estimate based on thermal zone or CPU load."""
|
|
if sys.platform.startswith("linux"):
|
|
try:
|
|
for path in ["/sys/class/thermal/thermal_zone0/temp", "/sys/class/thermal/thermal_zone1/temp"]:
|
|
if os.path.exists(path):
|
|
with open(path, "r") as f:
|
|
return round(int(f.read().strip()) / 1000.0, 1)
|
|
except Exception:
|
|
pass
|
|
|
|
# Dynamic fallback matching CPU load average
|
|
try:
|
|
if hasattr(os, "getloadavg"):
|
|
load = os.getloadavg()[0]
|
|
else:
|
|
load = 0.15
|
|
temp = 36.0 + min(30.0, load * 8.0)
|
|
return round(temp, 1)
|
|
except Exception:
|
|
return 38.5
|
|
|
|
def get_local_ip(self):
|
|
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
try:
|
|
s.connect(('10.255.255.255', 1))
|
|
IP = s.getsockname()[0]
|
|
except Exception:
|
|
IP = '127.0.0.1'
|
|
finally:
|
|
s.close()
|
|
return IP
|
|
|
|
# --- AUDIO INPUT/OUTPUT ENGINES ---
|
|
|
|
def play_wav_native(self, filepath, volume=50):
|
|
"""Launches native player asynchronously."""
|
|
self.audio_status = f"Playing WAV ({volume}%)"
|
|
|
|
# Native Windows Sound module
|
|
if sys.platform == "win32":
|
|
try:
|
|
import winsound
|
|
def run_winsound():
|
|
try:
|
|
winsound.PlaySound(filepath, winsound.SND_FILENAME)
|
|
finally:
|
|
self.audio_status = "Idle"
|
|
threading.Thread(target=run_winsound, daemon=True).start()
|
|
return True
|
|
except Exception as e:
|
|
sys.stderr.write(f"winsound failed: {e}\n")
|
|
sys.stderr.flush()
|
|
self.audio_status = "Error"
|
|
return False
|
|
|
|
# MacOS / Linux CLI launchers
|
|
cmd = None
|
|
if sys.platform == "darwin":
|
|
# afplay supports volume scaling (0.0 to 1.0)
|
|
vol_factor = volume / 100.0
|
|
cmd = ["afplay", "-v", str(vol_factor), filepath]
|
|
elif sys.platform.startswith("linux"):
|
|
vol_factor = volume / 100.0
|
|
for player in ["paplay", "pw-play", "aplay"]:
|
|
try:
|
|
subprocess.run(["which", player], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
if player == "paplay":
|
|
pa_vol = int((volume / 100.0) * 65536)
|
|
cmd = ["paplay", "--volume", str(pa_vol), filepath]
|
|
elif player == "pw-play":
|
|
cmd = ["pw-play", "--volume", str(vol_factor), filepath]
|
|
else:
|
|
cmd = ["aplay", filepath]
|
|
break
|
|
except subprocess.CalledProcessError:
|
|
continue
|
|
|
|
if cmd:
|
|
def run_player():
|
|
try:
|
|
subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
except Exception as ex:
|
|
sys.stderr.write(f"Audio command failed: {ex}\n")
|
|
sys.stderr.flush()
|
|
finally:
|
|
self.audio_status = "Idle"
|
|
|
|
threading.Thread(target=run_player, daemon=True).start()
|
|
return True
|
|
else:
|
|
sys.stderr.write("No supported command line audio player found.\n")
|
|
sys.stderr.flush()
|
|
self.audio_status = "Idle"
|
|
return False
|
|
|
|
def generate_and_play_tone(self, freq, duration_ms, volume):
|
|
self.audio_status = f"Playing {freq}Hz Tone"
|
|
sample_rate = 16000
|
|
num_samples = int(sample_rate * (duration_ms / 1000.0))
|
|
volume_scale = int((volume / 100.0) * 32767)
|
|
|
|
# Generate raw 16-bit PCM mono sine wave
|
|
pcm_data = bytearray()
|
|
for i in range(num_samples):
|
|
val = int(volume_scale * math.sin(2 * math.pi * freq * i / sample_rate))
|
|
pcm_data.extend(struct.pack("<h", val))
|
|
|
|
# Standard WAV Header
|
|
num_channels = 1
|
|
bits_per_sample = 16
|
|
byte_rate = sample_rate * num_channels * (bits_per_sample // 8)
|
|
block_align = num_channels * (bits_per_sample // 8)
|
|
subchunk2_size = num_samples * block_align
|
|
chunk_size = 36 + subchunk2_size
|
|
|
|
header = struct.pack(
|
|
"<4sI4s4sIHHIIHH4sI",
|
|
b"RIFF", chunk_size, b"WAVE",
|
|
b"fmt ", 16, 1, num_channels, sample_rate, byte_rate, block_align, bits_per_sample,
|
|
b"data", subchunk2_size
|
|
)
|
|
|
|
temp_fd, temp_path = tempfile.mkstemp(suffix=".wav")
|
|
with os.fdopen(temp_fd, "wb") as f:
|
|
f.write(header)
|
|
f.write(pcm_data)
|
|
|
|
# Play asynchronously
|
|
self.play_wav_native(temp_path, volume)
|
|
|
|
# Schedule deletion of temp file after tone finishes
|
|
threading.Timer((duration_ms + 1000) / 1000.0, lambda: self.safe_delete(temp_path)).start()
|
|
|
|
def record_voice(self, duration_sec, filename):
|
|
self.mic_status = f"Recording ({duration_sec}s)"
|
|
threading.Thread(target=self._run_record_thread, args=(duration_sec, filename), daemon=True).start()
|
|
return True
|
|
|
|
def _run_record_thread(self, duration_sec, filename):
|
|
try:
|
|
# 1. Try ALSA arecord on Linux
|
|
if sys.platform.startswith("linux"):
|
|
try:
|
|
subprocess.run([
|
|
"arecord",
|
|
"-d", str(duration_sec),
|
|
"-f", "S16_LE",
|
|
"-r", "16000",
|
|
"-c", "2",
|
|
"-t", "wav",
|
|
filename
|
|
], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
return
|
|
except Exception as ex:
|
|
sys.stderr.write(f"arecord failed: {ex}. Trying next method...\n")
|
|
sys.stderr.flush()
|
|
|
|
# 2. Try sounddevice
|
|
try:
|
|
import sounddevice as sd
|
|
import numpy as np
|
|
import scipy.io.wavfile as wav
|
|
sample_rate = 16000
|
|
|
|
# Try 2 channels first (Stereo)
|
|
try:
|
|
rec = sd.rec(int(duration_sec * sample_rate), samplerate=sample_rate, channels=2, dtype='int16')
|
|
sd.wait()
|
|
wav.write(filename, sample_rate, rec)
|
|
return
|
|
except Exception as ex2:
|
|
sys.stderr.write(f"sounddevice stereo recording failed: {ex2}. Trying mono...\n")
|
|
sys.stderr.flush()
|
|
# Try 1 channel (Mono)
|
|
rec = sd.rec(int(duration_sec * sample_rate), samplerate=sample_rate, channels=1, dtype='int16')
|
|
sd.wait()
|
|
# Convert mono array (shape N,1 or N) to stereo (shape N,2) by replicating
|
|
if rec.ndim == 1:
|
|
stereo_rec = np.column_stack((rec, rec))
|
|
else:
|
|
stereo_rec = np.column_stack((rec[:, 0], rec[:, 0]))
|
|
wav.write(filename, sample_rate, stereo_rec)
|
|
return
|
|
except ImportError:
|
|
pass
|
|
except Exception as ex:
|
|
sys.stderr.write(f"sounddevice method failed: {ex}. Trying next method...\n")
|
|
sys.stderr.flush()
|
|
|
|
# 3. Try pyaudio
|
|
try:
|
|
import pyaudio
|
|
import wave
|
|
p = pyaudio.PyAudio()
|
|
|
|
# Try 2 channels
|
|
try:
|
|
stream = p.open(format=pyaudio.paInt16, channels=2, rate=16000, input=True, frames_per_buffer=1024)
|
|
frames = []
|
|
for _ in range(0, int(16000 / 1024 * duration_sec)):
|
|
data = stream.read(1024, exception_on_overflow=False)
|
|
frames.append(data)
|
|
stream.stop_stream()
|
|
stream.close()
|
|
p.terminate()
|
|
|
|
wf = wave.open(filename, 'wb')
|
|
wf.setnchannels(2)
|
|
wf.setsampwidth(2)
|
|
wf.setframerate(16000)
|
|
wf.writeframes(b''.join(frames))
|
|
wf.close()
|
|
return
|
|
except Exception as ep2:
|
|
sys.stderr.write(f"pyaudio stereo failed: {ep2}. Trying mono...\n")
|
|
sys.stderr.flush()
|
|
# Try 1 channel
|
|
stream = p.open(format=pyaudio.paInt16, channels=1, rate=16000, input=True, frames_per_buffer=1024)
|
|
frames = []
|
|
for _ in range(0, int(16000 / 1024 * duration_sec)):
|
|
mono_data = stream.read(1024, exception_on_overflow=False)
|
|
# Replicate mono samples (2 bytes each) to stereo
|
|
stereo_frames = bytearray()
|
|
for i in range(0, len(mono_data), 2):
|
|
sample = mono_data[i:i+2]
|
|
stereo_frames.extend(sample)
|
|
stereo_frames.extend(sample)
|
|
frames.append(bytes(stereo_frames))
|
|
stream.stop_stream()
|
|
stream.close()
|
|
p.terminate()
|
|
|
|
wf = wave.open(filename, 'wb')
|
|
wf.setnchannels(2)
|
|
wf.setsampwidth(2)
|
|
wf.setframerate(16000)
|
|
wf.writeframes(b''.join(frames))
|
|
wf.close()
|
|
return
|
|
except ImportError:
|
|
pass
|
|
except Exception as ex:
|
|
sys.stderr.write(f"pyaudio method failed: {ex}. Trying next method...\n")
|
|
sys.stderr.flush()
|
|
|
|
# 4. Mock fallback
|
|
sys.stderr.write("No working recording hardware found. Simulating recording...\n")
|
|
sys.stderr.flush()
|
|
time.sleep(duration_sec)
|
|
sample_rate = 16000
|
|
num_samples = int(sample_rate * duration_sec)
|
|
pcm_data = bytearray()
|
|
import random
|
|
for i in range(num_samples):
|
|
hum = int(800 * math.sin(2 * math.pi * 220 * i / sample_rate))
|
|
noise = random.randint(-150, 150)
|
|
val = max(-32768, min(32767, hum + noise))
|
|
pcm_data.extend(struct.pack("<hh", val, val))
|
|
|
|
num_channels = 2
|
|
bits_per_sample = 16
|
|
byte_rate = sample_rate * num_channels * (bits_per_sample // 8)
|
|
block_align = num_channels * (bits_per_sample // 8)
|
|
subchunk2_size = num_samples * block_align
|
|
chunk_size = 36 + subchunk2_size
|
|
|
|
header = struct.pack(
|
|
"<4sI4s4sIHHIIHH4sI",
|
|
b"RIFF", chunk_size, b"WAVE",
|
|
b"fmt ", 16, 1, num_channels, sample_rate, byte_rate, block_align, bits_per_sample,
|
|
b"data", subchunk2_size
|
|
)
|
|
with open(filename, "wb") as f:
|
|
f.write(header)
|
|
f.write(pcm_data)
|
|
except Exception as e:
|
|
sys.stderr.write(f"Background recording thread crashed: {e}\n")
|
|
sys.stderr.flush()
|
|
finally:
|
|
self.mic_status = "Idle"
|
|
|
|
def safe_delete(self, filepath):
|
|
try:
|
|
if os.path.exists(filepath):
|
|
os.remove(filepath)
|
|
except Exception:
|
|
pass
|
|
|
|
# --- VIDEO DECODING ENGINE (RLCD layout decoding) ---
|
|
|
|
def rlcd_to_mono(self, rlcd_buf):
|
|
"""Converts raw 15,000-byte RLCD buffer into standard 480x320 monochrome canvas bytes (19,200 bytes)."""
|
|
canvas_buf = bytearray(19200)
|
|
X_OFF = 40
|
|
Y_OFF = 10
|
|
|
|
for index in range(15000):
|
|
val = rlcd_buf[index]
|
|
if val == 0:
|
|
continue
|
|
|
|
byte_x = index // 75
|
|
block_y = index % 75
|
|
x_base = 2 * byte_x + X_OFF
|
|
y_base = 299 - 4 * block_y + Y_OFF
|
|
|
|
# bit 7
|
|
if val & 0x80:
|
|
cx, cy = x_base, y_base
|
|
canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7)))
|
|
# bit 6
|
|
if val & 0x40:
|
|
cx, cy = x_base + 1, y_base
|
|
canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7)))
|
|
# bit 5
|
|
if val & 0x20:
|
|
cx, cy = x_base, y_base - 1
|
|
canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7)))
|
|
# bit 4
|
|
if val & 0x10:
|
|
cx, cy = x_base + 1, y_base - 1
|
|
canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7)))
|
|
# bit 3
|
|
if val & 0x08:
|
|
cx, cy = x_base, y_base - 2
|
|
canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7)))
|
|
# bit 2
|
|
if val & 0x04:
|
|
cx, cy = x_base + 1, y_base - 2
|
|
canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7)))
|
|
# bit 1
|
|
if val & 0x02:
|
|
cx, cy = x_base, y_base - 3
|
|
canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7)))
|
|
# bit 0
|
|
if val & 0x01:
|
|
cx, cy = x_base + 1, y_base - 3
|
|
canvas_buf[cy * 60 + (cx >> 3)] |= (1 << (7 - (cx & 7)))
|
|
|
|
return canvas_buf
|
|
|
|
def process_rlcd_frame(self, rlcd_frame_bytes):
|
|
"""Converts incoming raw RLCD bytes to PIL and triggers GUI refresh."""
|
|
self.override_active = True
|
|
self.show_window_requested = True
|
|
canvas_bytes = self.rlcd_to_mono(rlcd_frame_bytes)
|
|
|
|
# Load directly into our Visual Buffer (thread-safe lock)
|
|
with self.canvas_lock:
|
|
img = Image.frombytes("1", (self.width, self.height), bytes(canvas_bytes))
|
|
self.canvas_img.paste(img)
|
|
|
|
# Calculate dynamic FPS
|
|
now = time.time()
|
|
self.fps_frame_count += 1
|
|
elapsed = now - self.fps_start_time
|
|
if elapsed >= 1.0:
|
|
self.current_fps = self.fps_frame_count / elapsed
|
|
self.fps_frame_count = 0
|
|
self.fps_start_time = now
|
|
|
|
if not self.is_streaming_active:
|
|
self.bring_to_front()
|
|
|
|
self.is_streaming_active = True
|
|
self.last_stream_packet_time = now
|
|
|
|
# --- MCP SERVER LOGIC ---
|
|
|
|
def handle_mcp_request(self, req):
|
|
"""Processes an incoming JSON-RPC dictionary and executes the matching tool."""
|
|
rpc_id = req.get("id")
|
|
method = req.get("method")
|
|
params = req.get("params", {})
|
|
|
|
if method == "initialize":
|
|
return {
|
|
"jsonrpc": "2.0",
|
|
"id": rpc_id,
|
|
"result": {
|
|
"protocolVersion": "2024-11-05",
|
|
"capabilities": {"tools": {}},
|
|
"serverInfo": {
|
|
"name": "desktop-companion-client",
|
|
"version": "1.0.0"
|
|
}
|
|
}
|
|
}
|
|
|
|
elif method == "tools/list":
|
|
return {
|
|
"jsonrpc": "2.0",
|
|
"id": rpc_id,
|
|
"result": {
|
|
"tools": self.get_mcp_tools_list()
|
|
}
|
|
}
|
|
|
|
elif method == "tools/call":
|
|
tool_name = params.get("name")
|
|
arguments = params.get("arguments", {})
|
|
try:
|
|
res_content = self.execute_tool(tool_name, arguments)
|
|
if isinstance(res_content, list):
|
|
content = res_content
|
|
else:
|
|
content = [{"type": "text", "text": str(res_content)}]
|
|
return {
|
|
"jsonrpc": "2.0",
|
|
"id": rpc_id,
|
|
"result": {"content": content}
|
|
}
|
|
except Exception as e:
|
|
return {
|
|
"jsonrpc": "2.0",
|
|
"id": rpc_id,
|
|
"error": {
|
|
"code": -32000,
|
|
"message": str(e)
|
|
}
|
|
}
|
|
|
|
return {
|
|
"jsonrpc": "2.0",
|
|
"id": rpc_id,
|
|
"error": {
|
|
"code": -32601,
|
|
"message": f"Method {method} not found"
|
|
}
|
|
}
|
|
|
|
def get_mcp_tools_list(self):
|
|
return [
|
|
{
|
|
"name": "clear_screen",
|
|
"description": "Clear the 480x320 canvas screen to white (0) or black (1).",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"color": {"type": "integer", "enum": [0, 1], "description": "0 = White, 1 = Black"}
|
|
},
|
|
"required": ["color"]
|
|
}
|
|
},
|
|
{
|
|
"name": "draw_text",
|
|
"description": "Draw text on the canvas screen at specified (x,y) coordinates.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"text": {"type": "string", "description": "The message to display"},
|
|
"x": {"type": "integer", "description": "X coordinate (0-470)"},
|
|
"y": {"type": "integer", "description": "Y coordinate (0-310)"},
|
|
"size": {"type": "integer", "enum": [1, 2], "description": "Text scale (1=normal, 2=large)"}
|
|
},
|
|
"required": ["text", "x", "y"]
|
|
}
|
|
},
|
|
{
|
|
"name": "set_led",
|
|
"description": "Control the visual RGB LED. Animates breathes/rainbows dynamically.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"r": {"type": "integer", "minimum": 0, "maximum": 255, "description": "Red (0-255)"},
|
|
"g": {"type": "integer", "minimum": 0, "maximum": 255, "description": "Green (0-255)"},
|
|
"b": {"type": "integer", "minimum": 0, "maximum": 255, "description": "Blue (0-255)"},
|
|
"mode": {"type": "string", "enum": ["static", "breath", "rainbow", "off"], "description": "LED animation mode"}
|
|
},
|
|
"required": ["r", "g", "b", "mode"]
|
|
}
|
|
},
|
|
{
|
|
"name": "get_battery",
|
|
"description": "Read the host computer's actual battery voltage and estimated capacity percentage.",
|
|
"inputSchema": {"type": "object", "properties": {}}
|
|
},
|
|
{
|
|
"name": "get_sensors",
|
|
"description": "Read telemetry values representing the host computer's CPU temperature and relative indoor humidity.",
|
|
"inputSchema": {"type": "object", "properties": {}}
|
|
},
|
|
{
|
|
"name": "scan_ble",
|
|
"description": "Scan for nearby Bluetooth Low Energy devices using the host controller.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"duration_ms": {"type": "integer", "description": "Scan duration in milliseconds (default 3000)"}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "get_screenshot",
|
|
"description": "Capture the current LCD display canvas rendering as a PNG image.",
|
|
"inputSchema": {"type": "object", "properties": {}}
|
|
},
|
|
{
|
|
"name": "draw_image",
|
|
"description": "Draw an image (PNG, JPEG, GIF, BMP, etc.) on the canvas. The image is converted to 1-bit monochrome and placed on the coordinates.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"image_base64": {"type": "string", "description": "Base64 encoded string of the source image file"},
|
|
"x": {"type": "integer", "description": "X coordinate (default 0)", "default": 0},
|
|
"y": {"type": "integer", "description": "Y coordinate (default 0)", "default": 0},
|
|
"dither": {"type": "boolean", "description": "Whether to dither (default true)", "default": True}
|
|
},
|
|
"required": ["image_base64"]
|
|
}
|
|
},
|
|
{
|
|
"name": "write_file",
|
|
"description": "Write a text file directly to the client's storage folder.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"path": {"type": "string", "description": "The destination file path (e.g. 'test.txt')"},
|
|
"content": {"type": "string", "description": "The content to write"}
|
|
},
|
|
"required": ["path", "content"]
|
|
}
|
|
},
|
|
{
|
|
"name": "read_file",
|
|
"description": "Read a text file from the client's storage folder.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"path": {"type": "string", "description": "The file path to read"}
|
|
},
|
|
"required": ["path"]
|
|
}
|
|
},
|
|
{
|
|
"name": "execute_python",
|
|
"description": "Execute arbitrary Python code dynamically on the host machine. Returns prints and errors.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"code": {"type": "string", "description": "The Python code to execute"}
|
|
},
|
|
"required": ["code"]
|
|
}
|
|
},
|
|
{
|
|
"name": "play_tone",
|
|
"description": "Play a pure sine wave tone on the host computer's speakers.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"frequency": {"type": "integer", "description": "Frequency in Hz (default 440)", "default": 440},
|
|
"duration_ms": {"type": "integer", "description": "Duration in milliseconds (default 1000)", "default": 1000},
|
|
"volume": {"type": "integer", "description": "Volume from 0 to 100 (default 50)", "default": 50}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "play_audio",
|
|
"description": "Play a WAV audio file on the host computer's speakers.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"filename": {"type": "string", "description": "WAV file path to play"},
|
|
"volume": {"type": "integer", "description": "Volume from 0 to 100 (default 50)", "default": 50}
|
|
},
|
|
"required": ["filename"]
|
|
}
|
|
},
|
|
{
|
|
"name": "record_voice",
|
|
"description": "Record voice command from the computer microphone to a WAV file.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"duration_sec": {"type": "integer", "description": "Duration in seconds (default 4)", "default": 4},
|
|
"filename": {"type": "string", "description": "Destination WAV file (default 'recording.wav')", "default": "recording.wav"}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "play_audio_base64",
|
|
"description": "Play a base64 encoded WAV file directly on the host computer's speakers.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"volume": {"type": "integer", "description": "Volume from 0 to 100 (default 50)", "default": 50},
|
|
"wav_base64": {"type": "string", "description": "Base64 encoded string of the WAV audio file"},
|
|
},
|
|
"required": [
|
|
"wav_base64"
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "download_file",
|
|
"description": "Download a file from a URL to the local storage.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"filename": {"type": "string", "description": "The destination filename"},
|
|
"url": {"type": "string", "description": "The URL of the file to download"}
|
|
},
|
|
"required": [
|
|
"url",
|
|
"filename"
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "get_video_streaming_instructions",
|
|
"description": "Get instructions to stream video directly into the companion canvas.",
|
|
"inputSchema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"protocol": {"type": "string", "enum": ["tcp", "udp", "both"], "description": "The protocol to query (default 'both')"}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"name": "get_stream_stats",
|
|
"description": "Get real-time performance and frame-rate statistics for the video stream.",
|
|
"inputSchema": {"type": "object", "properties": {}}
|
|
}
|
|
]
|
|
|
|
def execute_tool(self, name, args):
|
|
"""Local tool implementation executing on the host PC."""
|
|
if name == "clear_screen":
|
|
color = int(args.get("color", 0))
|
|
self.override_active = True
|
|
self.show_window_requested = True
|
|
with self.canvas_lock:
|
|
self.draw.rectangle([0, 0, self.width, self.height], fill=color)
|
|
self.bring_to_front()
|
|
return "Screen cleared."
|
|
|
|
elif name == "draw_text":
|
|
text = str(args.get("text", ""))
|
|
x = int(args.get("x", 10))
|
|
y = int(args.get("y", 10))
|
|
size = int(args.get("size", 1))
|
|
self.override_active = True
|
|
self.show_window_requested = True
|
|
|
|
# Draw on PIL image using standard font size
|
|
font_size = 12 if size == 1 else 24
|
|
font = ImageFont.load_default()
|
|
try:
|
|
# Try loading standard monospace font for neat look
|
|
font = ImageFont.truetype("Courier New", font_size)
|
|
except:
|
|
try:
|
|
font = ImageFont.truetype("Arial", font_size)
|
|
except:
|
|
pass
|
|
with self.canvas_lock:
|
|
self.draw.text((x, y), text, fill=1, font=font)
|
|
self.bring_to_front()
|
|
return f"Drew text '{text}' at ({x}, {y})."
|
|
|
|
elif name == "set_led":
|
|
r = int(args.get("r", 0))
|
|
g = int(args.get("g", 0))
|
|
b = int(args.get("b", 0))
|
|
mode = str(args.get("mode", "static"))
|
|
|
|
self.led_r, self.led_g, self.led_b, self.led_mode = r, g, b, mode
|
|
return f"LED status set to {mode} ({r}, {g}, {b})."
|
|
|
|
elif name == "get_battery":
|
|
volts, pct = self.get_host_battery()
|
|
return json.dumps({"voltage_v": volts, "percentage_pct": pct})
|
|
|
|
elif name == "get_sensors":
|
|
temp = self.get_host_cpu_temp()
|
|
# Return indoor humidity mock
|
|
return json.dumps({"temperature_c": temp, "humidity_pct": 45.0})
|
|
|
|
elif name == "scan_ble":
|
|
dur = int(args.get("duration_ms", 3000))
|
|
return json.dumps(scan_ble_devices(dur))
|
|
|
|
elif name == "get_screenshot":
|
|
# Capture the current 480x320 PIL visual buffer, save as PNG in memory, base64 encode
|
|
import io
|
|
bg_color = (180, 195, 174)
|
|
fg_color = (26, 36, 22)
|
|
|
|
with self.canvas_lock:
|
|
gray_img = self.canvas_img.convert("L")
|
|
colored_img = ImageOps.colorize(gray_img, bg_color, fg_color)
|
|
|
|
buffered = io.BytesIO()
|
|
colored_img.save(buffered, format="PNG")
|
|
b64_png = base64.b64encode(buffered.getvalue()).decode("utf-8")
|
|
|
|
return [
|
|
{
|
|
"type": "image",
|
|
"data": b64_png,
|
|
"mimeType": "image/png"
|
|
}
|
|
]
|
|
|
|
elif name == "draw_image":
|
|
pbm_b64 = args.get("pbm_base64")
|
|
image_base64 = args.get("image_base64")
|
|
x = int(args.get("x", 0))
|
|
y = int(args.get("y", 0))
|
|
self.override_active = True
|
|
self.show_window_requested = True
|
|
|
|
if pbm_b64:
|
|
pbm_bytes = base64.b64decode(pbm_b64)
|
|
import io
|
|
img = Image.open(io.BytesIO(pbm_bytes))
|
|
with self.canvas_lock:
|
|
self.canvas_img.paste(img, (x, y))
|
|
elif image_base64:
|
|
if "," in image_base64:
|
|
image_base64 = image_base64.split(",")[1]
|
|
img_bytes = base64.b64decode(image_base64)
|
|
import io
|
|
img = Image.open(io.BytesIO(img_bytes))
|
|
dither = args.get("dither", True)
|
|
img_1bit = img.convert("1", dither=Image.FLOYDSTEINBERG if dither else Image.NONE)
|
|
with self.canvas_lock:
|
|
self.canvas_img.paste(img_1bit, (x, y))
|
|
else:
|
|
raise ValueError("Missing image data (pbm_base64 or image_base64)")
|
|
|
|
self.bring_to_front()
|
|
return "Image rendered successfully on canvas."
|
|
|
|
elif name == "write_file":
|
|
path = str(args.get("path"))
|
|
content = str(args.get("content"))
|
|
# Save locally
|
|
with open(path, 'w') as f:
|
|
f.write(content)
|
|
return f"Wrote {len(content)} characters to local file '{path}'."
|
|
|
|
elif name == "read_file":
|
|
path = str(args.get("path"))
|
|
with open(path, 'r') as f:
|
|
return f.read()
|
|
|
|
elif name == "execute_python":
|
|
code = str(args.get("code"))
|
|
import builtins
|
|
import sys
|
|
import io
|
|
|
|
output_buffer = []
|
|
old_print = builtins.print
|
|
|
|
def custom_print(*args, **kwargs):
|
|
sep = kwargs.get('sep', ' ')
|
|
end = kwargs.get('end', '\n')
|
|
str_args = [str(arg) for arg in args]
|
|
msg = sep.join(str_args) + end
|
|
output_buffer.append(msg)
|
|
|
|
builtins.print = custom_print
|
|
error = None
|
|
try:
|
|
exec(code, globals())
|
|
except Exception as e:
|
|
import traceback
|
|
error_msg = traceback.format_exc()
|
|
output_buffer.append(f"\nExecution Failed:\n{error_msg}")
|
|
error = e
|
|
finally:
|
|
builtins.print = old_print
|
|
|
|
output = "".join(output_buffer)
|
|
if error:
|
|
return output
|
|
return f"Execution Succeeded. Console output:\n{output}"
|
|
|
|
elif name == "play_tone":
|
|
freq = int(args.get("frequency", 440))
|
|
duration = int(args.get("duration_ms", 1000))
|
|
vol = int(args.get("volume", 50))
|
|
|
|
self.generate_and_play_tone(freq, duration, vol)
|
|
return f"Played synthesized tone at {freq}Hz for {duration}ms (vol={vol})."
|
|
|
|
elif name == "play_audio":
|
|
filename = str(args.get("filename"))
|
|
vol = int(args.get("volume", 50))
|
|
|
|
success = self.play_wav_native(filename, vol)
|
|
if success:
|
|
return f"Playing audio file '{filename}'."
|
|
else:
|
|
raise RuntimeError(f"Failed to play audio file '{filename}'.")
|
|
|
|
elif name == "record_voice":
|
|
duration = int(args.get("duration_sec", 4))
|
|
filename = str(args.get("filename", "recording.wav"))
|
|
|
|
success = self.record_voice(duration, filename)
|
|
if success:
|
|
return f"Voice recording active. Saving to '{filename}' ({duration} seconds)."
|
|
else:
|
|
raise RuntimeError("Failed to trigger audio recording.")
|
|
|
|
elif name == "play_audio_base64":
|
|
b64_data = args.get("wav_base64")
|
|
vol = int(args.get("volume", 50))
|
|
if not b64_data:
|
|
raise ValueError("Missing wav_base64 parameter.")
|
|
|
|
audio_bytes = base64.b64decode(b64_data)
|
|
temp_fd, temp_path = tempfile.mkstemp(suffix=".wav")
|
|
with os.fdopen(temp_fd, "wb") as f:
|
|
f.write(audio_bytes)
|
|
|
|
success = self.play_wav_native(temp_path, vol)
|
|
# Schedule file deletion
|
|
threading.Timer(5.0, lambda: self.safe_delete(temp_path)).start()
|
|
|
|
if success:
|
|
return "Playing base64 decoded audio stream."
|
|
else:
|
|
raise RuntimeError("Failed to play audio stream.")
|
|
|
|
elif name == "download_file":
|
|
url = str(args.get("url"))
|
|
filename = str(args.get("filename"))
|
|
|
|
# Simple HTTP download
|
|
urllib.request.urlretrieve(url, filename)
|
|
return f"Successfully downloaded file to local storage: '{filename}'"
|
|
|
|
elif name == "get_video_streaming_instructions":
|
|
instructions = [
|
|
"### Native Client Companion Video Streaming Instructions",
|
|
"Display Area: 400x300 (drawn centered on the 480x320 physical screen), 1-bit monochrome.",
|
|
"To stream, connect to local network services:",
|
|
"1. **TCP (Port 8081)**: Open a TCP socket connection and stream consecutive 15,000-byte frame blocks.",
|
|
"2. **UDP (Port 8082)**: Stream 1,002-byte datagrams: byte 0 = frame_id, byte 1 = chunk_idx (0..14), bytes 2..1001 = payload."
|
|
]
|
|
return "\n".join(instructions)
|
|
|
|
elif name == "get_stream_stats":
|
|
return json.dumps({
|
|
"frames_drawn": self.fps_frame_count,
|
|
"tcp_bytes_received": self.tcp_bytes_received,
|
|
"udp_packets_received": self.udp_packets_received,
|
|
"udp_frames_complete": self.udp_frames_complete,
|
|
"dropped_udp_frames": self.dropped_udp_frames,
|
|
"last_fps": round(self.current_fps, 1),
|
|
"active": self.is_streaming_active
|
|
})
|
|
|
|
else:
|
|
raise ValueError(f"Unknown tool: {name}")
|
|
|
|
def scan_ble_devices(duration_ms):
|
|
"""Scans for nearby BLE tags. Uses Bleak if installed, falls back to simulated beacons."""
|
|
try:
|
|
import asyncio
|
|
from bleak import BleakScanner
|
|
|
|
async def run_scan():
|
|
scanner = BleakScanner()
|
|
devices = await scanner.discover(timeout=duration_ms/1000.0)
|
|
result = {}
|
|
for d in devices:
|
|
result[d.address] = {"rssi": d.rssi, "name": d.name or "Unknown"}
|
|
return result
|
|
|
|
try:
|
|
loop = asyncio.new_event_loop()
|
|
asyncio.set_event_loop(loop)
|
|
res = loop.run_until_complete(run_scan())
|
|
loop.close()
|
|
return res
|
|
except Exception:
|
|
pass
|
|
except ImportError:
|
|
pass
|
|
|
|
# Simulated Beacons Fallback
|
|
import random
|
|
mock_tags = [
|
|
{"mac": "C8:FD:19:30:A5:11", "name": "Desk Tracker Tag"},
|
|
{"mac": "E5:31:02:8C:F4:9A", "name": "Companion Keyring"},
|
|
{"mac": "A4:38:DE:1C:B0:02", "name": "BLE Temp Beacon"},
|
|
]
|
|
result = {}
|
|
for tag in mock_tags:
|
|
result[tag["mac"]] = {"rssi": random.randint(-85, -45), "name": tag["name"]}
|
|
return result
|
|
|
|
# --- NETWORK BACKGROUND SERVICES ---
|
|
|
|
def run_http_server(app, port):
|
|
"""Runs a background HTTP server providing /api/mcp endpoint parity."""
|
|
class HTTPHandler(http.server.BaseHTTPRequestHandler):
|
|
def log_message(self, format, *args):
|
|
pass # Suppress logging to stdout
|
|
|
|
def do_OPTIONS(self):
|
|
self.send_response(204)
|
|
self.send_header("Access-Control-Allow-Origin", "*")
|
|
self.send_header("Access-Control-Allow-Methods", "POST, GET, OPTIONS")
|
|
self.send_header("Access-Control-Allow-Headers", "Content-Type")
|
|
self.end_headers()
|
|
|
|
def do_POST(self):
|
|
if self.path == "/api/mcp":
|
|
content_length = int(self.headers.get('Content-Length', 0))
|
|
body = self.rfile.read(content_length).decode('utf-8')
|
|
try:
|
|
req_data = json.loads(body)
|
|
resp_data = app.handle_mcp_request(req_data)
|
|
body_bytes = json.dumps(resp_data).encode('utf-8')
|
|
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Access-Control-Allow-Origin", "*")
|
|
self.send_header("Content-Length", len(body_bytes))
|
|
self.end_headers()
|
|
self.wfile.write(body_bytes)
|
|
except Exception as e:
|
|
self.send_response(500)
|
|
self.end_headers()
|
|
self.wfile.write(str(e).encode('utf-8'))
|
|
else:
|
|
self.send_response(404)
|
|
self.end_headers()
|
|
|
|
httpd = http.server.HTTPServer(('', port), HTTPHandler)
|
|
sys.stderr.write(f"[HTTP Service] Listening on port {port}...\n")
|
|
sys.stderr.flush()
|
|
httpd.serve_forever()
|
|
|
|
def run_tcp_video_server(app, port):
|
|
"""Listens on port 8081 for incoming raw 15,000-byte RLCD video frame streams."""
|
|
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
server.bind(('', port))
|
|
server.listen(1)
|
|
|
|
sys.stderr.write(f"[TCP Video] Listening on port {port}...\n")
|
|
sys.stderr.flush()
|
|
|
|
while True:
|
|
try:
|
|
client, addr = server.accept()
|
|
sys.stderr.write(f"[TCP Video] Connection accepted from {addr}\n")
|
|
sys.stderr.flush()
|
|
|
|
frame_buffer = bytearray(15000)
|
|
view = memoryview(frame_buffer)
|
|
bytes_received = 0
|
|
|
|
while True:
|
|
remaining = 15000 - bytes_received
|
|
n = client.recv_into(view[bytes_received : bytes_received + remaining])
|
|
if not n:
|
|
break # disconnected
|
|
bytes_received += n
|
|
app.tcp_bytes_received += n
|
|
|
|
if bytes_received == 15000:
|
|
app.process_rlcd_frame(frame_buffer)
|
|
bytes_received = 0
|
|
client.close()
|
|
except Exception as e:
|
|
sys.stderr.write(f"[TCP Video] Error: {e}\n")
|
|
sys.stderr.flush()
|
|
time.sleep(1)
|
|
|
|
def run_udp_video_server(app, port):
|
|
"""Listens on port 8082 for fragmented RLCD UDP frame packages."""
|
|
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
sock.bind(('', port))
|
|
|
|
sys.stderr.write(f"[UDP Video] Listening on port {port}...\n")
|
|
sys.stderr.flush()
|
|
|
|
frame_buffer = bytearray(15000)
|
|
current_frame_id = -1
|
|
chunks_mask = 0
|
|
temp_pack = bytearray(1002)
|
|
|
|
while True:
|
|
try:
|
|
n, addr = sock.recvfrom_into(temp_pack)
|
|
if n < 2:
|
|
continue
|
|
|
|
app.udp_packets_received += 1
|
|
frame_id = temp_pack[0]
|
|
chunk_idx = temp_pack[1]
|
|
|
|
if chunk_idx < 15:
|
|
if frame_id != current_frame_id:
|
|
if chunks_mask != 0:
|
|
app.dropped_udp_frames += 1
|
|
current_frame_id = frame_id
|
|
chunks_mask = 0
|
|
|
|
# Copy segment payload
|
|
start_offset = chunk_idx * 1000
|
|
frame_buffer[start_offset : start_offset + 1000] = temp_pack[2:1002]
|
|
chunks_mask |= (1 << chunk_idx)
|
|
|
|
# Check if all 15 chunks (0 to 14) are in (0x7FFF)
|
|
if chunks_mask == 0x7FFF:
|
|
app.udp_frames_complete += 1
|
|
app.process_rlcd_frame(frame_buffer)
|
|
chunks_mask = 0
|
|
except Exception as e:
|
|
sys.stderr.write(f"[UDP Video] Error: {e}\n")
|
|
sys.stderr.flush()
|
|
time.sleep(0.5)
|
|
|
|
def run_udp_discovery_responder(app, port, http_port):
|
|
"""Responds to discovery broadcast messages from clients like mcp_bridge."""
|
|
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
sock.bind(('', port))
|
|
|
|
sys.stderr.write(f"[Discovery responder] Listening on port {port}...\n")
|
|
sys.stderr.flush()
|
|
|
|
while True:
|
|
try:
|
|
data, addr = sock.recvfrom(128)
|
|
if data == b"DISCOVER_SCREEN":
|
|
# Respond with matching HTTP port configuration
|
|
response = f"SCREEN_IP_{http_port}".encode('utf-8')
|
|
sock.sendto(response, addr)
|
|
except Exception as e:
|
|
sys.stderr.write(f"[Discovery responder] Error: {e}\n")
|
|
sys.stderr.flush()
|
|
time.sleep(1)
|
|
|
|
def run_stdio_mcp_server(app):
|
|
"""Reads JSON-RPC messages line-by-line from stdin, writing responses to stdout."""
|
|
sys.stderr.write("[MCP Stdio Server] Stdio listener active.\n")
|
|
sys.stderr.flush()
|
|
|
|
while True:
|
|
try:
|
|
line = sys.stdin.readline()
|
|
if not line:
|
|
break
|
|
req = json.loads(line.strip())
|
|
resp = app.handle_mcp_request(req)
|
|
|
|
# Write explicitly to raw stdout
|
|
payload = json.dumps(resp) + "\n"
|
|
app.mcp_stdout.write(payload)
|
|
app.mcp_stdout.flush()
|
|
except Exception as e:
|
|
sys.stderr.write(f"[MCP Stdio] Error handling command: {e}\n")
|
|
sys.stderr.flush()
|
|
|
|
# --- MAIN ---
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Native Desktop Companion Client")
|
|
parser.add_argument("--port", type=int, default=HTTP_PORT, help="HTTP Server port (default 8080)")
|
|
parser.add_argument("--width", type=int, default=480, help="Canvas width (default 480)")
|
|
parser.add_argument("--height", type=int, default=320, help="Canvas height (default 320)")
|
|
parser.add_argument("--headless", action="store_true", help="Run in headless mode without GUI")
|
|
args = parser.parse_args()
|
|
|
|
client = NativeCompanionClient(width=args.width, height=args.height, http_port=args.port)
|
|
|
|
# Start background servers
|
|
threading.Thread(target=run_http_server, args=(client, args.port), daemon=True).start()
|
|
threading.Thread(target=run_tcp_video_server, args=(client, TCP_STREAM_PORT), daemon=True).start()
|
|
threading.Thread(target=run_udp_video_server, args=(client, UDP_STREAM_PORT), daemon=True).start()
|
|
threading.Thread(target=run_udp_discovery_responder, args=(client, UDP_DISCOVERY_PORT, args.port), daemon=True).start()
|
|
threading.Thread(target=run_stdio_mcp_server, args=(client,), daemon=True).start()
|
|
|
|
# Run Pygame GUI if not headless and Pygame is installed
|
|
if args.headless or not PYGAME_AVAILABLE:
|
|
if not PYGAME_AVAILABLE and not args.headless:
|
|
sys.stderr.write("Warning: Pygame is not installed. Running in headless mode.\n")
|
|
sys.stderr.flush()
|
|
else:
|
|
sys.stderr.write("Headless mode active. Running background services...\n")
|
|
sys.stderr.flush()
|
|
|
|
try:
|
|
while True:
|
|
time.sleep(3600)
|
|
except KeyboardInterrupt:
|
|
sys.stderr.write("Stopping background services.\n")
|
|
sys.stderr.flush()
|
|
os._exit(0)
|
|
else:
|
|
# Start Pygame loop on main thread
|
|
client.run_pygame_loop()
|
|
|
|
if __name__ == "__main__":
|
|
main()
|