From ad30662a4810a95b9497a5b9b14aa9513556ddf2 Mon Sep 17 00:00:00 2001 From: Adolfo Reyna Date: Sun, 21 Jun 2026 21:25:55 -0400 Subject: [PATCH] Add jailbroken iPhone MCP screen app --- .gitignore | 7 +- README.md | 3 +- iphone_app/IMCAppDelegate.h | 5 + iphone_app/IMCAppDelegate.m | 23 + iphone_app/IMCCanvasView.h | 10 + iphone_app/IMCCanvasView.m | 218 ++++++ iphone_app/IMCMCPServer.h | 8 + iphone_app/IMCMCPServer.m | 163 +++++ iphone_app/IMCVideoStreamServer.h | 11 + iphone_app/IMCVideoStreamServer.m | 447 +++++++++++++ iphone_app/IMCViewController.h | 5 + iphone_app/IMCViewController.m | 846 ++++++++++++++++++++++++ iphone_app/Makefile | 13 + iphone_app/README.md | 174 +++++ iphone_app/Resources/Info.plist | 57 ++ iphone_app/control | 8 + iphone_app/iphone_mcp_bridge.py | 47 ++ iphone_app/main.m | 9 + iphone_app/manual_build_iphone.sh | 38 ++ iphone_app/tools/mock_hermes_gateway.py | 60 ++ iphone_app/tools/stream_color.py | 56 ++ 21 files changed, 2206 insertions(+), 2 deletions(-) create mode 100644 iphone_app/IMCAppDelegate.h create mode 100644 iphone_app/IMCAppDelegate.m create mode 100644 iphone_app/IMCCanvasView.h create mode 100644 iphone_app/IMCCanvasView.m create mode 100644 iphone_app/IMCMCPServer.h create mode 100644 iphone_app/IMCMCPServer.m create mode 100644 iphone_app/IMCVideoStreamServer.h create mode 100644 iphone_app/IMCVideoStreamServer.m create mode 100644 iphone_app/IMCViewController.h create mode 100644 iphone_app/IMCViewController.m create mode 100644 iphone_app/Makefile create mode 100644 iphone_app/README.md create mode 100644 iphone_app/Resources/Info.plist create mode 100644 iphone_app/control create mode 100644 iphone_app/iphone_mcp_bridge.py create mode 100644 iphone_app/main.m create mode 100644 iphone_app/manual_build_iphone.sh create mode 100644 iphone_app/tools/mock_hermes_gateway.py create mode 100644 iphone_app/tools/stream_color.py diff --git a/.gitignore b/.gitignore index 8e19916..096de21 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,12 @@ firmware/ __pycache__/ *.pyc +# iPhone app build output +iphone_app/.manual_build/ +iphone_app/.manual_pkg/ +iphone_app/.theos/ +iphone_app/packages/ + # OS generated files .DS_Store @@ -32,4 +38,3 @@ pcm2wav.py plot_audio.py *.pdf - diff --git a/README.md b/README.md index 6157410..6b6f661 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ This repository contains the MicroPython firmware and host-side bridge script to expose the **Waveshare ESP32-S3-RLCD-4.2** development board as a physical, Model Context Protocol (MCP)-compliant agentic companion. +It also includes a native jailbroken iPhone 6 target in [`iphone_app/`](iphone_app/README.md). The iPhone implementation exposes the same MCP-style screen, audio, camera, file, and monochrome streaming features, plus RGB565 color streaming and Hermes voice turns. + With this setup, local AI harnesses (such as Claude Desktop, OpenClaude, Cursor, or Hermes) can directly control the board's screen, NeoPixels, audio codec, microphone array, sensors, and remote filesystem. --- @@ -143,4 +145,3 @@ Here is a summary of the MCP tools exposed by the bridge: ## 5. Hardware & Troubleshooting Findings For detailed technical findings regarding the board's hardware (e.g. I2S bit-depth configuration, I2C bus lockup recovery, pinout configuration, and ES7210 microphone clock/OSR registers to resolve popping sound issues), refer to [hardware_findings.md](hardware_findings.md). - diff --git a/iphone_app/IMCAppDelegate.h b/iphone_app/IMCAppDelegate.h new file mode 100644 index 0000000..4c4d021 --- /dev/null +++ b/iphone_app/IMCAppDelegate.h @@ -0,0 +1,5 @@ +#import + +@interface IMCAppDelegate : UIResponder +@property (nonatomic, strong) UIWindow *window; +@end diff --git a/iphone_app/IMCAppDelegate.m b/iphone_app/IMCAppDelegate.m new file mode 100644 index 0000000..c00846a --- /dev/null +++ b/iphone_app/IMCAppDelegate.m @@ -0,0 +1,23 @@ +#import "IMCAppDelegate.h" +#import "IMCViewController.h" + +@implementation IMCAppDelegate + +- (BOOL)application:(UIApplication *)application + didFinishLaunchingWithOptions:(NSDictionary *)launchOptions { + application.idleTimerDisabled = YES; + self.window = [[UIWindow alloc] initWithFrame:[UIScreen mainScreen].bounds]; + self.window.rootViewController = [[IMCViewController alloc] init]; + [self.window makeKeyAndVisible]; + return YES; +} + +- (void)applicationDidBecomeActive:(UIApplication *)application { + application.idleTimerDisabled = YES; +} + +- (void)applicationWillTerminate:(UIApplication *)application { + application.idleTimerDisabled = NO; +} + +@end diff --git a/iphone_app/IMCCanvasView.h b/iphone_app/IMCCanvasView.h new file mode 100644 index 0000000..a0a892c --- /dev/null +++ b/iphone_app/IMCCanvasView.h @@ -0,0 +1,10 @@ +#import + +@interface IMCCanvasView : UIView +- (void)clearWithColor:(NSInteger)color; +- (void)drawText:(NSString *)text x:(CGFloat)x y:(CGFloat)y size:(NSInteger)size; +- (void)drawImageData:(NSData *)data x:(CGFloat)x y:(CGFloat)y; +- (void)displayMonochromeFrameBuffer:(NSData *)frameData; +- (BOOL)displayRGB565FrameBuffer:(NSData *)frameData width:(NSUInteger)width height:(NSUInteger)height; +- (NSString *)snapshotPNGBase64; +@end diff --git a/iphone_app/IMCCanvasView.m b/iphone_app/IMCCanvasView.m new file mode 100644 index 0000000..6c05ec5 --- /dev/null +++ b/iphone_app/IMCCanvasView.m @@ -0,0 +1,218 @@ +#import "IMCCanvasView.h" + +@interface IMCCanvasView () +@property (nonatomic, assign) NSInteger clearColor; +@property (nonatomic, strong) NSMutableArray *commands; +@property (nonatomic, strong) UIImage *streamImage; +@end + +@implementation IMCCanvasView + +- (instancetype)initWithFrame:(CGRect)frame { + self = [super initWithFrame:frame]; + if (self) { + _clearColor = 0; + _commands = [NSMutableArray array]; + self.opaque = YES; + } + return self; +} + +- (void)clearWithColor:(NSInteger)color { + self.clearColor = color == 1 ? 1 : 0; + self.streamImage = nil; + [self.commands removeAllObjects]; + [self setNeedsDisplay]; +} + +- (void)drawText:(NSString *)text x:(CGFloat)x y:(CGFloat)y size:(NSInteger)size { + if (!text) { + text = @""; + } + self.streamImage = nil; + [self.commands addObject:@{ + @"type": @"text", + @"text": text, + @"x": @(x), + @"y": @(y), + @"size": @(size == 2 ? 2 : 1) + }]; + [self setNeedsDisplay]; +} + +- (void)drawImageData:(NSData *)data x:(CGFloat)x y:(CGFloat)y { + UIImage *image = [UIImage imageWithData:data]; + if (!image) { + return; + } + self.streamImage = nil; + [self.commands addObject:@{ + @"type": @"image", + @"image": image, + @"x": @(x), + @"y": @(y) + }]; + [self setNeedsDisplay]; +} + +- (void)displayMonochromeFrameBuffer:(NSData *)frameData { + if (frameData.length < 15000) { + return; + } + + const NSUInteger width = 400; + const NSUInteger height = 300; + NSMutableData *rgba = [NSMutableData dataWithLength:width * height * 4]; + const uint8_t *source = frameData.bytes; + uint8_t *pixels = rgba.mutableBytes; + + for (NSUInteger y = 0; y < height; y++) { + NSUInteger invY = height - 1 - y; + for (NSUInteger x = 0; x < width; x++) { + NSUInteger bx = x / 2; + NSUInteger by = invY / 4; + NSUInteger index = bx * 75 + by; + NSUInteger bit = 7 - ((invY % 4) * 2 + (x % 2)); + BOOL on = (source[index] & (1 << bit)) != 0; + uint8_t value = on ? 255 : 0; + NSUInteger pixelIndex = (y * width + x) * 4; + pixels[pixelIndex + 0] = value; + pixels[pixelIndex + 1] = value; + pixels[pixelIndex + 2] = value; + pixels[pixelIndex + 3] = 255; + } + } + + CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); + CGDataProviderRef provider = CGDataProviderCreateWithCFData((__bridge CFDataRef)rgba); + CGImageRef cgImage = CGImageCreate(width, + height, + 8, + 32, + width * 4, + colorSpace, + kCGImageAlphaLast | kCGBitmapByteOrderDefault, + provider, + NULL, + false, + kCGRenderingIntentDefault); + if (cgImage) { + self.streamImage = [UIImage imageWithCGImage:cgImage]; + [self.commands removeAllObjects]; + [self setNeedsDisplay]; + CGImageRelease(cgImage); + } + CGDataProviderRelease(provider); + CGColorSpaceRelease(colorSpace); +} + +- (BOOL)displayRGB565FrameBuffer:(NSData *)frameData width:(NSUInteger)width height:(NSUInteger)height { + if (width == 0 || height == 0 || width > 2048 || height > 2048 || + frameData.length != width * height * 2) { + return NO; + } + + const uint8_t *source = frameData.bytes; + NSMutableData *rgba = [NSMutableData dataWithLength:width * height * 4]; + uint8_t *pixels = rgba.mutableBytes; + for (NSUInteger i = 0; i < width * height; i++) { + // Network color frames use big-endian RGB565. + uint16_t value = ((uint16_t)source[i * 2] << 8) | source[i * 2 + 1]; + uint8_t red5 = (value >> 11) & 0x1f; + uint8_t green6 = (value >> 5) & 0x3f; + uint8_t blue5 = value & 0x1f; + pixels[i * 4 + 0] = (red5 << 3) | (red5 >> 2); + pixels[i * 4 + 1] = (green6 << 2) | (green6 >> 4); + pixels[i * 4 + 2] = (blue5 << 3) | (blue5 >> 2); + pixels[i * 4 + 3] = 255; + } + + CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); + CGDataProviderRef provider = CGDataProviderCreateWithCFData((__bridge CFDataRef)rgba); + CGImageRef cgImage = CGImageCreate(width, + height, + 8, + 32, + width * 4, + colorSpace, + kCGImageAlphaLast | kCGBitmapByteOrderDefault, + provider, + NULL, + false, + kCGRenderingIntentDefault); + if (cgImage) { + self.streamImage = [UIImage imageWithCGImage:cgImage]; + [self.commands removeAllObjects]; + [self setNeedsDisplay]; + CGImageRelease(cgImage); + } + CGDataProviderRelease(provider); + CGColorSpaceRelease(colorSpace); + return cgImage != NULL; +} + +- (NSString *)snapshotPNGBase64 { + UIGraphicsBeginImageContextWithOptions(self.bounds.size, YES, 0.0); + [self drawViewHierarchyInRect:self.bounds afterScreenUpdates:YES]; + UIImage *image = UIGraphicsGetImageFromCurrentImageContext(); + UIGraphicsEndImageContext(); + + NSData *pngData = UIImagePNGRepresentation(image); + return [pngData base64EncodedStringWithOptions:0]; +} + +- (CGPoint)pointForDeviceX:(CGFloat)x y:(CGFloat)y { + CGFloat scaleX = self.bounds.size.width / 400.0; + CGFloat scaleY = self.bounds.size.height / 300.0; + return CGPointMake(x * scaleX, y * scaleY); +} + +- (void)drawRect:(CGRect)rect { + UIColor *background = self.clearColor == 1 ? [UIColor blackColor] : [UIColor whiteColor]; + UIColor *foreground = self.clearColor == 1 ? [UIColor whiteColor] : [UIColor blackColor]; + + [background setFill]; + UIRectFill(self.bounds); + + if (self.streamImage) { + [self.streamImage drawInRect:self.bounds]; + return; + } + + CGContextRef context = UIGraphicsGetCurrentContext(); + CGContextSetStrokeColorWithColor(context, foreground.CGColor); + CGContextSetLineWidth(context, 2.0); + CGContextStrokeRect(context, CGRectInset(self.bounds, 1.0, 1.0)); + + for (NSDictionary *command in self.commands) { + NSString *type = command[@"type"]; + CGFloat x = [command[@"x"] doubleValue]; + CGFloat y = [command[@"y"] doubleValue]; + CGPoint point = [self pointForDeviceX:x y:y]; + + if ([type isEqualToString:@"text"]) { + NSInteger size = [command[@"size"] integerValue]; + UIFont *font = [UIFont boldSystemFontOfSize:size == 2 ? 26.0 : 15.0]; + NSDictionary *attrs = @{ + NSFontAttributeName: font, + NSForegroundColorAttributeName: foreground + }; + [command[@"text"] drawAtPoint:point withAttributes:attrs]; + } else if ([type isEqualToString:@"image"]) { + UIImage *image = command[@"image"]; + CGFloat maxWidth = self.bounds.size.width - point.x; + CGFloat maxHeight = self.bounds.size.height - point.y; + CGSize imageSize = image.size; + CGFloat scale = MIN(maxWidth / MAX(imageSize.width, 1.0), + maxHeight / MAX(imageSize.height, 1.0)); + scale = MIN(scale, 1.0); + CGRect imageRect = CGRectMake(point.x, + point.y, + imageSize.width * scale, + imageSize.height * scale); + [image drawInRect:imageRect]; + } + } +} + +@end diff --git a/iphone_app/IMCMCPServer.h b/iphone_app/IMCMCPServer.h new file mode 100644 index 0000000..4be4157 --- /dev/null +++ b/iphone_app/IMCMCPServer.h @@ -0,0 +1,8 @@ +#import + +@class IMCViewController; + +@interface IMCMCPServer : NSObject +- (instancetype)initWithViewController:(IMCViewController *)viewController port:(NSInteger)port; +- (void)start; +@end diff --git a/iphone_app/IMCMCPServer.m b/iphone_app/IMCMCPServer.m new file mode 100644 index 0000000..d9425dc --- /dev/null +++ b/iphone_app/IMCMCPServer.m @@ -0,0 +1,163 @@ +#import "IMCMCPServer.h" +#import "IMCViewController.h" +#import +#import +#import +#import + +@interface IMCMCPServer () +@property (nonatomic, weak) IMCViewController *viewController; +@property (nonatomic, assign) NSInteger port; +@property (nonatomic, assign) BOOL running; +@end + +@implementation IMCMCPServer + +- (instancetype)initWithViewController:(IMCViewController *)viewController port:(NSInteger)port { + self = [super init]; + if (self) { + _viewController = viewController; + _port = port; + } + return self; +} + +- (void)start { + if (self.running) { + return; + } + self.running = YES; + [NSThread detachNewThreadSelector:@selector(serverLoop) toTarget:self withObject:nil]; +} + +- (void)serverLoop { + @autoreleasepool { + int serverSocket = socket(AF_INET, SOCK_STREAM, 0); + if (serverSocket < 0) { + return; + } + + int yes = 1; + setsockopt(serverSocket, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + + struct sockaddr_in address; + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_addr.s_addr = INADDR_ANY; + address.sin_port = htons((uint16_t)self.port); + + if (bind(serverSocket, (struct sockaddr *)&address, sizeof(address)) < 0) { + close(serverSocket); + return; + } + + if (listen(serverSocket, 8) < 0) { + close(serverSocket); + return; + } + + while (self.running) { + @autoreleasepool { + struct sockaddr_in clientAddress; + socklen_t clientLength = sizeof(clientAddress); + int clientSocket = accept(serverSocket, + (struct sockaddr *)&clientAddress, + &clientLength); + if (clientSocket >= 0) { + [self handleClientSocket:clientSocket]; + close(clientSocket); + } + } + } + + close(serverSocket); + } +} + +- (void)handleClientSocket:(int)clientSocket { + NSMutableData *requestData = [NSMutableData data]; + char buffer[2048]; + + while (requestData.length < 1024 * 1024) { + ssize_t count = recv(clientSocket, buffer, sizeof(buffer), 0); + if (count <= 0) { + break; + } + [requestData appendBytes:buffer length:(NSUInteger)count]; + + NSString *partial = [[NSString alloc] initWithData:requestData encoding:NSUTF8StringEncoding]; + NSRange separator = [partial rangeOfString:@"\r\n\r\n"]; + if (separator.location == NSNotFound) { + continue; + } + + NSInteger contentLength = [self contentLengthFromHeader:partial]; + NSUInteger bodyStart = separator.location + separator.length; + if (requestData.length >= bodyStart + MAX(contentLength, 0)) { + break; + } + } + + NSString *request = [[NSString alloc] initWithData:requestData encoding:NSUTF8StringEncoding]; + NSDictionary *responseObject = [self responseObjectForHTTPRequest:request]; + NSData *responseJSON = [NSJSONSerialization dataWithJSONObject:responseObject options:0 error:nil]; + if (!responseJSON) { + responseJSON = [@"{}" dataUsingEncoding:NSUTF8StringEncoding]; + } + + NSString *status = responseObject[@"httpStatus"] ?: @"200 OK"; + NSMutableDictionary *bodyObject = [responseObject mutableCopy]; + [bodyObject removeObjectForKey:@"httpStatus"]; + responseJSON = [NSJSONSerialization dataWithJSONObject:bodyObject options:0 error:nil]; + + NSMutableData *responseData = [NSMutableData data]; + NSString *header = [NSString stringWithFormat: + @"HTTP/1.1 %@\r\nContent-Type: application/json\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", + status, + (unsigned long)responseJSON.length]; + [responseData appendData:[header dataUsingEncoding:NSUTF8StringEncoding]]; + [responseData appendData:responseJSON]; + send(clientSocket, responseData.bytes, responseData.length, 0); +} + +- (NSDictionary *)responseObjectForHTTPRequest:(NSString *)request { + if (![request hasPrefix:@"POST /api/mcp "]) { + return @{@"httpStatus": @"404 Not Found", + @"jsonrpc": @"2.0", + @"error": @{@"code": @-32601, @"message": @"POST /api/mcp required"}, + @"id": [NSNull null]}; + } + + NSRange separator = [request rangeOfString:@"\r\n\r\n"]; + if (separator.location == NSNotFound) { + return [self parseError:@"Missing HTTP body."]; + } + + NSString *body = [request substringFromIndex:separator.location + separator.length]; + NSData *bodyData = [body dataUsingEncoding:NSUTF8StringEncoding]; + NSError *jsonError = nil; + NSDictionary *rpcRequest = [NSJSONSerialization JSONObjectWithData:bodyData options:0 error:&jsonError]; + if (jsonError || ![rpcRequest isKindOfClass:NSDictionary.class]) { + return [self parseError:@"Invalid JSON-RPC object."]; + } + + return [self.viewController handleRPCRequest:rpcRequest]; +} + +- (NSDictionary *)parseError:(NSString *)message { + return @{@"jsonrpc": @"2.0", + @"error": @{@"code": @-32700, @"message": message ?: @"Parse error"}, + @"id": [NSNull null]}; +} + +- (NSInteger)contentLengthFromHeader:(NSString *)request { + NSArray *lines = [request componentsSeparatedByString:@"\r\n"]; + for (NSString *line in lines) { + if ([[line lowercaseString] hasPrefix:@"content-length:"]) { + return [[line substringFromIndex:15] integerValue]; + } + } + return 0; +} + +@end diff --git a/iphone_app/IMCVideoStreamServer.h b/iphone_app/IMCVideoStreamServer.h new file mode 100644 index 0000000..db9d9e3 --- /dev/null +++ b/iphone_app/IMCVideoStreamServer.h @@ -0,0 +1,11 @@ +#import + +@class IMCCanvasView; + +@interface IMCVideoStreamServer : NSObject +- (instancetype)initWithCanvasView:(IMCCanvasView *)canvasView; +- (void)start; +- (NSDictionary *)streamStats; +- (void)resetStreamStats; +- (void)setDebugEnabled:(BOOL)enabled; +@end diff --git a/iphone_app/IMCVideoStreamServer.m b/iphone_app/IMCVideoStreamServer.m new file mode 100644 index 0000000..f73afc9 --- /dev/null +++ b/iphone_app/IMCVideoStreamServer.m @@ -0,0 +1,447 @@ +#import "IMCVideoStreamServer.h" +#import "IMCCanvasView.h" +#import +#import +#import +#import +#import + +static const NSInteger IMCFrameSize = 15000; +static const NSInteger IMCUDPChunkSize = 1000; +static const NSInteger IMCUDPChunkCount = 15; +static const NSInteger IMCColorStreamPort = 8083; +static const NSUInteger IMCColorHeaderSize = 16; + +@interface IMCVideoStreamServer () +@property (nonatomic, weak) IMCCanvasView *canvasView; +@property (nonatomic, assign) BOOL running; +@property (nonatomic, assign) BOOL debugEnabled; +@property (nonatomic, strong) NSDate *startedAt; +@property (nonatomic, strong) NSDate *lastFrameAt; +@property (nonatomic, copy) NSString *lastProtocol; +@property (nonatomic, assign) uint64_t tcpConnections; +@property (nonatomic, assign) uint64_t tcpFrames; +@property (nonatomic, assign) uint64_t tcpBytes; +@property (nonatomic, assign) uint64_t udpFrames; +@property (nonatomic, assign) uint64_t udpPackets; +@property (nonatomic, assign) uint64_t udpBytes; +@property (nonatomic, assign) uint64_t invalidUDPPackets; +@property (nonatomic, assign) uint64_t discoveryRequests; +@property (nonatomic, assign) uint64_t colorConnections; +@property (nonatomic, assign) uint64_t colorFrames; +@property (nonatomic, assign) uint64_t colorBytes; +@property (nonatomic, assign) NSUInteger lastColorWidth; +@property (nonatomic, assign) NSUInteger lastColorHeight; +@end + +@implementation IMCVideoStreamServer + +- (instancetype)initWithCanvasView:(IMCCanvasView *)canvasView { + self = [super init]; + if (self) { + _canvasView = canvasView; + _startedAt = [NSDate date]; + _lastProtocol = @"none"; + } + return self; +} + +- (void)start { + if (self.running) { + return; + } + self.running = YES; + [NSThread detachNewThreadSelector:@selector(tcpLoop) toTarget:self withObject:nil]; + [NSThread detachNewThreadSelector:@selector(udpLoop) toTarget:self withObject:nil]; + [NSThread detachNewThreadSelector:@selector(discoveryLoop) toTarget:self withObject:nil]; + [NSThread detachNewThreadSelector:@selector(colorTCPLoop) toTarget:self withObject:nil]; +} + +- (BOOL)readExactly:(NSUInteger)length fromSocket:(int)socketFD intoData:(NSMutableData *)data { + [data setLength:length]; + NSUInteger received = 0; + while (self.running && received < length) { + ssize_t count = recv(socketFD, + (uint8_t *)data.mutableBytes + received, + length - received, + 0); + if (count <= 0) { + return NO; + } + received += (NSUInteger)count; + } + return received == length; +} + +- (void)colorTCPLoop { + @autoreleasepool { + int serverSocket = socket(AF_INET, SOCK_STREAM, 0); + if (serverSocket < 0) { + return; + } + + int yes = 1; + setsockopt(serverSocket, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + struct sockaddr_in address; + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_addr.s_addr = INADDR_ANY; + address.sin_port = htons((uint16_t)IMCColorStreamPort); + if (bind(serverSocket, (struct sockaddr *)&address, sizeof(address)) < 0 || + listen(serverSocket, 1) < 0) { + close(serverSocket); + return; + } + + while (self.running) { + @autoreleasepool { + int clientSocket = accept(serverSocket, NULL, NULL); + if (clientSocket < 0) { + continue; + } + @synchronized (self) { + self.colorConnections += 1; + } + + NSMutableData *header = [NSMutableData data]; + while ([self readExactly:IMCColorHeaderSize + fromSocket:clientSocket + intoData:header]) { + const uint8_t *bytes = header.bytes; + if (memcmp(bytes, "IMCR", 4) != 0 || bytes[4] != 1 || bytes[5] != 1) { + break; + } + NSUInteger width = ((NSUInteger)bytes[6] << 8) | bytes[7]; + NSUInteger height = ((NSUInteger)bytes[8] << 8) | bytes[9]; + uint32_t payloadLength = ((uint32_t)bytes[10] << 24) | + ((uint32_t)bytes[11] << 16) | + ((uint32_t)bytes[12] << 8) | + bytes[13]; + if (width == 0 || height == 0 || width > 2048 || height > 2048 || + payloadLength != width * height * 2 || payloadLength > 8 * 1024 * 1024) { + break; + } + + NSMutableData *payload = [NSMutableData data]; + if (![self readExactly:payloadLength + fromSocket:clientSocket + intoData:payload]) { + break; + } + NSData *snapshot = [payload copy]; + @synchronized (self) { + self.colorFrames += 1; + self.colorBytes += payloadLength; + self.lastColorWidth = width; + self.lastColorHeight = height; + self.lastProtocol = @"color_tcp"; + self.lastFrameAt = [NSDate date]; + } + dispatch_async(dispatch_get_main_queue(), ^{ + [self.canvasView displayRGB565FrameBuffer:snapshot width:width height:height]; + }); + } + close(clientSocket); + } + } + close(serverSocket); + } +} + +- (void)tcpLoop { + @autoreleasepool { + int serverSocket = socket(AF_INET, SOCK_STREAM, 0); + if (serverSocket < 0) { + return; + } + + int yes = 1; + setsockopt(serverSocket, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + + struct sockaddr_in address; + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_addr.s_addr = INADDR_ANY; + address.sin_port = htons(8081); + + if (bind(serverSocket, (struct sockaddr *)&address, sizeof(address)) < 0) { + close(serverSocket); + return; + } + + if (listen(serverSocket, 1) < 0) { + close(serverSocket); + return; + } + + while (self.running) { + @autoreleasepool { + int clientSocket = accept(serverSocket, NULL, NULL); + if (clientSocket < 0) { + continue; + } + [self noteTCPConnection]; + + NSMutableData *frame = [NSMutableData dataWithLength:IMCFrameSize]; + NSUInteger received = 0; + while (self.running) { + ssize_t count = recv(clientSocket, + (uint8_t *)frame.mutableBytes + received, + IMCFrameSize - received, + 0); + if (count <= 0) { + break; + } + + received += (NSUInteger)count; + [self noteTCPBytes:(NSUInteger)count]; + if (received == IMCFrameSize) { + [self displayFrame:frame protocol:@"tcp"]; + received = 0; + } + } + + close(clientSocket); + } + } + + close(serverSocket); + } +} + +- (void)udpLoop { + @autoreleasepool { + int udpSocket = socket(AF_INET, SOCK_DGRAM, 0); + if (udpSocket < 0) { + return; + } + + int yes = 1; + setsockopt(udpSocket, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + + struct sockaddr_in address; + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_addr.s_addr = INADDR_ANY; + address.sin_port = htons(8082); + + if (bind(udpSocket, (struct sockaddr *)&address, sizeof(address)) < 0) { + close(udpSocket); + return; + } + + NSMutableData *frame = [NSMutableData dataWithLength:IMCFrameSize]; + uint8_t currentFrameID = 0; + uint16_t chunksReceived = 0; + BOOL hasFrame = NO; + uint8_t packet[1002]; + + while (self.running) { + ssize_t count = recvfrom(udpSocket, packet, sizeof(packet), 0, NULL, NULL); + if (count < 2) { + [self noteInvalidUDPPacket]; + continue; + } + + uint8_t frameID = packet[0]; + uint8_t chunkIndex = packet[1]; + if (chunkIndex >= IMCUDPChunkCount || count < 2 + IMCUDPChunkSize) { + [self noteInvalidUDPPacket]; + continue; + } + [self noteUDPPacketBytes:(NSUInteger)count]; + + if (!hasFrame || frameID != currentFrameID) { + currentFrameID = frameID; + chunksReceived = 0; + hasFrame = YES; + } + + memcpy((uint8_t *)frame.mutableBytes + chunkIndex * IMCUDPChunkSize, + packet + 2, + IMCUDPChunkSize); + chunksReceived |= (uint16_t)(1 << chunkIndex); + + if (chunksReceived == 0x7fff) { + [self displayFrame:frame protocol:@"udp"]; + chunksReceived = 0; + } + } + + close(udpSocket); + } +} + +- (void)discoveryLoop { + @autoreleasepool { + int udpSocket = socket(AF_INET, SOCK_DGRAM, 0); + if (udpSocket < 0) { + return; + } + + int yes = 1; + setsockopt(udpSocket, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + setsockopt(udpSocket, SOL_SOCKET, SO_BROADCAST, &yes, sizeof(yes)); + + struct sockaddr_in address; + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_addr.s_addr = INADDR_ANY; + address.sin_port = htons(5000); + + if (bind(udpSocket, (struct sockaddr *)&address, sizeof(address)) < 0) { + close(udpSocket); + return; + } + + char buffer[64]; + while (self.running) { + struct sockaddr_in clientAddress; + socklen_t clientLength = sizeof(clientAddress); + ssize_t count = recvfrom(udpSocket, + buffer, + sizeof(buffer), + 0, + (struct sockaddr *)&clientAddress, + &clientLength); + if (count == 15 && memcmp(buffer, "DISCOVER_SCREEN", 15) == 0) { + [self noteDiscoveryRequest]; + const char *reply = "SCREEN_IP_8080"; + sendto(udpSocket, + reply, + strlen(reply), + 0, + (struct sockaddr *)&clientAddress, + clientLength); + } + } + + close(udpSocket); + } +} + +- (void)displayFrame:(NSData *)frame protocol:(NSString *)protocol { + [self noteFrameForProtocol:protocol]; + NSData *snapshot = [frame copy]; + dispatch_async(dispatch_get_main_queue(), ^{ + [self.canvasView displayMonochromeFrameBuffer:snapshot]; + }); +} + +- (NSDictionary *)streamStats { + @synchronized (self) { + NSTimeInterval uptime = [[NSDate date] timeIntervalSinceDate:self.startedAt ?: [NSDate date]]; + NSTimeInterval secondsSinceLastFrame = self.lastFrameAt ? [[NSDate date] timeIntervalSinceDate:self.lastFrameAt] : -1.0; + return @{ + @"debug_enabled": @(self.debugEnabled), + @"running": @(self.running), + @"tcp_port": @8081, + @"udp_port": @8082, + @"color_tcp_port": @(IMCColorStreamPort), + @"discovery_port": @5000, + @"frame_size_bytes": @(IMCFrameSize), + @"udp_chunk_size_bytes": @(IMCUDPChunkSize), + @"udp_chunks_per_frame": @(IMCUDPChunkCount), + @"uptime_sec": @((NSInteger)uptime), + @"tcp_connections": @(self.tcpConnections), + @"tcp_frames": @(self.tcpFrames), + @"tcp_bytes": @(self.tcpBytes), + @"udp_frames": @(self.udpFrames), + @"udp_packets": @(self.udpPackets), + @"udp_bytes": @(self.udpBytes), + @"invalid_udp_packets": @(self.invalidUDPPackets), + @"discovery_requests": @(self.discoveryRequests), + @"color_connections": @(self.colorConnections), + @"color_frames": @(self.colorFrames), + @"color_bytes": @(self.colorBytes), + @"last_color_width": @(self.lastColorWidth), + @"last_color_height": @(self.lastColorHeight), + @"last_protocol": self.lastProtocol ?: @"none", + @"seconds_since_last_frame": @(secondsSinceLastFrame) + }; + } +} + +- (void)resetStreamStats { + @synchronized (self) { + self.startedAt = [NSDate date]; + self.lastFrameAt = nil; + self.lastProtocol = @"none"; + self.tcpConnections = 0; + self.tcpFrames = 0; + self.tcpBytes = 0; + self.udpFrames = 0; + self.udpPackets = 0; + self.udpBytes = 0; + self.invalidUDPPackets = 0; + self.discoveryRequests = 0; + self.colorConnections = 0; + self.colorFrames = 0; + self.colorBytes = 0; + self.lastColorWidth = 0; + self.lastColorHeight = 0; + } +} + +- (void)setDebugEnabled:(BOOL)enabled { + @synchronized (self) { + _debugEnabled = enabled; + } +} + +- (void)noteTCPConnection { + @synchronized (self) { + self.tcpConnections += 1; + if (self.debugEnabled) { + NSLog(@"IPhoneMCP stream: TCP client connected"); + } + } +} + +- (void)noteTCPBytes:(NSUInteger)count { + @synchronized (self) { + self.tcpBytes += count; + } +} + +- (void)noteUDPPacketBytes:(NSUInteger)count { + @synchronized (self) { + self.udpPackets += 1; + self.udpBytes += count; + } +} + +- (void)noteInvalidUDPPacket { + @synchronized (self) { + self.invalidUDPPackets += 1; + if (self.debugEnabled) { + NSLog(@"IPhoneMCP stream: invalid UDP packet"); + } + } +} + +- (void)noteDiscoveryRequest { + @synchronized (self) { + self.discoveryRequests += 1; + if (self.debugEnabled) { + NSLog(@"IPhoneMCP stream: discovery request"); + } + } +} + +- (void)noteFrameForProtocol:(NSString *)protocol { + @synchronized (self) { + if ([protocol isEqualToString:@"tcp"]) { + self.tcpFrames += 1; + } else if ([protocol isEqualToString:@"udp"]) { + self.udpFrames += 1; + } + self.lastProtocol = protocol ?: @"none"; + self.lastFrameAt = [NSDate date]; + if (self.debugEnabled) { + NSLog(@"IPhoneMCP stream: %@ frame displayed", self.lastProtocol); + } + } +} + +@end diff --git a/iphone_app/IMCViewController.h b/iphone_app/IMCViewController.h new file mode 100644 index 0000000..2a0f95c --- /dev/null +++ b/iphone_app/IMCViewController.h @@ -0,0 +1,5 @@ +#import + +@interface IMCViewController : UIViewController +- (NSDictionary *)handleRPCRequest:(NSDictionary *)request; +@end diff --git a/iphone_app/IMCViewController.m b/iphone_app/IMCViewController.m new file mode 100644 index 0000000..03e435e --- /dev/null +++ b/iphone_app/IMCViewController.m @@ -0,0 +1,846 @@ +#import "IMCViewController.h" +#import "IMCCanvasView.h" +#import "IMCMCPServer.h" +#import "IMCVideoStreamServer.h" +#import +#import + +@interface IMCPhotoCaptureDelegate : NSObject +@property (nonatomic, copy) void (^completion)(NSData *jpegData, NSError *error); +@end + +@implementation IMCPhotoCaptureDelegate + +- (void)captureOutput:(AVCapturePhotoOutput *)output +didFinishProcessingPhoto:(AVCapturePhoto *)photo + error:(NSError *)error { + NSData *data = error ? nil : [photo fileDataRepresentation]; + if (self.completion) { + self.completion(data, error); + } +} + +@end + +@interface IMCViewController () +@property (nonatomic, strong) IMCCanvasView *canvasView; +@property (nonatomic, strong) IMCMCPServer *server; +@property (nonatomic, strong) IMCVideoStreamServer *videoStreamServer; +@property (nonatomic, strong) AVAudioPlayer *audioPlayer; +@property (nonatomic, strong) AVAudioRecorder *audioRecorder; +@end + +@implementation IMCViewController + +- (void)viewDidLoad { + [super viewDidLoad]; + + self.view.backgroundColor = [UIColor colorWithRed:0.03 green:0.06 blue:0.12 alpha:1.0]; + + self.canvasView = [[IMCCanvasView alloc] initWithFrame:CGRectZero]; + self.canvasView.translatesAutoresizingMaskIntoConstraints = NO; + [self.view addSubview:self.canvasView]; + + [NSLayoutConstraint activateConstraints:@[ + [self.canvasView.topAnchor constraintEqualToAnchor:self.view.topAnchor], + [self.canvasView.bottomAnchor constraintEqualToAnchor:self.view.bottomAnchor], + [self.canvasView.leadingAnchor constraintEqualToAnchor:self.view.leadingAnchor], + [self.canvasView.trailingAnchor constraintEqualToAnchor:self.view.trailingAnchor] + ]]; + + [self.canvasView clearWithColor:0]; + [self.canvasView drawText:@"iPhone MCP landscape server ready" x:18 y:18 size:2]; + [self.canvasView drawText:@"POST /api/mcp on port 8080" x:18 y:58 size:1]; + + self.server = [[IMCMCPServer alloc] initWithViewController:self port:8080]; + [self.server start]; + + self.videoStreamServer = [[IMCVideoStreamServer alloc] initWithCanvasView:self.canvasView]; + [self.videoStreamServer start]; +} + +- (NSDictionary *)handleRPCRequest:(NSDictionary *)request { + __block NSDictionary *response = nil; + dispatch_sync(dispatch_get_main_queue(), ^{ + response = [self handleRPCRequestOnMainThread:request]; + }); + return response; +} + +- (NSDictionary *)handleRPCRequestOnMainThread:(NSDictionary *)request { + id rpcID = request[@"id"] ?: [NSNull null]; + NSString *method = request[@"method"]; + + if ([method isEqualToString:@"initialize"]) { + return [self result:@{ + @"protocolVersion": @"2024-11-05", + @"capabilities": @{@"tools": @{}}, + @"serverInfo": @{@"name": @"iphone6-mcp", @"version": @"0.0.4"} + } rpcID:rpcID]; + } + + if ([method isEqualToString:@"notifications/initialized"]) { + return [self result:@{} rpcID:rpcID]; + } + + if ([method isEqualToString:@"tools/list"]) { + return [self result:@{@"tools": [self tools]} rpcID:rpcID]; + } + + if ([method isEqualToString:@"tools/call"]) { + NSDictionary *params = request[@"params"] ?: @{}; + NSString *name = params[@"name"]; + NSDictionary *arguments = params[@"arguments"] ?: @{}; + return [self callTool:name arguments:arguments rpcID:rpcID]; + } + + return [self errorWithCode:-32601 message:[NSString stringWithFormat:@"Method %@ not found", method] rpcID:rpcID]; +} + +- (NSDictionary *)callTool:(NSString *)name arguments:(NSDictionary *)arguments rpcID:(id)rpcID { + @try { + if ([name isEqualToString:@"clear_screen"]) { + NSInteger color = [arguments[@"color"] integerValue]; + [self.canvasView clearWithColor:color]; + return [self textResult:@"Screen cleared." rpcID:rpcID]; + } + + if ([name isEqualToString:@"draw_text"]) { + NSString *text = [NSString stringWithFormat:@"%@", arguments[@"text"] ?: @""]; + CGFloat x = [arguments[@"x"] doubleValue]; + CGFloat y = [arguments[@"y"] doubleValue]; + NSInteger size = arguments[@"size"] ? [arguments[@"size"] integerValue] : 1; + [self.canvasView drawText:text x:x y:y size:size]; + return [self textResult:[NSString stringWithFormat:@"Drew text '%@' at (%.0f, %.0f).", text, x, y] rpcID:rpcID]; + } + + if ([name isEqualToString:@"draw_image"]) { + NSString *base64 = [NSString stringWithFormat:@"%@", arguments[@"image_base64"] ?: @""]; + NSRange marker = [base64 rangeOfString:@";base64,"]; + if (marker.location != NSNotFound) { + base64 = [base64 substringFromIndex:marker.location + marker.length]; + } + NSData *data = [[NSData alloc] initWithBase64EncodedString:base64 options:NSDataBase64DecodingIgnoreUnknownCharacters]; + if (!data) { + return [self errorWithCode:-32602 message:@"Invalid image_base64." rpcID:rpcID]; + } + [self.canvasView drawImageData:data + x:[arguments[@"x"] doubleValue] + y:[arguments[@"y"] doubleValue]]; + return [self textResult:@"Image displayed." rpcID:rpcID]; + } + + if ([name isEqualToString:@"get_screenshot"]) { + NSString *png = [self.canvasView snapshotPNGBase64]; + return [self result:@{@"content": @[@{@"type": @"image", @"data": png, @"mimeType": @"image/png"}]} rpcID:rpcID]; + } + + if ([name isEqualToString:@"get_battery"]) { + UIDevice *device = [UIDevice currentDevice]; + device.batteryMonitoringEnabled = YES; + NSDictionary *battery = @{ + @"level_pct": @(MAX(device.batteryLevel, 0.0) * 100.0), + @"state": [self batteryStateName:device.batteryState] + }; + return [self textResult:[self jsonString:battery] rpcID:rpcID]; + } + + if ([name isEqualToString:@"get_sensors"] || [name isEqualToString:@"scan_ble"] || [name isEqualToString:@"set_led"]) { + return [self textResult:[NSString stringWithFormat:@"%@ is not implemented in this first iPhone build.", name] rpcID:rpcID]; + } + + if ([name isEqualToString:@"play_tone"]) { + NSInteger frequency = arguments[@"frequency"] ? [arguments[@"frequency"] integerValue] : 440; + NSInteger duration = arguments[@"duration_ms"] ? [arguments[@"duration_ms"] integerValue] : 1000; + NSInteger volume = arguments[@"volume"] ? [arguments[@"volume"] integerValue] : 50; + + frequency = MAX(50, MIN(10000, frequency)); + duration = MAX(50, MIN(5000, duration)); + volume = MAX(0, MIN(100, volume)); + + NSData *toneData = [self wavDataForToneFrequency:frequency + durationMS:duration + volume:volume]; + NSDictionary *error = [self playAudioData:toneData volume:volume]; + if (error) { + return [self errorWithCode:[error[@"code"] integerValue] + message:error[@"message"] + rpcID:rpcID]; + } + return [self textResult:[NSString stringWithFormat:@"Played tone of %ldHz for %ldms at volume %ld.", + (long)frequency, + (long)duration, + (long)volume] + rpcID:rpcID]; + } + + if ([name isEqualToString:@"play_audio_base64"]) { + NSString *base64 = [self base64PayloadFromString:[NSString stringWithFormat:@"%@", arguments[@"wav_base64"] ?: @""]]; + NSData *audioData = [[NSData alloc] initWithBase64EncodedString:base64 + options:NSDataBase64DecodingIgnoreUnknownCharacters]; + if (!audioData) { + return [self errorWithCode:-32602 message:@"Invalid wav_base64." rpcID:rpcID]; + } + + NSInteger volumeArg = arguments[@"volume"] ? [arguments[@"volume"] integerValue] : 50; + volumeArg = MAX(0, MIN(100, volumeArg)); + NSDictionary *error = [self playAudioData:audioData volume:volumeArg]; + if (error) { + return [self errorWithCode:[error[@"code"] integerValue] + message:error[@"message"] + rpcID:rpcID]; + } + return [self textResult:[NSString stringWithFormat:@"Playing base64 WAV audio (%lu bytes) at volume %ld.", + (unsigned long)audioData.length, + (long)volumeArg] + rpcID:rpcID]; + } + + if ([name isEqualToString:@"play_audio"]) { + NSString *filename = [NSString stringWithFormat:@"%@", arguments[@"filename"] ?: @""]; + NSURL *url = [self safeFileURLForPath:filename]; + if (!url) { + return [self errorWithCode:-32602 + message:@"filename must be relative and cannot contain '..'." + rpcID:rpcID]; + } + + NSData *audioData = [NSData dataWithContentsOfURL:url]; + if (!audioData) { + return [self errorWithCode:-32000 + message:[NSString stringWithFormat:@"Could not read audio file '%@' from Documents/MCPFiles.", filename] + rpcID:rpcID]; + } + + NSInteger volumeArg = arguments[@"volume"] ? [arguments[@"volume"] integerValue] : 50; + volumeArg = MAX(0, MIN(100, volumeArg)); + NSDictionary *error = [self playAudioData:audioData volume:volumeArg]; + if (error) { + return [self errorWithCode:[error[@"code"] integerValue] + message:error[@"message"] + rpcID:rpcID]; + } + return [self textResult:[NSString stringWithFormat:@"Playing WAV file '%@' (%lu bytes) at volume %ld.", + filename, + (unsigned long)audioData.length, + (long)volumeArg] + rpcID:rpcID]; + } + + if ([name isEqualToString:@"hermes_voice_turn"]) { + NSDictionary *voiceResult = [self performHermesVoiceTurn:arguments]; + if (voiceResult[@"error"]) { + return [self errorWithCode:-32000 message:voiceResult[@"error"] rpcID:rpcID]; + } + return [self textResult:[self jsonString:voiceResult] rpcID:rpcID]; + } + + if ([name isEqualToString:@"record_voice"]) { + if (![self ensureMicrophonePermission]) { + return [self errorWithCode:-32000 + message:@"Microphone permission is not granted for iPhone MCP." + rpcID:rpcID]; + } + + NSInteger duration = arguments[@"duration_sec"] ? [arguments[@"duration_sec"] integerValue] : 4; + duration = MAX(1, MIN(15, duration)); + NSString *filename = [NSString stringWithFormat:@"%@", arguments[@"filename"] ?: @"recording.wav"]; + NSURL *url = [self safeFileURLForPath:filename]; + if (!url) { + return [self errorWithCode:-32602 + message:@"filename must be relative and cannot contain '..'." + rpcID:rpcID]; + } + + NSDictionary *result = [self recordVoiceToURL:url duration:duration]; + if (result[@"error"]) { + return [self errorWithCode:-32000 message:result[@"error"] rpcID:rpcID]; + } + return [self textResult:[self jsonString:result] rpcID:rpcID]; + } + + if ([name isEqualToString:@"capture_selfie"]) { + if (![self ensureCameraPermission]) { + return [self errorWithCode:-32000 + message:@"Camera permission is not granted for iPhone MCP." + rpcID:rpcID]; + } + + NSDictionary *result = [self captureSelfiePNGBase64]; + if (result[@"error"]) { + return [self errorWithCode:-32000 message:result[@"error"] rpcID:rpcID]; + } + return [self result:@{@"content": @[@{@"type": @"image", + @"data": result[@"png_base64"], + @"mimeType": @"image/png"}]} rpcID:rpcID]; + } + + if ([name isEqualToString:@"write_file"]) { + NSString *path = [NSString stringWithFormat:@"%@", arguments[@"path"] ?: @""]; + NSString *content = [NSString stringWithFormat:@"%@", arguments[@"content"] ?: @""]; + NSURL *url = [self safeFileURLForPath:path]; + if (!url) { + return [self errorWithCode:-32602 message:@"Path must be relative and cannot contain '..'." rpcID:rpcID]; + } + [[NSFileManager defaultManager] createDirectoryAtURL:[url URLByDeletingLastPathComponent] + withIntermediateDirectories:YES + attributes:nil + error:nil]; + NSError *writeError = nil; + [content writeToURL:url atomically:YES encoding:NSUTF8StringEncoding error:&writeError]; + if (writeError) { + return [self errorWithCode:-32000 message:writeError.localizedDescription rpcID:rpcID]; + } + return [self textResult:[NSString stringWithFormat:@"Wrote %lu characters to %@.", (unsigned long)content.length, path] rpcID:rpcID]; + } + + if ([name isEqualToString:@"read_file"]) { + NSString *path = [NSString stringWithFormat:@"%@", arguments[@"path"] ?: @""]; + NSURL *url = [self safeFileURLForPath:path]; + if (!url) { + return [self errorWithCode:-32602 message:@"Path must be relative and cannot contain '..'." rpcID:rpcID]; + } + NSError *readError = nil; + NSString *content = [NSString stringWithContentsOfURL:url encoding:NSUTF8StringEncoding error:&readError]; + if (readError) { + return [self errorWithCode:-32000 message:readError.localizedDescription rpcID:rpcID]; + } + return [self textResult:content rpcID:rpcID]; + } + + if ([name isEqualToString:@"download_file"]) { + NSString *urlString = [NSString stringWithFormat:@"%@", arguments[@"url"] ?: @""]; + NSString *filename = [NSString stringWithFormat:@"%@", arguments[@"filename"] ?: @""]; + BOOL useSD = arguments[@"use_sd"] ? [arguments[@"use_sd"] boolValue] : NO; + NSURL *sourceURL = [NSURL URLWithString:urlString]; + NSURL *destinationURL = [self safeFileURLForPath:filename]; + if (!sourceURL || !destinationURL) { + return [self errorWithCode:-32602 message:@"Invalid url or filename." rpcID:rpcID]; + } + NSData *data = [NSData dataWithContentsOfURL:sourceURL]; + if (!data) { + return [self errorWithCode:-32000 message:@"Download failed." rpcID:rpcID]; + } + [[NSFileManager defaultManager] createDirectoryAtURL:[destinationURL URLByDeletingLastPathComponent] + withIntermediateDirectories:YES + attributes:nil + error:nil]; + [data writeToURL:destinationURL atomically:YES]; + NSString *storageNote = useSD ? @" The iPhone build has no SD card, so use_sd was accepted and stored in app Documents instead." : @""; + return [self textResult:[NSString stringWithFormat:@"Downloaded %lu bytes to %@.%@", (unsigned long)data.length, filename, storageNote] rpcID:rpcID]; + } + + if ([name isEqualToString:@"get_video_streaming_instructions"]) { + NSString *protocol = [[NSString stringWithFormat:@"%@", arguments[@"protocol"] ?: @"both"] lowercaseString]; + return [self textResult:[self videoStreamingInstructionsForProtocol:protocol] rpcID:rpcID]; + } + + if ([name isEqualToString:@"get_stream_stats"]) { + return [self textResult:[self jsonString:[self.videoStreamServer streamStats]] rpcID:rpcID]; + } + + if ([name isEqualToString:@"reset_stream_stats"]) { + [self.videoStreamServer resetStreamStats]; + return [self textResult:[self jsonString:[self.videoStreamServer streamStats]] rpcID:rpcID]; + } + + if ([name isEqualToString:@"set_stream_debug"]) { + BOOL enabled = arguments[@"enabled"] ? [arguments[@"enabled"] boolValue] : NO; + [self.videoStreamServer setDebugEnabled:enabled]; + return [self textResult:[self jsonString:@{@"debug_enabled": @(enabled)}] rpcID:rpcID]; + } + + if ([name isEqualToString:@"execute_python"]) { + return [self textResult:@"execute_python is available on the MicroPython ESP32 target, but the native iPhone app does not execute arbitrary Python code. Use write_file/read_file/download_file for files and the dedicated drawing/audio/camera tools for app actions." rpcID:rpcID]; + } + + return [self errorWithCode:-32602 message:[NSString stringWithFormat:@"Unknown tool: %@", name] rpcID:rpcID]; + } @catch (NSException *exception) { + return [self errorWithCode:-32000 message:exception.reason ?: @"Tool failed." rpcID:rpcID]; + } +} + +- (NSArray *)tools { + return @[ + [self tool:@"clear_screen" description:@"Clear the iPhone canvas to white (0) or black (1)." schema:@{@"type": @"object", @"properties": @{@"color": @{@"type": @"integer", @"enum": @[@0, @1]}}, @"required": @[@"color"]}], + [self tool:@"draw_text" description:@"Draw text on the iPhone canvas using 400x300 logical coordinates." schema:@{@"type": @"object", @"properties": @{@"text": @{@"type": @"string"}, @"x": @{@"type": @"integer"}, @"y": @{@"type": @"integer"}, @"size": @{@"type": @"integer", @"enum": @[@1, @2]}}, @"required": @[@"text", @"x", @"y"]}], + [self tool:@"draw_image" description:@"Draw a base64 PNG/JPEG/GIF image on the iPhone canvas." schema:@{@"type": @"object", @"properties": @{@"image_base64": @{@"type": @"string"}, @"x": @{@"type": @"integer", @"default": @0}, @"y": @{@"type": @"integer", @"default": @0}}, @"required": @[@"image_base64"]}], + [self tool:@"get_screenshot" description:@"Return the current iPhone canvas as a PNG image." schema:@{@"type": @"object", @"properties": @{}}], + [self tool:@"get_battery" description:@"Read the iPhone battery percentage and charge state." schema:@{@"type": @"object", @"properties": @{}}], + [self tool:@"play_tone" description:@"Generate and play a sine-wave tone with the requested frequency, duration, and volume." schema:@{@"type": @"object", @"properties": @{@"frequency": @{@"type": @"integer", @"default": @440}, @"duration_ms": @{@"type": @"integer", @"default": @1000}, @"volume": @{@"type": @"integer", @"default": @50}}}], + [self tool:@"play_audio_base64" description:@"Decode a base64 WAV payload and play it through the iPhone speaker." schema:@{@"type": @"object", @"properties": @{@"wav_base64": @{@"type": @"string"}, @"volume": @{@"type": @"integer", @"default": @50}}, @"required": @[@"wav_base64"]}], + [self tool:@"play_audio" description:@"Play a WAV file from the app's Documents/MCPFiles directory." schema:@{@"type": @"object", @"properties": @{@"filename": @{@"type": @"string"}, @"volume": @{@"type": @"integer", @"default": @50}}, @"required": @[@"filename"]}], + [self tool:@"hermes_voice_turn" description:@"Record a voice command, send it to the Hermes voice gateway, play the returned WAV, and return transcript/response diagnostics." schema:@{@"type": @"object", @"properties": @{@"url": @{@"type": @"string", @"default": @"http://192.168.68.126:8642/api/esp32/voice"}, @"api_token": @{@"type": @"string"}, @"device_id": @{@"type": @"string", @"default": @"iphone6"}, @"duration_sec": @{@"type": @"integer", @"default": @4}, @"volume": @{@"type": @"integer", @"default": @80}, @"instructions": @{@"type": @"string"}, @"reply_mode": @{@"type": @"string", @"enum": @[@"sync", @"ack"], @"default": @"sync"}, @"screen_url": @{@"type": @"string"}}}], + [self tool:@"record_voice" description:@"Record microphone audio to a WAV file inside Documents/MCPFiles." schema:@{@"type": @"object", @"properties": @{@"duration_sec": @{@"type": @"integer", @"default": @4}, @"filename": @{@"type": @"string", @"default": @"recording.wav"}}}], + [self tool:@"capture_selfie" description:@"Capture a still image from the front camera and return it as PNG image content." schema:@{@"type": @"object", @"properties": @{}}], + [self tool:@"write_file" description:@"Write a UTF-8 file inside the app's Documents/MCPFiles directory." schema:@{@"type": @"object", @"properties": @{@"path": @{@"type": @"string"}, @"content": @{@"type": @"string"}}, @"required": @[@"path", @"content"]}], + [self tool:@"read_file" description:@"Read a UTF-8 file from the app's Documents/MCPFiles directory." schema:@{@"type": @"object", @"properties": @{@"path": @{@"type": @"string"}}, @"required": @[@"path"]}], + [self tool:@"download_file" description:@"Download a URL into the app's Documents/MCPFiles directory. Accepts ESP32-compatible use_sd, but stores locally on iPhone." schema:@{@"type": @"object", @"properties": @{@"url": @{@"type": @"string"}, @"filename": @{@"type": @"string"}, @"use_sd": @{@"type": @"boolean", @"default": @NO}}, @"required": @[@"url", @"filename"]}], + [self tool:@"get_video_streaming_instructions" description:@"Get details for monochrome TCP/UDP streaming and RGB565 color TCP streaming." schema:@{@"type": @"object", @"properties": @{@"protocol": @{@"type": @"string", @"enum": @[@"tcp", @"udp", @"color", @"both", @"all"], @"default": @"all"}}}], + [self tool:@"get_stream_stats" description:@"Return TCP/UDP streaming counters, ports, uptime, last protocol, and debug state." schema:@{@"type": @"object", @"properties": @{}}], + [self tool:@"reset_stream_stats" description:@"Reset TCP/UDP streaming counters and last-frame state." schema:@{@"type": @"object", @"properties": @{}}], + [self tool:@"set_stream_debug" description:@"Enable or disable stream debug logging on the iPhone." schema:@{@"type": @"object", @"properties": @{@"enabled": @{@"type": @"boolean", @"default": @NO}}}], + [self tool:@"execute_python" description:@"ESP32 compatibility stub; arbitrary Python execution is intentionally not implemented in the native iPhone app." schema:@{@"type": @"object", @"properties": @{@"code": @{@"type": @"string"}}, @"required": @[@"code"]}], + [self tool:@"set_led" description:@"ESP32 compatibility stub; the iPhone 6 has no NeoPixel LED." schema:@{@"type": @"object", @"properties": @{}}], + [self tool:@"get_sensors" description:@"ESP32 compatibility stub; SHTC3 temperature/humidity is not present." schema:@{@"type": @"object", @"properties": @{}}], + [self tool:@"scan_ble" description:@"ESP32 compatibility stub; BLE scanning is not implemented in this first build." schema:@{@"type": @"object", @"properties": @{}}] + ]; +} + +- (NSDictionary *)tool:(NSString *)name description:(NSString *)description schema:(NSDictionary *)schema { + return @{@"name": name, @"description": description, @"inputSchema": schema}; +} + +- (NSDictionary *)textResult:(NSString *)text rpcID:(id)rpcID { + return [self result:@{@"content": @[@{@"type": @"text", @"text": text ?: @""}]} rpcID:rpcID]; +} + +- (NSDictionary *)result:(NSDictionary *)result rpcID:(id)rpcID { + return @{@"jsonrpc": @"2.0", @"result": result ?: @{}, @"id": rpcID ?: [NSNull null]}; +} + +- (NSDictionary *)errorWithCode:(NSInteger)code message:(NSString *)message rpcID:(id)rpcID { + return @{@"jsonrpc": @"2.0", + @"error": @{@"code": @(code), @"message": message ?: @"Error"}, + @"id": rpcID ?: [NSNull null]}; +} + +- (NSString *)jsonString:(id)object { + NSData *data = [NSJSONSerialization dataWithJSONObject:object options:0 error:nil]; + return [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] ?: @"{}"; +} + +- (NSString *)base64PayloadFromString:(NSString *)string { + NSRange marker = [string rangeOfString:@";base64,"]; + if (marker.location != NSNotFound) { + return [string substringFromIndex:marker.location + marker.length]; + } + return string ?: @""; +} + +- (NSDictionary *)playAudioData:(NSData *)audioData volume:(NSInteger)volume { + NSError *sessionError = nil; + [[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayback error:&sessionError]; + [[AVAudioSession sharedInstance] setActive:YES error:&sessionError]; + + NSError *playerError = nil; + self.audioPlayer = [[AVAudioPlayer alloc] initWithData:audioData error:&playerError]; + if (!self.audioPlayer || playerError) { + return @{@"code": @-32000, + @"message": playerError.localizedDescription ?: @"Failed to create audio player."}; + } + + self.audioPlayer.volume = MAX(0.0, MIN(1.0, volume / 100.0)); + [self.audioPlayer prepareToPlay]; + [self.audioPlayer play]; + return nil; +} + +- (NSString *)headerValue:(NSString *)name fromResponse:(NSHTTPURLResponse *)response { + for (id key in response.allHeaderFields) { + if ([[NSString stringWithFormat:@"%@", key] caseInsensitiveCompare:name] == NSOrderedSame) { + return [NSString stringWithFormat:@"%@", response.allHeaderFields[key]]; + } + } + return @""; +} + +- (NSDictionary *)performHermesVoiceTurn:(NSDictionary *)arguments { + if (![self ensureMicrophonePermission]) { + return @{@"error": @"Microphone permission is not granted for iPhone MCP."}; + } + + NSInteger duration = arguments[@"duration_sec"] ? [arguments[@"duration_sec"] integerValue] : 4; + duration = MAX(1, MIN(20, duration)); + NSInteger volume = arguments[@"volume"] ? [arguments[@"volume"] integerValue] : 80; + volume = MAX(0, MIN(100, volume)); + NSString *urlString = [NSString stringWithFormat:@"%@", arguments[@"url"] ?: @"http://192.168.68.126:8642/api/esp32/voice"]; + NSURL *gatewayURL = [NSURL URLWithString:urlString]; + if (!gatewayURL || ![@[@"http", @"https"] containsObject:gatewayURL.scheme.lowercaseString]) { + return @{@"error": @"Hermes url must be an http or https URL."}; + } + + NSURL *recordingURL = [self safeFileURLForPath:@"hermes/request.wav"]; + NSDictionary *recording = [self recordVoiceToURL:recordingURL duration:duration]; + if (recording[@"error"]) { + return recording; + } + NSData *wavData = [NSData dataWithContentsOfURL:recordingURL]; + if (!wavData.length) { + return @{@"error": @"Hermes recording produced no audio data."}; + } + + NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:gatewayURL + cachePolicy:NSURLRequestReloadIgnoringLocalCacheData + timeoutInterval:90.0]; + request.HTTPMethod = @"POST"; + request.HTTPBody = wavData; + [request setValue:@"audio/wav" forHTTPHeaderField:@"Content-Type"]; + [request setValue:[NSString stringWithFormat:@"%@", arguments[@"device_id"] ?: @"iphone6"] + forHTTPHeaderField:@"X-Device-ID"]; + [request setValue:[NSString stringWithFormat:@"%@", arguments[@"reply_mode"] ?: @"sync"] + forHTTPHeaderField:@"X-Hermes-Reply-Mode"]; + + NSString *instructions = [NSString stringWithFormat:@"%@", arguments[@"instructions"] ?: @""]; + if (instructions.length) { + [request setValue:instructions forHTTPHeaderField:@"X-Hermes-Instructions"]; + } + NSString *screenURL = [NSString stringWithFormat:@"%@", arguments[@"screen_url"] ?: @""]; + if (screenURL.length) { + [request setValue:screenURL forHTTPHeaderField:@"X-Hermes-Screen-Url"]; + } + NSString *token = [NSString stringWithFormat:@"%@", arguments[@"api_token"] ?: @""]; + if (token.length) { + if (![token.lowercaseString hasPrefix:@"bearer "]) { + token = [@"Bearer " stringByAppendingString:token]; + } + [request setValue:token forHTTPHeaderField:@"Authorization"]; + } + + NSURLResponse *rawResponse = nil; + NSError *requestError = nil; + NSData *responseData = [NSURLConnection sendSynchronousRequest:request + returningResponse:&rawResponse + error:&requestError]; + if (requestError || !responseData) { + if ([requestError.domain isEqualToString:NSURLErrorDomain] && + (requestError.code == NSURLErrorUserCancelledAuthentication || + requestError.code == NSURLErrorUserAuthenticationRequired)) { + return @{@"error": @"Hermes authorization failed. Provide a valid api_token to hermes_voice_turn."}; + } + return @{@"error": requestError.localizedDescription ?: @"Hermes request failed."}; + } + NSHTTPURLResponse *response = (NSHTTPURLResponse *)rawResponse; + if (response.statusCode < 200 || response.statusCode >= 300) { + NSString *body = [[NSString alloc] initWithData:responseData encoding:NSUTF8StringEncoding] ?: @""; + return @{@"error": [NSString stringWithFormat:@"Hermes returned HTTP %ld: %@", + (long)response.statusCode, + [body substringToIndex:MIN(body.length, 500)]]}; + } + + NSDictionary *playError = [self playAudioData:responseData volume:volume]; + if (playError) { + return @{@"error": playError[@"message"] ?: @"Hermes audio playback failed."}; + } + NSURL *responseURL = [self safeFileURLForPath:@"hermes/response.wav"]; + [responseData writeToURL:responseURL atomically:YES]; + + return @{ + @"status": @"playing", + @"http_status": @(response.statusCode), + @"request_bytes": @(wavData.length), + @"response_bytes": @(responseData.length), + @"transcript": [self headerValue:@"X-Hermes-Transcript" fromResponse:response], + @"response_text": [self headerValue:@"X-Hermes-Text-Response" fromResponse:response], + @"response_id": [self headerValue:@"X-Hermes-Response-Id" fromResponse:response], + @"conversation": [self headerValue:@"X-Hermes-Conversation" fromResponse:response], + @"reply_mode": [NSString stringWithFormat:@"%@", arguments[@"reply_mode"] ?: @"sync"], + @"saved_response": @"hermes/response.wav" + }; +} + +- (NSDictionary *)recordVoiceToURL:(NSURL *)url duration:(NSInteger)duration { + NSError *directoryError = nil; + [[NSFileManager defaultManager] createDirectoryAtURL:[url URLByDeletingLastPathComponent] + withIntermediateDirectories:YES + attributes:nil + error:&directoryError]; + if (directoryError) { + return @{@"error": directoryError.localizedDescription}; + } + + NSError *sessionError = nil; + AVAudioSession *session = [AVAudioSession sharedInstance]; + [session setCategory:AVAudioSessionCategoryPlayAndRecord error:&sessionError]; + [session setActive:YES error:&sessionError]; + if (sessionError) { + return @{@"error": sessionError.localizedDescription}; + } + + NSDictionary *settings = @{ + AVFormatIDKey: @(kAudioFormatLinearPCM), + AVSampleRateKey: @16000, + AVNumberOfChannelsKey: @1, + AVLinearPCMBitDepthKey: @16, + AVLinearPCMIsFloatKey: @NO, + AVLinearPCMIsBigEndianKey: @NO + }; + + NSError *recordError = nil; + self.audioRecorder = [[AVAudioRecorder alloc] initWithURL:url + settings:settings + error:&recordError]; + if (!self.audioRecorder || recordError) { + return @{@"error": recordError.localizedDescription ?: @"Failed to create audio recorder."}; + } + + [self.audioRecorder prepareToRecord]; + [self.audioRecorder recordForDuration:duration]; + NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:duration + 0.25]; + while (self.audioRecorder.recording && [deadline timeIntervalSinceNow] > 0) { + [[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode + beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.05]]; + } + [self.audioRecorder stop]; + + NSDictionary *attributes = [[NSFileManager defaultManager] attributesOfItemAtPath:url.path error:nil]; + NSNumber *bytes = attributes[NSFileSize] ?: @0; + return @{@"filename": url.lastPathComponent ?: @"recording.wav", + @"bytes": bytes, + @"duration_sec": @(duration)}; +} + +- (NSDictionary *)captureSelfiePNGBase64 { + AVCaptureDeviceDiscoverySession *discovery = [AVCaptureDeviceDiscoverySession discoverySessionWithDeviceTypes:@[ + AVCaptureDeviceTypeBuiltInWideAngleCamera + ] + mediaType:AVMediaTypeVideo + position:AVCaptureDevicePositionFront]; + AVCaptureDevice *camera = discovery.devices.firstObject; + if (!camera) { + return @{@"error": @"Front camera not found."}; + } + + NSError *inputError = nil; + AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:camera error:&inputError]; + if (!input || inputError) { + return @{@"error": inputError.localizedDescription ?: @"Failed to open front camera."}; + } + + AVCaptureSession *session = [[AVCaptureSession alloc] init]; + session.sessionPreset = AVCaptureSessionPresetPhoto; + if (![session canAddInput:input]) { + return @{@"error": @"Could not add front camera input."}; + } + [session addInput:input]; + + AVCaptureStillImageOutput *output = [[AVCaptureStillImageOutput alloc] init]; + output.outputSettings = @{AVVideoCodecKey: AVVideoCodecJPEG}; + if (![session canAddOutput:output]) { + return @{@"error": @"Could not add still image output."}; + } + [session addOutput:output]; + [session startRunning]; + + AVCaptureConnection *connection = [output connectionWithMediaType:AVMediaTypeVideo]; + if (connection.isVideoOrientationSupported) { + connection.videoOrientation = AVCaptureVideoOrientationLandscapeRight; + } + + __block NSData *jpegData = nil; + __block NSError *captureError = nil; + __block BOOL finished = NO; + + [output captureStillImageAsynchronouslyFromConnection:connection + completionHandler:^(CMSampleBufferRef imageDataSampleBuffer, NSError *error) { + if (imageDataSampleBuffer) { + jpegData = [AVCaptureStillImageOutput jpegStillImageNSDataRepresentation:imageDataSampleBuffer]; + } + captureError = error; + finished = YES; + }]; + + NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:5.0]; + while (!finished && [deadline timeIntervalSinceNow] > 0) { + [[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode + beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.05]]; + } + [session stopRunning]; + + if (captureError || !jpegData) { + return @{@"error": captureError.localizedDescription ?: @"Selfie capture timed out."}; + } + + UIImage *image = [UIImage imageWithData:jpegData]; + NSData *pngData = UIImagePNGRepresentation(image); + if (!pngData) { + return @{@"error": @"Failed to convert selfie to PNG."}; + } + return @{@"png_base64": [pngData base64EncodedStringWithOptions:0]}; +} + +- (BOOL)ensureMicrophonePermission { + AVAudioSession *session = [AVAudioSession sharedInstance]; + if (session.recordPermission == AVAudioSessionRecordPermissionGranted) { + return YES; + } + if (session.recordPermission == AVAudioSessionRecordPermissionDenied) { + return NO; + } + + __block BOOL granted = NO; + __block BOOL finished = NO; + [session requestRecordPermission:^(BOOL didGrant) { + granted = didGrant; + finished = YES; + }]; + return [self waitForPermissionResult:&finished granted:&granted]; +} + +- (BOOL)ensureCameraPermission { + AVAuthorizationStatus status = [AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeVideo]; + if (status == AVAuthorizationStatusAuthorized) { + return YES; + } + if (status == AVAuthorizationStatusDenied || status == AVAuthorizationStatusRestricted) { + return NO; + } + + __block BOOL granted = NO; + __block BOOL finished = NO; + [AVCaptureDevice requestAccessForMediaType:AVMediaTypeVideo + completionHandler:^(BOOL didGrant) { + granted = didGrant; + finished = YES; + }]; + return [self waitForPermissionResult:&finished granted:&granted]; +} + +- (BOOL)waitForPermissionResult:(BOOL *)finished granted:(BOOL *)granted { + NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:30.0]; + while (!*finished && [deadline timeIntervalSinceNow] > 0) { + [[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode + beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.05]]; + } + return *finished && *granted; +} + +- (NSData *)wavDataForToneFrequency:(NSInteger)frequency durationMS:(NSInteger)durationMS volume:(NSInteger)volume { + const uint32_t sampleRate = 22050; + const uint16_t channels = 1; + const uint16_t bitsPerSample = 16; + const uint16_t blockAlign = channels * bitsPerSample / 8; + const uint32_t byteRate = sampleRate * blockAlign; + uint32_t sampleCount = (uint32_t)((sampleRate * durationMS) / 1000); + uint32_t dataSize = sampleCount * blockAlign; + uint32_t riffSize = 36 + dataSize; + + NSMutableData *data = [NSMutableData dataWithCapacity:44 + dataSize]; + [self appendASCII:"RIFF" toData:data]; + [self appendUInt32LE:riffSize toData:data]; + [self appendASCII:"WAVE" toData:data]; + [self appendASCII:"fmt " toData:data]; + [self appendUInt32LE:16 toData:data]; + [self appendUInt16LE:1 toData:data]; + [self appendUInt16LE:channels toData:data]; + [self appendUInt32LE:sampleRate toData:data]; + [self appendUInt32LE:byteRate toData:data]; + [self appendUInt16LE:blockAlign toData:data]; + [self appendUInt16LE:bitsPerSample toData:data]; + [self appendASCII:"data" toData:data]; + [self appendUInt32LE:dataSize toData:data]; + + double amplitude = 32767.0 * MAX(0.0, MIN(1.0, volume / 100.0)); + double phaseStep = 2.0 * M_PI * frequency / sampleRate; + for (uint32_t i = 0; i < sampleCount; i++) { + double fadeIn = MIN(1.0, i / (sampleRate * 0.01)); + double fadeOut = MIN(1.0, (sampleCount - i) / (sampleRate * 0.01)); + double envelope = MIN(fadeIn, fadeOut); + int16_t sample = (int16_t)(sin(i * phaseStep) * amplitude * envelope); + [self appendUInt16LE:(uint16_t)sample toData:data]; + } + + return data; +} + +- (void)appendASCII:(const char *)string toData:(NSMutableData *)data { + [data appendBytes:string length:4]; +} + +- (void)appendUInt16LE:(uint16_t)value toData:(NSMutableData *)data { + uint8_t bytes[] = {value & 0xff, (value >> 8) & 0xff}; + [data appendBytes:bytes length:sizeof(bytes)]; +} + +- (void)appendUInt32LE:(uint32_t)value toData:(NSMutableData *)data { + uint8_t bytes[] = { + value & 0xff, + (value >> 8) & 0xff, + (value >> 16) & 0xff, + (value >> 24) & 0xff + }; + [data appendBytes:bytes length:sizeof(bytes)]; +} + +- (NSString *)videoStreamingInstructionsForProtocol:(NSString *)protocol { + NSMutableArray *lines = [NSMutableArray arrayWithArray:@[ + @"### iPhone MCP Video Streaming Instructions", + @"Dimensions: 400x300, 1-bit monochrome, using the same 15,000-byte RLCD frame layout as the ESP32 server.", + @"The iPhone stretches each received frame over the full landscape screen.", + @"UDP discovery: send DISCOVER_SCREEN to port 5000; the iPhone replies SCREEN_IP_8080 because its JSON-RPC endpoint is on port 8080.", + @"Mapping logic (Python):", + @" def map_to_rlcd(pil_img):", + @" img_1bit = pil_img.convert('1', dither=1)", + @" px = img_1bit.load()", + @" buf = bytearray(15000)", + @" for y in range(300):", + @" for x in range(400):", + @" if px[x, y]:", + @" inv_y = 299 - y", + @" bx = x // 2", + @" by = inv_y // 4", + @" idx = bx * 75 + by", + @" lx, ly = x % 2, inv_y % 4", + @" bit = 7 - (ly * 2 + lx)", + @" buf[idx] |= (1 << bit)", + @" return buf" + ]]; + + if ([protocol isEqualToString:@"tcp"] || [protocol isEqualToString:@"both"] || [protocol isEqualToString:@"all"]) { + [lines addObject:@"\n**TCP Streaming (Port 8081):**"]; + [lines addObject:@"Open a TCP connection to the iPhone IP on port 8081 and send consecutive 15,000-byte frame blocks."]; + } + + if ([protocol isEqualToString:@"udp"] || [protocol isEqualToString:@"both"] || [protocol isEqualToString:@"all"]) { + [lines addObject:@"\n**UDP Streaming / Broadcast (Port 8082):**"]; + [lines addObject:@"Split each 15,000-byte frame into 15 chunks of 1,000 bytes. Send each chunk as a 1002-byte packet: byte 0 = frame_id (0-255), byte 1 = chunk_idx (0-14), bytes 2..1001 = chunk payload."]; + } + + if ([protocol isEqualToString:@"color"] || [protocol isEqualToString:@"all"]) { + [lines addObject:@"\n**RGB565 Color TCP Streaming (Port 8083):**"]; + [lines addObject:@"Send each frame as a 16-byte header followed by width*height*2 bytes of big-endian RGB565 pixels. Header: bytes 0..3='IMCR', byte 4=1 (version), byte 5=1 (RGB565BE), bytes 6..7=width big-endian, bytes 8..9=height big-endian, bytes 10..13=payload length big-endian, bytes 14..15=flags (zero). Frames may use any dimensions up to 2048x2048 and are scaled to the full iPhone canvas."]; + } + + return [lines componentsJoinedByString:@"\n"]; +} + +- (NSString *)batteryStateName:(UIDeviceBatteryState)state { + switch (state) { + case UIDeviceBatteryStateUnplugged: + return @"unplugged"; + case UIDeviceBatteryStateCharging: + return @"charging"; + case UIDeviceBatteryStateFull: + return @"full"; + default: + return @"unknown"; + } +} + +- (NSURL *)safeFileURLForPath:(NSString *)path { + if (path.length == 0 || [path hasPrefix:@"/"]) { + return nil; + } + if ([[path pathComponents] containsObject:@".."]) { + return nil; + } + + NSArray *directories = NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES); + NSString *documents = directories.firstObject; + NSString *root = [documents stringByAppendingPathComponent:@"MCPFiles"]; + return [NSURL fileURLWithPath:[root stringByAppendingPathComponent:path]]; +} + +- (BOOL)prefersStatusBarHidden { + return YES; +} + +- (UIInterfaceOrientationMask)supportedInterfaceOrientations { + return UIInterfaceOrientationMaskLandscape; +} + +- (UIInterfaceOrientation)preferredInterfaceOrientationForPresentation { + return UIInterfaceOrientationLandscapeRight; +} + +@end diff --git a/iphone_app/Makefile b/iphone_app/Makefile new file mode 100644 index 0000000..3a1957c --- /dev/null +++ b/iphone_app/Makefile @@ -0,0 +1,13 @@ +TARGET := iphone:clang:12.4:12.0 +ARCHS = arm64 +INSTALL_TARGET_PROCESSES = IPhoneMCP + +include $(THEOS)/makefiles/common.mk + +APPLICATION_NAME = IPhoneMCP + +IPhoneMCP_FILES = main.m IMCAppDelegate.m IMCViewController.m IMCCanvasView.m IMCMCPServer.m IMCVideoStreamServer.m +IPhoneMCP_FRAMEWORKS = UIKit Foundation AVFoundation CoreGraphics +IPhoneMCP_CFLAGS = -fobjc-arc -Wno-deprecated-declarations + +include $(THEOS_MAKE_PATH)/application.mk diff --git a/iphone_app/README.md b/iphone_app/README.md new file mode 100644 index 0000000..a863643 --- /dev/null +++ b/iphone_app/README.md @@ -0,0 +1,174 @@ +# iPhone MCP + +Native UIKit MCP screen server for a jailbroken arm64 iPhone running iOS 12. It shares the ESP32 screen's MCP tool shape while using the iPhone display, speaker, microphone, battery, and front camera. + +## Features + +- HTTP JSON-RPC MCP endpoint: `POST http://:8080/api/mcp` +- Full-screen landscape canvas with idle sleep disabled while the app is active +- ESP32-compatible 400x300 monochrome streaming: TCP `8081`, UDP `8082` +- Framed RGB565 color streaming: TCP `8083` +- UDP discovery on `5000`: `DISCOVER_SCREEN` -> `SCREEN_IP_8080` +- True pitched tones, WAV playback, microphone recording, and selfie capture +- Hermes voice gateway turn: record -> transcribe/agent/TTS -> play response +- Stream statistics and MCP screenshots + +## Requirements + +- Jailbroken arm64 iPhone (tested on iPhone 6 / iOS 12.5.8) +- OpenSSH and `ldid` installed on the phone +- Theos at `/var/mobile/theos` with `iPhoneOS12.4.sdk` +- The phone and build computer on the same LAN + +## Copy To The Phone + +From the repository root: + +```sh +rsync -av iphone_app/ root@:/var/mobile/IPhoneMCP/ +``` + +Keep the phone unlocked while launching the app. On iOS 12, `uiopen` may refuse to launch a locked device. + +## Build + +The included direct build script is the most reliable path on the iPhone 6: + +```sh +ssh root@ +su - mobile -c 'chmod +x /var/mobile/IPhoneMCP/manual_build_iphone.sh && /var/mobile/IPhoneMCP/manual_build_iphone.sh /var/mobile/IPhoneMCP' +``` + +This creates and signs: + +```text +/var/mobile/IPhoneMCP/.manual_build/IPhoneMCP.app +``` + +The regular Theos build is also supported: + +```sh +su - mobile -c 'env THEOS=/var/mobile/theos PATH=/usr/bin:/bin:/usr/sbin:/sbin make -C /var/mobile/IPhoneMCP clean package SDKVERSION=12.4 INCLUDE_SDKVERSION=12.4 FAKEROOT="bash /var/mobile/theos/bin/fakeroot.sh -p /var/mobile/IPhoneMCP/.theos/fakeroot" _THEOS_PLATFORM_DPKG_DEB=dpkg-deb THEOS_PLATFORM_DEB_COMPRESSION_TYPE=gzip' +``` + +## Package And Install + +For a direct-build bundle: + +```sh +PKGROOT=/var/mobile/IPhoneMCP/.manual_pkg +DEB=/var/mobile/IPhoneMCP/packages/com.reynafamily.iphonemcp_manual_iphoneos-arm.deb +rm -rf "$PKGROOT" +mkdir -p "$PKGROOT/DEBIAN" "$PKGROOT/Applications/IPhoneMCP.app" /var/mobile/IPhoneMCP/packages +cp /var/mobile/IPhoneMCP/control "$PKGROOT/DEBIAN/control" +cp /var/mobile/IPhoneMCP/.manual_build/IPhoneMCP.app/IPhoneMCP "$PKGROOT/Applications/IPhoneMCP.app/IPhoneMCP" +cp /var/mobile/IPhoneMCP/.manual_build/IPhoneMCP.app/Info.plist "$PKGROOT/Applications/IPhoneMCP.app/Info.plist" +chmod 755 "$PKGROOT/Applications/IPhoneMCP.app/IPhoneMCP" +chown -R root:wheel "$PKGROOT" +dpkg-deb -b "$PKGROOT" "$DEB" +dpkg -i "$DEB" +uicache -p /Applications/IPhoneMCP.app +uiopen com.reynafamily.iphonemcp +``` + +Verify listeners: + +```sh +netstat -an | grep -E '\.8080|\.8081|\.8082|\.8083|\.5000' +``` + +## MCP Bridge + +```sh +python3 iphone_app/iphone_mcp_bridge.py --ip +``` + +Example client configuration: + +```json +{ + "mcpServers": { + "iphone6-mcp": { + "command": "python3", + "args": ["/absolute/path/to/mcp_screen/iphone_app/iphone_mcp_bridge.py", "--ip", "192.168.68.150"] + } + } +} +``` + +## Monochrome Streaming + +- TCP `8081`: consecutive 15,000-byte ESP32 RLCD frame buffers +- UDP `8082`: fifteen 1002-byte packets per frame + - byte 0: frame id + - byte 1: chunk index 0-14 + - bytes 2-1001: 1000-byte payload + +The mapping is identical to `test_stream.py` in the repository. + +## RGB565 Color Streaming + +Color frames use TCP port `8083`. Every frame has a 16-byte big-endian header followed by RGB565 data: + +| Offset | Size | Meaning | +|---|---:|---| +| 0 | 4 | ASCII `IMCR` | +| 4 | 1 | Version `1` | +| 5 | 1 | Format `1` = RGB565 big-endian | +| 6 | 2 | Width | +| 8 | 2 | Height | +| 10 | 4 | Payload length (`width * height * 2`) | +| 14 | 2 | Flags, currently `0` | + +Run the included example: + +```sh +python3 -m pip install Pillow +python3 iphone_app/tools/stream_color.py --ip +``` + +## Hermes Voice + +The `hermes_voice_turn` MCP tool records a mono 16 kHz, 16-bit WAV, posts it to Hermes, plays the returned WAV, and returns the transcript and answer headers. + +Default gateway: + +```text +http://192.168.68.126:8642/api/esp32/voice +``` + +Example JSON-RPC call: + +```json +{ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "hermes_voice_turn", + "arguments": { + "url": "http://192.168.68.126:8642/api/esp32/voice", + "api_token": "YOUR_HERMES_API_TOKEN", + "device_id": "iphone6", + "duration_sec": 4, + "volume": 80, + "reply_mode": "sync", + "screen_url": "http://192.168.68.150:8080/api/mcp" + } + } +} +``` + +Do not commit the Hermes API token. It is accepted per call and is not persisted by the app. + +## Implemented Tools + +`clear_screen`, `draw_text`, `draw_image`, `get_screenshot`, `get_battery`, `play_tone`, `play_audio`, `play_audio_base64`, `record_voice`, `hermes_voice_turn`, `capture_selfie`, `write_file`, `read_file`, `download_file`, `get_video_streaming_instructions`, `get_stream_stats`, `reset_stream_stats`, and `set_stream_debug`. + +Compatibility stubs remain for `set_led`, `get_sensors`, `scan_ble`, and `execute_python`. + +## Operational Notes + +- Keep the app foregrounded for reliable network service on iOS 12. +- The idle timer is disabled only while the app is active. +- Give each device a unique static DHCP reservation; duplicate IPs can make traffic reach the wrong host. diff --git a/iphone_app/Resources/Info.plist b/iphone_app/Resources/Info.plist new file mode 100644 index 0000000..480d4d8 --- /dev/null +++ b/iphone_app/Resources/Info.plist @@ -0,0 +1,57 @@ + + + + + CFBundleDisplayName + iPhone MCP + CFBundleExecutable + IPhoneMCP + CFBundleIdentifier + com.reynafamily.iphonemcp + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + IPhoneMCP + CFBundlePackageType + APPL + CFBundleSupportedPlatforms + + iPhoneOS + + CFBundleShortVersionString + 0.0.4 + CFBundleVersion + 4 + MinimumOSVersion + 12.0 + LSRequiresIPhoneOS + + NSAppTransportSecurity + + NSAllowsArbitraryLoads + + + NSCameraUsageDescription + iPhone MCP uses the camera when an MCP client calls the selfie capture tool. + NSMicrophoneUsageDescription + iPhone MCP uses the microphone when an MCP client calls the voice recording tool. + UIDeviceFamily + + 1 + + UIRequiredDeviceCapabilities + + arm64 + + UIStatusBarHidden + + UIViewControllerBasedStatusBarAppearance + + UISupportedInterfaceOrientations + + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + + diff --git a/iphone_app/control b/iphone_app/control new file mode 100644 index 0000000..bf8ee2b --- /dev/null +++ b/iphone_app/control @@ -0,0 +1,8 @@ +Package: com.reynafamily.iphonemcp +Name: iPhone MCP +Version: 0.0.4 +Architecture: iphoneos-arm +Description: A native UIKit MCP-style HTTP server for a jailbroken iPhone 6. +Maintainer: Adolfo Reyna +Author: Adolfo Reyna +Section: Utilities diff --git a/iphone_app/iphone_mcp_bridge.py b/iphone_app/iphone_mcp_bridge.py new file mode 100644 index 0000000..660e32f --- /dev/null +++ b/iphone_app/iphone_mcp_bridge.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +import argparse +import json +import sys +import urllib.request +import urllib.error + + +def main(): + parser = argparse.ArgumentParser(description="MCP stdio bridge for the jailbroken iPhone MCP app") + parser.add_argument("--ip", required=True, help="iPhone IP address") + parser.add_argument("--port", type=int, default=8080, help="iPhone MCP HTTP port") + args = parser.parse_args() + + url = f"http://{args.ip}:{args.port}/api/mcp" + sys.stderr.write(f"iPhone MCP bridge routing stdio to {url}\n") + sys.stderr.flush() + + for line in sys.stdin: + try: + request = json.loads(line) + http_request = urllib.request.Request( + url, + data=json.dumps(request).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(http_request, timeout=15.0) as response: + sys.stdout.write(response.read().decode("utf-8") + "\n") + sys.stdout.flush() + except urllib.error.URLError as exc: + write_error(-32000, f"Bridge failed to reach iPhone MCP app: {exc}", request if "request" in locals() else {}) + except Exception as exc: + write_error(-32603, f"Bridge error: {exc}", request if "request" in locals() else {}) + + +def write_error(code, message, request): + sys.stdout.write(json.dumps({ + "jsonrpc": "2.0", + "error": {"code": code, "message": message}, + "id": request.get("id"), + }) + "\n") + sys.stdout.flush() + + +if __name__ == "__main__": + main() diff --git a/iphone_app/main.m b/iphone_app/main.m new file mode 100644 index 0000000..3c0f279 --- /dev/null +++ b/iphone_app/main.m @@ -0,0 +1,9 @@ +#import +#import "IMCAppDelegate.h" + +int main(int argc, char *argv[]) { + @autoreleasepool { + return UIApplicationMain(argc, argv, nil, + NSStringFromClass(IMCAppDelegate.class)); + } +} diff --git a/iphone_app/manual_build_iphone.sh b/iphone_app/manual_build_iphone.sh new file mode 100644 index 0000000..18677ac --- /dev/null +++ b/iphone_app/manual_build_iphone.sh @@ -0,0 +1,38 @@ +#!/bin/sh +set -eu + +PROJECT_DIR="${1:-/var/mobile/IPhoneMCP}" +SDK="${SDK:-/var/mobile/theos/sdks/iPhoneOS12.4.sdk}" +BUILD_DIR="$PROJECT_DIR/.manual_build" +OBJ_DIR="$BUILD_DIR/obj" +APP_DIR="$BUILD_DIR/IPhoneMCP.app" + +mkdir -p "$OBJ_DIR" "$APP_DIR" +cp "$PROJECT_DIR/Resources/Info.plist" "$APP_DIR/Info.plist" + +CFLAGS="-arch arm64 -isysroot $SDK -miphoneos-version-min=12.0 -fobjc-arc -Wno-deprecated-declarations -I$PROJECT_DIR" +SOURCES="main.m IMCAppDelegate.m IMCViewController.m IMCCanvasView.m IMCMCPServer.m IMCVideoStreamServer.m" +OBJECTS="" + +for src in $SOURCES; do + obj="$OBJ_DIR/${src%.m}.o" + echo "Compiling $src" + clang $CFLAGS -c "$PROJECT_DIR/$src" -o "$obj" + OBJECTS="$OBJECTS $obj" +done + +echo "Linking IPhoneMCP" +clang -arch arm64 \ + -isysroot "$SDK" \ + -miphoneos-version-min=12.0 \ + -o "$APP_DIR/IPhoneMCP" \ + $OBJECTS \ + -framework UIKit \ + -framework Foundation \ + -framework AVFoundation \ + -framework CoreGraphics + +echo "Signing IPhoneMCP" +ldid -S "$APP_DIR/IPhoneMCP" + +echo "$APP_DIR" diff --git a/iphone_app/tools/mock_hermes_gateway.py b/iphone_app/tools/mock_hermes_gateway.py new file mode 100644 index 0000000..97951df --- /dev/null +++ b/iphone_app/tools/mock_hermes_gateway.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Minimal Hermes-compatible voice endpoint for iPhone integration testing.""" + +import argparse +import io +import math +import struct +import wave +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + +def response_wav() -> bytes: + output = io.BytesIO() + with wave.open(output, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16000) + frames = bytearray() + for index in range(int(16000 * 0.3)): + sample = int(7000 * math.sin(2 * math.pi * 660 * index / 16000)) + frames.extend(struct.pack(" bytes: + image = image.convert("RGB") + output = bytearray(image.width * image.height * 2) + for index, (red, green, blue) in enumerate(image.getdata()): + value = ((red >> 3) << 11) | ((green >> 2) << 5) | (blue >> 3) + output[index * 2] = value >> 8 + output[index * 2 + 1] = value & 0xFF + return bytes(output) + + +def send_frame(sock: socket.socket, image: Image.Image) -> None: + payload = rgb565be(image) + header = struct.pack(">4sBBHHIH", b"IMCR", 1, 1, image.width, image.height, len(payload), 0) + sock.sendall(header + payload) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--ip", required=True, help="iPhone Wi-Fi IP") + parser.add_argument("--port", type=int, default=8083) + parser.add_argument("--width", type=int, default=400) + parser.add_argument("--height", type=int, default=300) + args = parser.parse_args() + + with socket.create_connection((args.ip, args.port), timeout=8) as sock: + frame = 0 + while True: + hue = (frame * 4) % 256 + image = Image.new("RGB", (args.width, args.height), (12, 18, 35)) + draw = ImageDraw.Draw(image) + for x in range(args.width): + red = (x * 255 // max(1, args.width - 1) + hue) % 256 + blue = 255 - (x * 255 // max(1, args.width - 1)) + draw.line((x, 0, x, args.height - 1), fill=(red, 70, blue)) + draw.rounded_rectangle((20, 20, args.width - 20, args.height - 20), radius=22, + outline=(255, 255, 255), width=4) + draw.text((40, 45), "iPhone MCP RGB565", fill=(255, 255, 255)) + draw.text((40, 72), f"Frame {frame}", fill=(255, 240, 80)) + send_frame(sock, image) + frame += 1 + time.sleep(1 / 20) + + +if __name__ == "__main__": + main()