Add jailbroken iPhone MCP screen app
This commit is contained in:
Adolfo Reyna
@@ -0,0 +1,5 @@
|
||||
#import <UIKit/UIKit.h>
|
||||
|
||||
@interface IMCAppDelegate : UIResponder <UIApplicationDelegate>
|
||||
@property (nonatomic, strong) UIWindow *window;
|
||||
@end
|
||||
@@ -0,0 +1,23 @@
|
||||
#import "IMCAppDelegate.h"
|
||||
#import "IMCViewController.h"
|
||||
|
||||
@implementation IMCAppDelegate
|
||||
|
||||
- (BOOL)application:(UIApplication *)application
|
||||
didFinishLaunchingWithOptions:(NSDictionary *)launchOptions {
|
||||
application.idleTimerDisabled = YES;
|
||||
self.window = [[UIWindow alloc] initWithFrame:[UIScreen mainScreen].bounds];
|
||||
self.window.rootViewController = [[IMCViewController alloc] init];
|
||||
[self.window makeKeyAndVisible];
|
||||
return YES;
|
||||
}
|
||||
|
||||
- (void)applicationDidBecomeActive:(UIApplication *)application {
|
||||
application.idleTimerDisabled = YES;
|
||||
}
|
||||
|
||||
- (void)applicationWillTerminate:(UIApplication *)application {
|
||||
application.idleTimerDisabled = NO;
|
||||
}
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,10 @@
|
||||
#import <UIKit/UIKit.h>
|
||||
|
||||
@interface IMCCanvasView : UIView
|
||||
- (void)clearWithColor:(NSInteger)color;
|
||||
- (void)drawText:(NSString *)text x:(CGFloat)x y:(CGFloat)y size:(NSInteger)size;
|
||||
- (void)drawImageData:(NSData *)data x:(CGFloat)x y:(CGFloat)y;
|
||||
- (void)displayMonochromeFrameBuffer:(NSData *)frameData;
|
||||
- (BOOL)displayRGB565FrameBuffer:(NSData *)frameData width:(NSUInteger)width height:(NSUInteger)height;
|
||||
- (NSString *)snapshotPNGBase64;
|
||||
@end
|
||||
@@ -0,0 +1,218 @@
|
||||
#import "IMCCanvasView.h"
|
||||
|
||||
@interface IMCCanvasView ()
|
||||
@property (nonatomic, assign) NSInteger clearColor;
|
||||
@property (nonatomic, strong) NSMutableArray *commands;
|
||||
@property (nonatomic, strong) UIImage *streamImage;
|
||||
@end
|
||||
|
||||
@implementation IMCCanvasView
|
||||
|
||||
- (instancetype)initWithFrame:(CGRect)frame {
|
||||
self = [super initWithFrame:frame];
|
||||
if (self) {
|
||||
_clearColor = 0;
|
||||
_commands = [NSMutableArray array];
|
||||
self.opaque = YES;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)clearWithColor:(NSInteger)color {
|
||||
self.clearColor = color == 1 ? 1 : 0;
|
||||
self.streamImage = nil;
|
||||
[self.commands removeAllObjects];
|
||||
[self setNeedsDisplay];
|
||||
}
|
||||
|
||||
- (void)drawText:(NSString *)text x:(CGFloat)x y:(CGFloat)y size:(NSInteger)size {
|
||||
if (!text) {
|
||||
text = @"";
|
||||
}
|
||||
self.streamImage = nil;
|
||||
[self.commands addObject:@{
|
||||
@"type": @"text",
|
||||
@"text": text,
|
||||
@"x": @(x),
|
||||
@"y": @(y),
|
||||
@"size": @(size == 2 ? 2 : 1)
|
||||
}];
|
||||
[self setNeedsDisplay];
|
||||
}
|
||||
|
||||
- (void)drawImageData:(NSData *)data x:(CGFloat)x y:(CGFloat)y {
|
||||
UIImage *image = [UIImage imageWithData:data];
|
||||
if (!image) {
|
||||
return;
|
||||
}
|
||||
self.streamImage = nil;
|
||||
[self.commands addObject:@{
|
||||
@"type": @"image",
|
||||
@"image": image,
|
||||
@"x": @(x),
|
||||
@"y": @(y)
|
||||
}];
|
||||
[self setNeedsDisplay];
|
||||
}
|
||||
|
||||
- (void)displayMonochromeFrameBuffer:(NSData *)frameData {
|
||||
if (frameData.length < 15000) {
|
||||
return;
|
||||
}
|
||||
|
||||
const NSUInteger width = 400;
|
||||
const NSUInteger height = 300;
|
||||
NSMutableData *rgba = [NSMutableData dataWithLength:width * height * 4];
|
||||
const uint8_t *source = frameData.bytes;
|
||||
uint8_t *pixels = rgba.mutableBytes;
|
||||
|
||||
for (NSUInteger y = 0; y < height; y++) {
|
||||
NSUInteger invY = height - 1 - y;
|
||||
for (NSUInteger x = 0; x < width; x++) {
|
||||
NSUInteger bx = x / 2;
|
||||
NSUInteger by = invY / 4;
|
||||
NSUInteger index = bx * 75 + by;
|
||||
NSUInteger bit = 7 - ((invY % 4) * 2 + (x % 2));
|
||||
BOOL on = (source[index] & (1 << bit)) != 0;
|
||||
uint8_t value = on ? 255 : 0;
|
||||
NSUInteger pixelIndex = (y * width + x) * 4;
|
||||
pixels[pixelIndex + 0] = value;
|
||||
pixels[pixelIndex + 1] = value;
|
||||
pixels[pixelIndex + 2] = value;
|
||||
pixels[pixelIndex + 3] = 255;
|
||||
}
|
||||
}
|
||||
|
||||
CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
|
||||
CGDataProviderRef provider = CGDataProviderCreateWithCFData((__bridge CFDataRef)rgba);
|
||||
CGImageRef cgImage = CGImageCreate(width,
|
||||
height,
|
||||
8,
|
||||
32,
|
||||
width * 4,
|
||||
colorSpace,
|
||||
kCGImageAlphaLast | kCGBitmapByteOrderDefault,
|
||||
provider,
|
||||
NULL,
|
||||
false,
|
||||
kCGRenderingIntentDefault);
|
||||
if (cgImage) {
|
||||
self.streamImage = [UIImage imageWithCGImage:cgImage];
|
||||
[self.commands removeAllObjects];
|
||||
[self setNeedsDisplay];
|
||||
CGImageRelease(cgImage);
|
||||
}
|
||||
CGDataProviderRelease(provider);
|
||||
CGColorSpaceRelease(colorSpace);
|
||||
}
|
||||
|
||||
- (BOOL)displayRGB565FrameBuffer:(NSData *)frameData width:(NSUInteger)width height:(NSUInteger)height {
|
||||
if (width == 0 || height == 0 || width > 2048 || height > 2048 ||
|
||||
frameData.length != width * height * 2) {
|
||||
return NO;
|
||||
}
|
||||
|
||||
const uint8_t *source = frameData.bytes;
|
||||
NSMutableData *rgba = [NSMutableData dataWithLength:width * height * 4];
|
||||
uint8_t *pixels = rgba.mutableBytes;
|
||||
for (NSUInteger i = 0; i < width * height; i++) {
|
||||
// Network color frames use big-endian RGB565.
|
||||
uint16_t value = ((uint16_t)source[i * 2] << 8) | source[i * 2 + 1];
|
||||
uint8_t red5 = (value >> 11) & 0x1f;
|
||||
uint8_t green6 = (value >> 5) & 0x3f;
|
||||
uint8_t blue5 = value & 0x1f;
|
||||
pixels[i * 4 + 0] = (red5 << 3) | (red5 >> 2);
|
||||
pixels[i * 4 + 1] = (green6 << 2) | (green6 >> 4);
|
||||
pixels[i * 4 + 2] = (blue5 << 3) | (blue5 >> 2);
|
||||
pixels[i * 4 + 3] = 255;
|
||||
}
|
||||
|
||||
CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
|
||||
CGDataProviderRef provider = CGDataProviderCreateWithCFData((__bridge CFDataRef)rgba);
|
||||
CGImageRef cgImage = CGImageCreate(width,
|
||||
height,
|
||||
8,
|
||||
32,
|
||||
width * 4,
|
||||
colorSpace,
|
||||
kCGImageAlphaLast | kCGBitmapByteOrderDefault,
|
||||
provider,
|
||||
NULL,
|
||||
false,
|
||||
kCGRenderingIntentDefault);
|
||||
if (cgImage) {
|
||||
self.streamImage = [UIImage imageWithCGImage:cgImage];
|
||||
[self.commands removeAllObjects];
|
||||
[self setNeedsDisplay];
|
||||
CGImageRelease(cgImage);
|
||||
}
|
||||
CGDataProviderRelease(provider);
|
||||
CGColorSpaceRelease(colorSpace);
|
||||
return cgImage != NULL;
|
||||
}
|
||||
|
||||
- (NSString *)snapshotPNGBase64 {
|
||||
UIGraphicsBeginImageContextWithOptions(self.bounds.size, YES, 0.0);
|
||||
[self drawViewHierarchyInRect:self.bounds afterScreenUpdates:YES];
|
||||
UIImage *image = UIGraphicsGetImageFromCurrentImageContext();
|
||||
UIGraphicsEndImageContext();
|
||||
|
||||
NSData *pngData = UIImagePNGRepresentation(image);
|
||||
return [pngData base64EncodedStringWithOptions:0];
|
||||
}
|
||||
|
||||
- (CGPoint)pointForDeviceX:(CGFloat)x y:(CGFloat)y {
|
||||
CGFloat scaleX = self.bounds.size.width / 400.0;
|
||||
CGFloat scaleY = self.bounds.size.height / 300.0;
|
||||
return CGPointMake(x * scaleX, y * scaleY);
|
||||
}
|
||||
|
||||
- (void)drawRect:(CGRect)rect {
|
||||
UIColor *background = self.clearColor == 1 ? [UIColor blackColor] : [UIColor whiteColor];
|
||||
UIColor *foreground = self.clearColor == 1 ? [UIColor whiteColor] : [UIColor blackColor];
|
||||
|
||||
[background setFill];
|
||||
UIRectFill(self.bounds);
|
||||
|
||||
if (self.streamImage) {
|
||||
[self.streamImage drawInRect:self.bounds];
|
||||
return;
|
||||
}
|
||||
|
||||
CGContextRef context = UIGraphicsGetCurrentContext();
|
||||
CGContextSetStrokeColorWithColor(context, foreground.CGColor);
|
||||
CGContextSetLineWidth(context, 2.0);
|
||||
CGContextStrokeRect(context, CGRectInset(self.bounds, 1.0, 1.0));
|
||||
|
||||
for (NSDictionary *command in self.commands) {
|
||||
NSString *type = command[@"type"];
|
||||
CGFloat x = [command[@"x"] doubleValue];
|
||||
CGFloat y = [command[@"y"] doubleValue];
|
||||
CGPoint point = [self pointForDeviceX:x y:y];
|
||||
|
||||
if ([type isEqualToString:@"text"]) {
|
||||
NSInteger size = [command[@"size"] integerValue];
|
||||
UIFont *font = [UIFont boldSystemFontOfSize:size == 2 ? 26.0 : 15.0];
|
||||
NSDictionary *attrs = @{
|
||||
NSFontAttributeName: font,
|
||||
NSForegroundColorAttributeName: foreground
|
||||
};
|
||||
[command[@"text"] drawAtPoint:point withAttributes:attrs];
|
||||
} else if ([type isEqualToString:@"image"]) {
|
||||
UIImage *image = command[@"image"];
|
||||
CGFloat maxWidth = self.bounds.size.width - point.x;
|
||||
CGFloat maxHeight = self.bounds.size.height - point.y;
|
||||
CGSize imageSize = image.size;
|
||||
CGFloat scale = MIN(maxWidth / MAX(imageSize.width, 1.0),
|
||||
maxHeight / MAX(imageSize.height, 1.0));
|
||||
scale = MIN(scale, 1.0);
|
||||
CGRect imageRect = CGRectMake(point.x,
|
||||
point.y,
|
||||
imageSize.width * scale,
|
||||
imageSize.height * scale);
|
||||
[image drawInRect:imageRect];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,8 @@
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
@class IMCViewController;
|
||||
|
||||
@interface IMCMCPServer : NSObject
|
||||
- (instancetype)initWithViewController:(IMCViewController *)viewController port:(NSInteger)port;
|
||||
- (void)start;
|
||||
@end
|
||||
@@ -0,0 +1,163 @@
|
||||
#import "IMCMCPServer.h"
|
||||
#import "IMCViewController.h"
|
||||
#import <arpa/inet.h>
|
||||
#import <netinet/in.h>
|
||||
#import <sys/socket.h>
|
||||
#import <unistd.h>
|
||||
|
||||
@interface IMCMCPServer ()
|
||||
@property (nonatomic, weak) IMCViewController *viewController;
|
||||
@property (nonatomic, assign) NSInteger port;
|
||||
@property (nonatomic, assign) BOOL running;
|
||||
@end
|
||||
|
||||
@implementation IMCMCPServer
|
||||
|
||||
- (instancetype)initWithViewController:(IMCViewController *)viewController port:(NSInteger)port {
|
||||
self = [super init];
|
||||
if (self) {
|
||||
_viewController = viewController;
|
||||
_port = port;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)start {
|
||||
if (self.running) {
|
||||
return;
|
||||
}
|
||||
self.running = YES;
|
||||
[NSThread detachNewThreadSelector:@selector(serverLoop) toTarget:self withObject:nil];
|
||||
}
|
||||
|
||||
- (void)serverLoop {
|
||||
@autoreleasepool {
|
||||
int serverSocket = socket(AF_INET, SOCK_STREAM, 0);
|
||||
if (serverSocket < 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
int yes = 1;
|
||||
setsockopt(serverSocket, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes));
|
||||
|
||||
struct sockaddr_in address;
|
||||
memset(&address, 0, sizeof(address));
|
||||
address.sin_family = AF_INET;
|
||||
address.sin_addr.s_addr = INADDR_ANY;
|
||||
address.sin_port = htons((uint16_t)self.port);
|
||||
|
||||
if (bind(serverSocket, (struct sockaddr *)&address, sizeof(address)) < 0) {
|
||||
close(serverSocket);
|
||||
return;
|
||||
}
|
||||
|
||||
if (listen(serverSocket, 8) < 0) {
|
||||
close(serverSocket);
|
||||
return;
|
||||
}
|
||||
|
||||
while (self.running) {
|
||||
@autoreleasepool {
|
||||
struct sockaddr_in clientAddress;
|
||||
socklen_t clientLength = sizeof(clientAddress);
|
||||
int clientSocket = accept(serverSocket,
|
||||
(struct sockaddr *)&clientAddress,
|
||||
&clientLength);
|
||||
if (clientSocket >= 0) {
|
||||
[self handleClientSocket:clientSocket];
|
||||
close(clientSocket);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
close(serverSocket);
|
||||
}
|
||||
}
|
||||
|
||||
- (void)handleClientSocket:(int)clientSocket {
|
||||
NSMutableData *requestData = [NSMutableData data];
|
||||
char buffer[2048];
|
||||
|
||||
while (requestData.length < 1024 * 1024) {
|
||||
ssize_t count = recv(clientSocket, buffer, sizeof(buffer), 0);
|
||||
if (count <= 0) {
|
||||
break;
|
||||
}
|
||||
[requestData appendBytes:buffer length:(NSUInteger)count];
|
||||
|
||||
NSString *partial = [[NSString alloc] initWithData:requestData encoding:NSUTF8StringEncoding];
|
||||
NSRange separator = [partial rangeOfString:@"\r\n\r\n"];
|
||||
if (separator.location == NSNotFound) {
|
||||
continue;
|
||||
}
|
||||
|
||||
NSInteger contentLength = [self contentLengthFromHeader:partial];
|
||||
NSUInteger bodyStart = separator.location + separator.length;
|
||||
if (requestData.length >= bodyStart + MAX(contentLength, 0)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
NSString *request = [[NSString alloc] initWithData:requestData encoding:NSUTF8StringEncoding];
|
||||
NSDictionary *responseObject = [self responseObjectForHTTPRequest:request];
|
||||
NSData *responseJSON = [NSJSONSerialization dataWithJSONObject:responseObject options:0 error:nil];
|
||||
if (!responseJSON) {
|
||||
responseJSON = [@"{}" dataUsingEncoding:NSUTF8StringEncoding];
|
||||
}
|
||||
|
||||
NSString *status = responseObject[@"httpStatus"] ?: @"200 OK";
|
||||
NSMutableDictionary *bodyObject = [responseObject mutableCopy];
|
||||
[bodyObject removeObjectForKey:@"httpStatus"];
|
||||
responseJSON = [NSJSONSerialization dataWithJSONObject:bodyObject options:0 error:nil];
|
||||
|
||||
NSMutableData *responseData = [NSMutableData data];
|
||||
NSString *header = [NSString stringWithFormat:
|
||||
@"HTTP/1.1 %@\r\nContent-Type: application/json\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n",
|
||||
status,
|
||||
(unsigned long)responseJSON.length];
|
||||
[responseData appendData:[header dataUsingEncoding:NSUTF8StringEncoding]];
|
||||
[responseData appendData:responseJSON];
|
||||
send(clientSocket, responseData.bytes, responseData.length, 0);
|
||||
}
|
||||
|
||||
- (NSDictionary *)responseObjectForHTTPRequest:(NSString *)request {
|
||||
if (![request hasPrefix:@"POST /api/mcp "]) {
|
||||
return @{@"httpStatus": @"404 Not Found",
|
||||
@"jsonrpc": @"2.0",
|
||||
@"error": @{@"code": @-32601, @"message": @"POST /api/mcp required"},
|
||||
@"id": [NSNull null]};
|
||||
}
|
||||
|
||||
NSRange separator = [request rangeOfString:@"\r\n\r\n"];
|
||||
if (separator.location == NSNotFound) {
|
||||
return [self parseError:@"Missing HTTP body."];
|
||||
}
|
||||
|
||||
NSString *body = [request substringFromIndex:separator.location + separator.length];
|
||||
NSData *bodyData = [body dataUsingEncoding:NSUTF8StringEncoding];
|
||||
NSError *jsonError = nil;
|
||||
NSDictionary *rpcRequest = [NSJSONSerialization JSONObjectWithData:bodyData options:0 error:&jsonError];
|
||||
if (jsonError || ![rpcRequest isKindOfClass:NSDictionary.class]) {
|
||||
return [self parseError:@"Invalid JSON-RPC object."];
|
||||
}
|
||||
|
||||
return [self.viewController handleRPCRequest:rpcRequest];
|
||||
}
|
||||
|
||||
- (NSDictionary *)parseError:(NSString *)message {
|
||||
return @{@"jsonrpc": @"2.0",
|
||||
@"error": @{@"code": @-32700, @"message": message ?: @"Parse error"},
|
||||
@"id": [NSNull null]};
|
||||
}
|
||||
|
||||
- (NSInteger)contentLengthFromHeader:(NSString *)request {
|
||||
NSArray *lines = [request componentsSeparatedByString:@"\r\n"];
|
||||
for (NSString *line in lines) {
|
||||
if ([[line lowercaseString] hasPrefix:@"content-length:"]) {
|
||||
return [[line substringFromIndex:15] integerValue];
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,11 @@
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
@class IMCCanvasView;
|
||||
|
||||
@interface IMCVideoStreamServer : NSObject
|
||||
- (instancetype)initWithCanvasView:(IMCCanvasView *)canvasView;
|
||||
- (void)start;
|
||||
- (NSDictionary *)streamStats;
|
||||
- (void)resetStreamStats;
|
||||
- (void)setDebugEnabled:(BOOL)enabled;
|
||||
@end
|
||||
@@ -0,0 +1,447 @@
|
||||
#import "IMCVideoStreamServer.h"
|
||||
#import "IMCCanvasView.h"
|
||||
#import <arpa/inet.h>
|
||||
#import <netinet/in.h>
|
||||
#import <sys/socket.h>
|
||||
#import <string.h>
|
||||
#import <unistd.h>
|
||||
|
||||
static const NSInteger IMCFrameSize = 15000;
|
||||
static const NSInteger IMCUDPChunkSize = 1000;
|
||||
static const NSInteger IMCUDPChunkCount = 15;
|
||||
static const NSInteger IMCColorStreamPort = 8083;
|
||||
static const NSUInteger IMCColorHeaderSize = 16;
|
||||
|
||||
@interface IMCVideoStreamServer ()
|
||||
@property (nonatomic, weak) IMCCanvasView *canvasView;
|
||||
@property (nonatomic, assign) BOOL running;
|
||||
@property (nonatomic, assign) BOOL debugEnabled;
|
||||
@property (nonatomic, strong) NSDate *startedAt;
|
||||
@property (nonatomic, strong) NSDate *lastFrameAt;
|
||||
@property (nonatomic, copy) NSString *lastProtocol;
|
||||
@property (nonatomic, assign) uint64_t tcpConnections;
|
||||
@property (nonatomic, assign) uint64_t tcpFrames;
|
||||
@property (nonatomic, assign) uint64_t tcpBytes;
|
||||
@property (nonatomic, assign) uint64_t udpFrames;
|
||||
@property (nonatomic, assign) uint64_t udpPackets;
|
||||
@property (nonatomic, assign) uint64_t udpBytes;
|
||||
@property (nonatomic, assign) uint64_t invalidUDPPackets;
|
||||
@property (nonatomic, assign) uint64_t discoveryRequests;
|
||||
@property (nonatomic, assign) uint64_t colorConnections;
|
||||
@property (nonatomic, assign) uint64_t colorFrames;
|
||||
@property (nonatomic, assign) uint64_t colorBytes;
|
||||
@property (nonatomic, assign) NSUInteger lastColorWidth;
|
||||
@property (nonatomic, assign) NSUInteger lastColorHeight;
|
||||
@end
|
||||
|
||||
@implementation IMCVideoStreamServer
|
||||
|
||||
- (instancetype)initWithCanvasView:(IMCCanvasView *)canvasView {
|
||||
self = [super init];
|
||||
if (self) {
|
||||
_canvasView = canvasView;
|
||||
_startedAt = [NSDate date];
|
||||
_lastProtocol = @"none";
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)start {
|
||||
if (self.running) {
|
||||
return;
|
||||
}
|
||||
self.running = YES;
|
||||
[NSThread detachNewThreadSelector:@selector(tcpLoop) toTarget:self withObject:nil];
|
||||
[NSThread detachNewThreadSelector:@selector(udpLoop) toTarget:self withObject:nil];
|
||||
[NSThread detachNewThreadSelector:@selector(discoveryLoop) toTarget:self withObject:nil];
|
||||
[NSThread detachNewThreadSelector:@selector(colorTCPLoop) toTarget:self withObject:nil];
|
||||
}
|
||||
|
||||
- (BOOL)readExactly:(NSUInteger)length fromSocket:(int)socketFD intoData:(NSMutableData *)data {
|
||||
[data setLength:length];
|
||||
NSUInteger received = 0;
|
||||
while (self.running && received < length) {
|
||||
ssize_t count = recv(socketFD,
|
||||
(uint8_t *)data.mutableBytes + received,
|
||||
length - received,
|
||||
0);
|
||||
if (count <= 0) {
|
||||
return NO;
|
||||
}
|
||||
received += (NSUInteger)count;
|
||||
}
|
||||
return received == length;
|
||||
}
|
||||
|
||||
- (void)colorTCPLoop {
|
||||
@autoreleasepool {
|
||||
int serverSocket = socket(AF_INET, SOCK_STREAM, 0);
|
||||
if (serverSocket < 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
int yes = 1;
|
||||
setsockopt(serverSocket, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes));
|
||||
struct sockaddr_in address;
|
||||
memset(&address, 0, sizeof(address));
|
||||
address.sin_family = AF_INET;
|
||||
address.sin_addr.s_addr = INADDR_ANY;
|
||||
address.sin_port = htons((uint16_t)IMCColorStreamPort);
|
||||
if (bind(serverSocket, (struct sockaddr *)&address, sizeof(address)) < 0 ||
|
||||
listen(serverSocket, 1) < 0) {
|
||||
close(serverSocket);
|
||||
return;
|
||||
}
|
||||
|
||||
while (self.running) {
|
||||
@autoreleasepool {
|
||||
int clientSocket = accept(serverSocket, NULL, NULL);
|
||||
if (clientSocket < 0) {
|
||||
continue;
|
||||
}
|
||||
@synchronized (self) {
|
||||
self.colorConnections += 1;
|
||||
}
|
||||
|
||||
NSMutableData *header = [NSMutableData data];
|
||||
while ([self readExactly:IMCColorHeaderSize
|
||||
fromSocket:clientSocket
|
||||
intoData:header]) {
|
||||
const uint8_t *bytes = header.bytes;
|
||||
if (memcmp(bytes, "IMCR", 4) != 0 || bytes[4] != 1 || bytes[5] != 1) {
|
||||
break;
|
||||
}
|
||||
NSUInteger width = ((NSUInteger)bytes[6] << 8) | bytes[7];
|
||||
NSUInteger height = ((NSUInteger)bytes[8] << 8) | bytes[9];
|
||||
uint32_t payloadLength = ((uint32_t)bytes[10] << 24) |
|
||||
((uint32_t)bytes[11] << 16) |
|
||||
((uint32_t)bytes[12] << 8) |
|
||||
bytes[13];
|
||||
if (width == 0 || height == 0 || width > 2048 || height > 2048 ||
|
||||
payloadLength != width * height * 2 || payloadLength > 8 * 1024 * 1024) {
|
||||
break;
|
||||
}
|
||||
|
||||
NSMutableData *payload = [NSMutableData data];
|
||||
if (![self readExactly:payloadLength
|
||||
fromSocket:clientSocket
|
||||
intoData:payload]) {
|
||||
break;
|
||||
}
|
||||
NSData *snapshot = [payload copy];
|
||||
@synchronized (self) {
|
||||
self.colorFrames += 1;
|
||||
self.colorBytes += payloadLength;
|
||||
self.lastColorWidth = width;
|
||||
self.lastColorHeight = height;
|
||||
self.lastProtocol = @"color_tcp";
|
||||
self.lastFrameAt = [NSDate date];
|
||||
}
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
[self.canvasView displayRGB565FrameBuffer:snapshot width:width height:height];
|
||||
});
|
||||
}
|
||||
close(clientSocket);
|
||||
}
|
||||
}
|
||||
close(serverSocket);
|
||||
}
|
||||
}
|
||||
|
||||
- (void)tcpLoop {
|
||||
@autoreleasepool {
|
||||
int serverSocket = socket(AF_INET, SOCK_STREAM, 0);
|
||||
if (serverSocket < 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
int yes = 1;
|
||||
setsockopt(serverSocket, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes));
|
||||
|
||||
struct sockaddr_in address;
|
||||
memset(&address, 0, sizeof(address));
|
||||
address.sin_family = AF_INET;
|
||||
address.sin_addr.s_addr = INADDR_ANY;
|
||||
address.sin_port = htons(8081);
|
||||
|
||||
if (bind(serverSocket, (struct sockaddr *)&address, sizeof(address)) < 0) {
|
||||
close(serverSocket);
|
||||
return;
|
||||
}
|
||||
|
||||
if (listen(serverSocket, 1) < 0) {
|
||||
close(serverSocket);
|
||||
return;
|
||||
}
|
||||
|
||||
while (self.running) {
|
||||
@autoreleasepool {
|
||||
int clientSocket = accept(serverSocket, NULL, NULL);
|
||||
if (clientSocket < 0) {
|
||||
continue;
|
||||
}
|
||||
[self noteTCPConnection];
|
||||
|
||||
NSMutableData *frame = [NSMutableData dataWithLength:IMCFrameSize];
|
||||
NSUInteger received = 0;
|
||||
while (self.running) {
|
||||
ssize_t count = recv(clientSocket,
|
||||
(uint8_t *)frame.mutableBytes + received,
|
||||
IMCFrameSize - received,
|
||||
0);
|
||||
if (count <= 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
received += (NSUInteger)count;
|
||||
[self noteTCPBytes:(NSUInteger)count];
|
||||
if (received == IMCFrameSize) {
|
||||
[self displayFrame:frame protocol:@"tcp"];
|
||||
received = 0;
|
||||
}
|
||||
}
|
||||
|
||||
close(clientSocket);
|
||||
}
|
||||
}
|
||||
|
||||
close(serverSocket);
|
||||
}
|
||||
}
|
||||
|
||||
- (void)udpLoop {
|
||||
@autoreleasepool {
|
||||
int udpSocket = socket(AF_INET, SOCK_DGRAM, 0);
|
||||
if (udpSocket < 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
int yes = 1;
|
||||
setsockopt(udpSocket, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes));
|
||||
|
||||
struct sockaddr_in address;
|
||||
memset(&address, 0, sizeof(address));
|
||||
address.sin_family = AF_INET;
|
||||
address.sin_addr.s_addr = INADDR_ANY;
|
||||
address.sin_port = htons(8082);
|
||||
|
||||
if (bind(udpSocket, (struct sockaddr *)&address, sizeof(address)) < 0) {
|
||||
close(udpSocket);
|
||||
return;
|
||||
}
|
||||
|
||||
NSMutableData *frame = [NSMutableData dataWithLength:IMCFrameSize];
|
||||
uint8_t currentFrameID = 0;
|
||||
uint16_t chunksReceived = 0;
|
||||
BOOL hasFrame = NO;
|
||||
uint8_t packet[1002];
|
||||
|
||||
while (self.running) {
|
||||
ssize_t count = recvfrom(udpSocket, packet, sizeof(packet), 0, NULL, NULL);
|
||||
if (count < 2) {
|
||||
[self noteInvalidUDPPacket];
|
||||
continue;
|
||||
}
|
||||
|
||||
uint8_t frameID = packet[0];
|
||||
uint8_t chunkIndex = packet[1];
|
||||
if (chunkIndex >= IMCUDPChunkCount || count < 2 + IMCUDPChunkSize) {
|
||||
[self noteInvalidUDPPacket];
|
||||
continue;
|
||||
}
|
||||
[self noteUDPPacketBytes:(NSUInteger)count];
|
||||
|
||||
if (!hasFrame || frameID != currentFrameID) {
|
||||
currentFrameID = frameID;
|
||||
chunksReceived = 0;
|
||||
hasFrame = YES;
|
||||
}
|
||||
|
||||
memcpy((uint8_t *)frame.mutableBytes + chunkIndex * IMCUDPChunkSize,
|
||||
packet + 2,
|
||||
IMCUDPChunkSize);
|
||||
chunksReceived |= (uint16_t)(1 << chunkIndex);
|
||||
|
||||
if (chunksReceived == 0x7fff) {
|
||||
[self displayFrame:frame protocol:@"udp"];
|
||||
chunksReceived = 0;
|
||||
}
|
||||
}
|
||||
|
||||
close(udpSocket);
|
||||
}
|
||||
}
|
||||
|
||||
- (void)discoveryLoop {
|
||||
@autoreleasepool {
|
||||
int udpSocket = socket(AF_INET, SOCK_DGRAM, 0);
|
||||
if (udpSocket < 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
int yes = 1;
|
||||
setsockopt(udpSocket, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes));
|
||||
setsockopt(udpSocket, SOL_SOCKET, SO_BROADCAST, &yes, sizeof(yes));
|
||||
|
||||
struct sockaddr_in address;
|
||||
memset(&address, 0, sizeof(address));
|
||||
address.sin_family = AF_INET;
|
||||
address.sin_addr.s_addr = INADDR_ANY;
|
||||
address.sin_port = htons(5000);
|
||||
|
||||
if (bind(udpSocket, (struct sockaddr *)&address, sizeof(address)) < 0) {
|
||||
close(udpSocket);
|
||||
return;
|
||||
}
|
||||
|
||||
char buffer[64];
|
||||
while (self.running) {
|
||||
struct sockaddr_in clientAddress;
|
||||
socklen_t clientLength = sizeof(clientAddress);
|
||||
ssize_t count = recvfrom(udpSocket,
|
||||
buffer,
|
||||
sizeof(buffer),
|
||||
0,
|
||||
(struct sockaddr *)&clientAddress,
|
||||
&clientLength);
|
||||
if (count == 15 && memcmp(buffer, "DISCOVER_SCREEN", 15) == 0) {
|
||||
[self noteDiscoveryRequest];
|
||||
const char *reply = "SCREEN_IP_8080";
|
||||
sendto(udpSocket,
|
||||
reply,
|
||||
strlen(reply),
|
||||
0,
|
||||
(struct sockaddr *)&clientAddress,
|
||||
clientLength);
|
||||
}
|
||||
}
|
||||
|
||||
close(udpSocket);
|
||||
}
|
||||
}
|
||||
|
||||
- (void)displayFrame:(NSData *)frame protocol:(NSString *)protocol {
|
||||
[self noteFrameForProtocol:protocol];
|
||||
NSData *snapshot = [frame copy];
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
[self.canvasView displayMonochromeFrameBuffer:snapshot];
|
||||
});
|
||||
}
|
||||
|
||||
- (NSDictionary *)streamStats {
|
||||
@synchronized (self) {
|
||||
NSTimeInterval uptime = [[NSDate date] timeIntervalSinceDate:self.startedAt ?: [NSDate date]];
|
||||
NSTimeInterval secondsSinceLastFrame = self.lastFrameAt ? [[NSDate date] timeIntervalSinceDate:self.lastFrameAt] : -1.0;
|
||||
return @{
|
||||
@"debug_enabled": @(self.debugEnabled),
|
||||
@"running": @(self.running),
|
||||
@"tcp_port": @8081,
|
||||
@"udp_port": @8082,
|
||||
@"color_tcp_port": @(IMCColorStreamPort),
|
||||
@"discovery_port": @5000,
|
||||
@"frame_size_bytes": @(IMCFrameSize),
|
||||
@"udp_chunk_size_bytes": @(IMCUDPChunkSize),
|
||||
@"udp_chunks_per_frame": @(IMCUDPChunkCount),
|
||||
@"uptime_sec": @((NSInteger)uptime),
|
||||
@"tcp_connections": @(self.tcpConnections),
|
||||
@"tcp_frames": @(self.tcpFrames),
|
||||
@"tcp_bytes": @(self.tcpBytes),
|
||||
@"udp_frames": @(self.udpFrames),
|
||||
@"udp_packets": @(self.udpPackets),
|
||||
@"udp_bytes": @(self.udpBytes),
|
||||
@"invalid_udp_packets": @(self.invalidUDPPackets),
|
||||
@"discovery_requests": @(self.discoveryRequests),
|
||||
@"color_connections": @(self.colorConnections),
|
||||
@"color_frames": @(self.colorFrames),
|
||||
@"color_bytes": @(self.colorBytes),
|
||||
@"last_color_width": @(self.lastColorWidth),
|
||||
@"last_color_height": @(self.lastColorHeight),
|
||||
@"last_protocol": self.lastProtocol ?: @"none",
|
||||
@"seconds_since_last_frame": @(secondsSinceLastFrame)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
- (void)resetStreamStats {
|
||||
@synchronized (self) {
|
||||
self.startedAt = [NSDate date];
|
||||
self.lastFrameAt = nil;
|
||||
self.lastProtocol = @"none";
|
||||
self.tcpConnections = 0;
|
||||
self.tcpFrames = 0;
|
||||
self.tcpBytes = 0;
|
||||
self.udpFrames = 0;
|
||||
self.udpPackets = 0;
|
||||
self.udpBytes = 0;
|
||||
self.invalidUDPPackets = 0;
|
||||
self.discoveryRequests = 0;
|
||||
self.colorConnections = 0;
|
||||
self.colorFrames = 0;
|
||||
self.colorBytes = 0;
|
||||
self.lastColorWidth = 0;
|
||||
self.lastColorHeight = 0;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)setDebugEnabled:(BOOL)enabled {
|
||||
@synchronized (self) {
|
||||
_debugEnabled = enabled;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)noteTCPConnection {
|
||||
@synchronized (self) {
|
||||
self.tcpConnections += 1;
|
||||
if (self.debugEnabled) {
|
||||
NSLog(@"IPhoneMCP stream: TCP client connected");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
- (void)noteTCPBytes:(NSUInteger)count {
|
||||
@synchronized (self) {
|
||||
self.tcpBytes += count;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)noteUDPPacketBytes:(NSUInteger)count {
|
||||
@synchronized (self) {
|
||||
self.udpPackets += 1;
|
||||
self.udpBytes += count;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)noteInvalidUDPPacket {
|
||||
@synchronized (self) {
|
||||
self.invalidUDPPackets += 1;
|
||||
if (self.debugEnabled) {
|
||||
NSLog(@"IPhoneMCP stream: invalid UDP packet");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
- (void)noteDiscoveryRequest {
|
||||
@synchronized (self) {
|
||||
self.discoveryRequests += 1;
|
||||
if (self.debugEnabled) {
|
||||
NSLog(@"IPhoneMCP stream: discovery request");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
- (void)noteFrameForProtocol:(NSString *)protocol {
|
||||
@synchronized (self) {
|
||||
if ([protocol isEqualToString:@"tcp"]) {
|
||||
self.tcpFrames += 1;
|
||||
} else if ([protocol isEqualToString:@"udp"]) {
|
||||
self.udpFrames += 1;
|
||||
}
|
||||
self.lastProtocol = protocol ?: @"none";
|
||||
self.lastFrameAt = [NSDate date];
|
||||
if (self.debugEnabled) {
|
||||
NSLog(@"IPhoneMCP stream: %@ frame displayed", self.lastProtocol);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,5 @@
|
||||
#import <UIKit/UIKit.h>
|
||||
|
||||
@interface IMCViewController : UIViewController
|
||||
- (NSDictionary *)handleRPCRequest:(NSDictionary *)request;
|
||||
@end
|
||||
@@ -0,0 +1,846 @@
|
||||
#import "IMCViewController.h"
|
||||
#import "IMCCanvasView.h"
|
||||
#import "IMCMCPServer.h"
|
||||
#import "IMCVideoStreamServer.h"
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
#import <math.h>
|
||||
|
||||
@interface IMCPhotoCaptureDelegate : NSObject <AVCapturePhotoCaptureDelegate>
|
||||
@property (nonatomic, copy) void (^completion)(NSData *jpegData, NSError *error);
|
||||
@end
|
||||
|
||||
@implementation IMCPhotoCaptureDelegate
|
||||
|
||||
- (void)captureOutput:(AVCapturePhotoOutput *)output
|
||||
didFinishProcessingPhoto:(AVCapturePhoto *)photo
|
||||
error:(NSError *)error {
|
||||
NSData *data = error ? nil : [photo fileDataRepresentation];
|
||||
if (self.completion) {
|
||||
self.completion(data, error);
|
||||
}
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
@interface IMCViewController ()
|
||||
@property (nonatomic, strong) IMCCanvasView *canvasView;
|
||||
@property (nonatomic, strong) IMCMCPServer *server;
|
||||
@property (nonatomic, strong) IMCVideoStreamServer *videoStreamServer;
|
||||
@property (nonatomic, strong) AVAudioPlayer *audioPlayer;
|
||||
@property (nonatomic, strong) AVAudioRecorder *audioRecorder;
|
||||
@end
|
||||
|
||||
@implementation IMCViewController
|
||||
|
||||
- (void)viewDidLoad {
|
||||
[super viewDidLoad];
|
||||
|
||||
self.view.backgroundColor = [UIColor colorWithRed:0.03 green:0.06 blue:0.12 alpha:1.0];
|
||||
|
||||
self.canvasView = [[IMCCanvasView alloc] initWithFrame:CGRectZero];
|
||||
self.canvasView.translatesAutoresizingMaskIntoConstraints = NO;
|
||||
[self.view addSubview:self.canvasView];
|
||||
|
||||
[NSLayoutConstraint activateConstraints:@[
|
||||
[self.canvasView.topAnchor constraintEqualToAnchor:self.view.topAnchor],
|
||||
[self.canvasView.bottomAnchor constraintEqualToAnchor:self.view.bottomAnchor],
|
||||
[self.canvasView.leadingAnchor constraintEqualToAnchor:self.view.leadingAnchor],
|
||||
[self.canvasView.trailingAnchor constraintEqualToAnchor:self.view.trailingAnchor]
|
||||
]];
|
||||
|
||||
[self.canvasView clearWithColor:0];
|
||||
[self.canvasView drawText:@"iPhone MCP landscape server ready" x:18 y:18 size:2];
|
||||
[self.canvasView drawText:@"POST /api/mcp on port 8080" x:18 y:58 size:1];
|
||||
|
||||
self.server = [[IMCMCPServer alloc] initWithViewController:self port:8080];
|
||||
[self.server start];
|
||||
|
||||
self.videoStreamServer = [[IMCVideoStreamServer alloc] initWithCanvasView:self.canvasView];
|
||||
[self.videoStreamServer start];
|
||||
}
|
||||
|
||||
- (NSDictionary *)handleRPCRequest:(NSDictionary *)request {
|
||||
__block NSDictionary *response = nil;
|
||||
dispatch_sync(dispatch_get_main_queue(), ^{
|
||||
response = [self handleRPCRequestOnMainThread:request];
|
||||
});
|
||||
return response;
|
||||
}
|
||||
|
||||
- (NSDictionary *)handleRPCRequestOnMainThread:(NSDictionary *)request {
|
||||
id rpcID = request[@"id"] ?: [NSNull null];
|
||||
NSString *method = request[@"method"];
|
||||
|
||||
if ([method isEqualToString:@"initialize"]) {
|
||||
return [self result:@{
|
||||
@"protocolVersion": @"2024-11-05",
|
||||
@"capabilities": @{@"tools": @{}},
|
||||
@"serverInfo": @{@"name": @"iphone6-mcp", @"version": @"0.0.4"}
|
||||
} rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([method isEqualToString:@"notifications/initialized"]) {
|
||||
return [self result:@{} rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([method isEqualToString:@"tools/list"]) {
|
||||
return [self result:@{@"tools": [self tools]} rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([method isEqualToString:@"tools/call"]) {
|
||||
NSDictionary *params = request[@"params"] ?: @{};
|
||||
NSString *name = params[@"name"];
|
||||
NSDictionary *arguments = params[@"arguments"] ?: @{};
|
||||
return [self callTool:name arguments:arguments rpcID:rpcID];
|
||||
}
|
||||
|
||||
return [self errorWithCode:-32601 message:[NSString stringWithFormat:@"Method %@ not found", method] rpcID:rpcID];
|
||||
}
|
||||
|
||||
- (NSDictionary *)callTool:(NSString *)name arguments:(NSDictionary *)arguments rpcID:(id)rpcID {
|
||||
@try {
|
||||
if ([name isEqualToString:@"clear_screen"]) {
|
||||
NSInteger color = [arguments[@"color"] integerValue];
|
||||
[self.canvasView clearWithColor:color];
|
||||
return [self textResult:@"Screen cleared." rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([name isEqualToString:@"draw_text"]) {
|
||||
NSString *text = [NSString stringWithFormat:@"%@", arguments[@"text"] ?: @""];
|
||||
CGFloat x = [arguments[@"x"] doubleValue];
|
||||
CGFloat y = [arguments[@"y"] doubleValue];
|
||||
NSInteger size = arguments[@"size"] ? [arguments[@"size"] integerValue] : 1;
|
||||
[self.canvasView drawText:text x:x y:y size:size];
|
||||
return [self textResult:[NSString stringWithFormat:@"Drew text '%@' at (%.0f, %.0f).", text, x, y] rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([name isEqualToString:@"draw_image"]) {
|
||||
NSString *base64 = [NSString stringWithFormat:@"%@", arguments[@"image_base64"] ?: @""];
|
||||
NSRange marker = [base64 rangeOfString:@";base64,"];
|
||||
if (marker.location != NSNotFound) {
|
||||
base64 = [base64 substringFromIndex:marker.location + marker.length];
|
||||
}
|
||||
NSData *data = [[NSData alloc] initWithBase64EncodedString:base64 options:NSDataBase64DecodingIgnoreUnknownCharacters];
|
||||
if (!data) {
|
||||
return [self errorWithCode:-32602 message:@"Invalid image_base64." rpcID:rpcID];
|
||||
}
|
||||
[self.canvasView drawImageData:data
|
||||
x:[arguments[@"x"] doubleValue]
|
||||
y:[arguments[@"y"] doubleValue]];
|
||||
return [self textResult:@"Image displayed." rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([name isEqualToString:@"get_screenshot"]) {
|
||||
NSString *png = [self.canvasView snapshotPNGBase64];
|
||||
return [self result:@{@"content": @[@{@"type": @"image", @"data": png, @"mimeType": @"image/png"}]} rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([name isEqualToString:@"get_battery"]) {
|
||||
UIDevice *device = [UIDevice currentDevice];
|
||||
device.batteryMonitoringEnabled = YES;
|
||||
NSDictionary *battery = @{
|
||||
@"level_pct": @(MAX(device.batteryLevel, 0.0) * 100.0),
|
||||
@"state": [self batteryStateName:device.batteryState]
|
||||
};
|
||||
return [self textResult:[self jsonString:battery] rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([name isEqualToString:@"get_sensors"] || [name isEqualToString:@"scan_ble"] || [name isEqualToString:@"set_led"]) {
|
||||
return [self textResult:[NSString stringWithFormat:@"%@ is not implemented in this first iPhone build.", name] rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([name isEqualToString:@"play_tone"]) {
|
||||
NSInteger frequency = arguments[@"frequency"] ? [arguments[@"frequency"] integerValue] : 440;
|
||||
NSInteger duration = arguments[@"duration_ms"] ? [arguments[@"duration_ms"] integerValue] : 1000;
|
||||
NSInteger volume = arguments[@"volume"] ? [arguments[@"volume"] integerValue] : 50;
|
||||
|
||||
frequency = MAX(50, MIN(10000, frequency));
|
||||
duration = MAX(50, MIN(5000, duration));
|
||||
volume = MAX(0, MIN(100, volume));
|
||||
|
||||
NSData *toneData = [self wavDataForToneFrequency:frequency
|
||||
durationMS:duration
|
||||
volume:volume];
|
||||
NSDictionary *error = [self playAudioData:toneData volume:volume];
|
||||
if (error) {
|
||||
return [self errorWithCode:[error[@"code"] integerValue]
|
||||
message:error[@"message"]
|
||||
rpcID:rpcID];
|
||||
}
|
||||
return [self textResult:[NSString stringWithFormat:@"Played tone of %ldHz for %ldms at volume %ld.",
|
||||
(long)frequency,
|
||||
(long)duration,
|
||||
(long)volume]
|
||||
rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([name isEqualToString:@"play_audio_base64"]) {
|
||||
NSString *base64 = [self base64PayloadFromString:[NSString stringWithFormat:@"%@", arguments[@"wav_base64"] ?: @""]];
|
||||
NSData *audioData = [[NSData alloc] initWithBase64EncodedString:base64
|
||||
options:NSDataBase64DecodingIgnoreUnknownCharacters];
|
||||
if (!audioData) {
|
||||
return [self errorWithCode:-32602 message:@"Invalid wav_base64." rpcID:rpcID];
|
||||
}
|
||||
|
||||
NSInteger volumeArg = arguments[@"volume"] ? [arguments[@"volume"] integerValue] : 50;
|
||||
volumeArg = MAX(0, MIN(100, volumeArg));
|
||||
NSDictionary *error = [self playAudioData:audioData volume:volumeArg];
|
||||
if (error) {
|
||||
return [self errorWithCode:[error[@"code"] integerValue]
|
||||
message:error[@"message"]
|
||||
rpcID:rpcID];
|
||||
}
|
||||
return [self textResult:[NSString stringWithFormat:@"Playing base64 WAV audio (%lu bytes) at volume %ld.",
|
||||
(unsigned long)audioData.length,
|
||||
(long)volumeArg]
|
||||
rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([name isEqualToString:@"play_audio"]) {
|
||||
NSString *filename = [NSString stringWithFormat:@"%@", arguments[@"filename"] ?: @""];
|
||||
NSURL *url = [self safeFileURLForPath:filename];
|
||||
if (!url) {
|
||||
return [self errorWithCode:-32602
|
||||
message:@"filename must be relative and cannot contain '..'."
|
||||
rpcID:rpcID];
|
||||
}
|
||||
|
||||
NSData *audioData = [NSData dataWithContentsOfURL:url];
|
||||
if (!audioData) {
|
||||
return [self errorWithCode:-32000
|
||||
message:[NSString stringWithFormat:@"Could not read audio file '%@' from Documents/MCPFiles.", filename]
|
||||
rpcID:rpcID];
|
||||
}
|
||||
|
||||
NSInteger volumeArg = arguments[@"volume"] ? [arguments[@"volume"] integerValue] : 50;
|
||||
volumeArg = MAX(0, MIN(100, volumeArg));
|
||||
NSDictionary *error = [self playAudioData:audioData volume:volumeArg];
|
||||
if (error) {
|
||||
return [self errorWithCode:[error[@"code"] integerValue]
|
||||
message:error[@"message"]
|
||||
rpcID:rpcID];
|
||||
}
|
||||
return [self textResult:[NSString stringWithFormat:@"Playing WAV file '%@' (%lu bytes) at volume %ld.",
|
||||
filename,
|
||||
(unsigned long)audioData.length,
|
||||
(long)volumeArg]
|
||||
rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([name isEqualToString:@"hermes_voice_turn"]) {
|
||||
NSDictionary *voiceResult = [self performHermesVoiceTurn:arguments];
|
||||
if (voiceResult[@"error"]) {
|
||||
return [self errorWithCode:-32000 message:voiceResult[@"error"] rpcID:rpcID];
|
||||
}
|
||||
return [self textResult:[self jsonString:voiceResult] rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([name isEqualToString:@"record_voice"]) {
|
||||
if (![self ensureMicrophonePermission]) {
|
||||
return [self errorWithCode:-32000
|
||||
message:@"Microphone permission is not granted for iPhone MCP."
|
||||
rpcID:rpcID];
|
||||
}
|
||||
|
||||
NSInteger duration = arguments[@"duration_sec"] ? [arguments[@"duration_sec"] integerValue] : 4;
|
||||
duration = MAX(1, MIN(15, duration));
|
||||
NSString *filename = [NSString stringWithFormat:@"%@", arguments[@"filename"] ?: @"recording.wav"];
|
||||
NSURL *url = [self safeFileURLForPath:filename];
|
||||
if (!url) {
|
||||
return [self errorWithCode:-32602
|
||||
message:@"filename must be relative and cannot contain '..'."
|
||||
rpcID:rpcID];
|
||||
}
|
||||
|
||||
NSDictionary *result = [self recordVoiceToURL:url duration:duration];
|
||||
if (result[@"error"]) {
|
||||
return [self errorWithCode:-32000 message:result[@"error"] rpcID:rpcID];
|
||||
}
|
||||
return [self textResult:[self jsonString:result] rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([name isEqualToString:@"capture_selfie"]) {
|
||||
if (![self ensureCameraPermission]) {
|
||||
return [self errorWithCode:-32000
|
||||
message:@"Camera permission is not granted for iPhone MCP."
|
||||
rpcID:rpcID];
|
||||
}
|
||||
|
||||
NSDictionary *result = [self captureSelfiePNGBase64];
|
||||
if (result[@"error"]) {
|
||||
return [self errorWithCode:-32000 message:result[@"error"] rpcID:rpcID];
|
||||
}
|
||||
return [self result:@{@"content": @[@{@"type": @"image",
|
||||
@"data": result[@"png_base64"],
|
||||
@"mimeType": @"image/png"}]} rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([name isEqualToString:@"write_file"]) {
|
||||
NSString *path = [NSString stringWithFormat:@"%@", arguments[@"path"] ?: @""];
|
||||
NSString *content = [NSString stringWithFormat:@"%@", arguments[@"content"] ?: @""];
|
||||
NSURL *url = [self safeFileURLForPath:path];
|
||||
if (!url) {
|
||||
return [self errorWithCode:-32602 message:@"Path must be relative and cannot contain '..'." rpcID:rpcID];
|
||||
}
|
||||
[[NSFileManager defaultManager] createDirectoryAtURL:[url URLByDeletingLastPathComponent]
|
||||
withIntermediateDirectories:YES
|
||||
attributes:nil
|
||||
error:nil];
|
||||
NSError *writeError = nil;
|
||||
[content writeToURL:url atomically:YES encoding:NSUTF8StringEncoding error:&writeError];
|
||||
if (writeError) {
|
||||
return [self errorWithCode:-32000 message:writeError.localizedDescription rpcID:rpcID];
|
||||
}
|
||||
return [self textResult:[NSString stringWithFormat:@"Wrote %lu characters to %@.", (unsigned long)content.length, path] rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([name isEqualToString:@"read_file"]) {
|
||||
NSString *path = [NSString stringWithFormat:@"%@", arguments[@"path"] ?: @""];
|
||||
NSURL *url = [self safeFileURLForPath:path];
|
||||
if (!url) {
|
||||
return [self errorWithCode:-32602 message:@"Path must be relative and cannot contain '..'." rpcID:rpcID];
|
||||
}
|
||||
NSError *readError = nil;
|
||||
NSString *content = [NSString stringWithContentsOfURL:url encoding:NSUTF8StringEncoding error:&readError];
|
||||
if (readError) {
|
||||
return [self errorWithCode:-32000 message:readError.localizedDescription rpcID:rpcID];
|
||||
}
|
||||
return [self textResult:content rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([name isEqualToString:@"download_file"]) {
|
||||
NSString *urlString = [NSString stringWithFormat:@"%@", arguments[@"url"] ?: @""];
|
||||
NSString *filename = [NSString stringWithFormat:@"%@", arguments[@"filename"] ?: @""];
|
||||
BOOL useSD = arguments[@"use_sd"] ? [arguments[@"use_sd"] boolValue] : NO;
|
||||
NSURL *sourceURL = [NSURL URLWithString:urlString];
|
||||
NSURL *destinationURL = [self safeFileURLForPath:filename];
|
||||
if (!sourceURL || !destinationURL) {
|
||||
return [self errorWithCode:-32602 message:@"Invalid url or filename." rpcID:rpcID];
|
||||
}
|
||||
NSData *data = [NSData dataWithContentsOfURL:sourceURL];
|
||||
if (!data) {
|
||||
return [self errorWithCode:-32000 message:@"Download failed." rpcID:rpcID];
|
||||
}
|
||||
[[NSFileManager defaultManager] createDirectoryAtURL:[destinationURL URLByDeletingLastPathComponent]
|
||||
withIntermediateDirectories:YES
|
||||
attributes:nil
|
||||
error:nil];
|
||||
[data writeToURL:destinationURL atomically:YES];
|
||||
NSString *storageNote = useSD ? @" The iPhone build has no SD card, so use_sd was accepted and stored in app Documents instead." : @"";
|
||||
return [self textResult:[NSString stringWithFormat:@"Downloaded %lu bytes to %@.%@", (unsigned long)data.length, filename, storageNote] rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([name isEqualToString:@"get_video_streaming_instructions"]) {
|
||||
NSString *protocol = [[NSString stringWithFormat:@"%@", arguments[@"protocol"] ?: @"both"] lowercaseString];
|
||||
return [self textResult:[self videoStreamingInstructionsForProtocol:protocol] rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([name isEqualToString:@"get_stream_stats"]) {
|
||||
return [self textResult:[self jsonString:[self.videoStreamServer streamStats]] rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([name isEqualToString:@"reset_stream_stats"]) {
|
||||
[self.videoStreamServer resetStreamStats];
|
||||
return [self textResult:[self jsonString:[self.videoStreamServer streamStats]] rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([name isEqualToString:@"set_stream_debug"]) {
|
||||
BOOL enabled = arguments[@"enabled"] ? [arguments[@"enabled"] boolValue] : NO;
|
||||
[self.videoStreamServer setDebugEnabled:enabled];
|
||||
return [self textResult:[self jsonString:@{@"debug_enabled": @(enabled)}] rpcID:rpcID];
|
||||
}
|
||||
|
||||
if ([name isEqualToString:@"execute_python"]) {
|
||||
return [self textResult:@"execute_python is available on the MicroPython ESP32 target, but the native iPhone app does not execute arbitrary Python code. Use write_file/read_file/download_file for files and the dedicated drawing/audio/camera tools for app actions." rpcID:rpcID];
|
||||
}
|
||||
|
||||
return [self errorWithCode:-32602 message:[NSString stringWithFormat:@"Unknown tool: %@", name] rpcID:rpcID];
|
||||
} @catch (NSException *exception) {
|
||||
return [self errorWithCode:-32000 message:exception.reason ?: @"Tool failed." rpcID:rpcID];
|
||||
}
|
||||
}
|
||||
|
||||
- (NSArray *)tools {
|
||||
return @[
|
||||
[self tool:@"clear_screen" description:@"Clear the iPhone canvas to white (0) or black (1)." schema:@{@"type": @"object", @"properties": @{@"color": @{@"type": @"integer", @"enum": @[@0, @1]}}, @"required": @[@"color"]}],
|
||||
[self tool:@"draw_text" description:@"Draw text on the iPhone canvas using 400x300 logical coordinates." schema:@{@"type": @"object", @"properties": @{@"text": @{@"type": @"string"}, @"x": @{@"type": @"integer"}, @"y": @{@"type": @"integer"}, @"size": @{@"type": @"integer", @"enum": @[@1, @2]}}, @"required": @[@"text", @"x", @"y"]}],
|
||||
[self tool:@"draw_image" description:@"Draw a base64 PNG/JPEG/GIF image on the iPhone canvas." schema:@{@"type": @"object", @"properties": @{@"image_base64": @{@"type": @"string"}, @"x": @{@"type": @"integer", @"default": @0}, @"y": @{@"type": @"integer", @"default": @0}}, @"required": @[@"image_base64"]}],
|
||||
[self tool:@"get_screenshot" description:@"Return the current iPhone canvas as a PNG image." schema:@{@"type": @"object", @"properties": @{}}],
|
||||
[self tool:@"get_battery" description:@"Read the iPhone battery percentage and charge state." schema:@{@"type": @"object", @"properties": @{}}],
|
||||
[self tool:@"play_tone" description:@"Generate and play a sine-wave tone with the requested frequency, duration, and volume." schema:@{@"type": @"object", @"properties": @{@"frequency": @{@"type": @"integer", @"default": @440}, @"duration_ms": @{@"type": @"integer", @"default": @1000}, @"volume": @{@"type": @"integer", @"default": @50}}}],
|
||||
[self tool:@"play_audio_base64" description:@"Decode a base64 WAV payload and play it through the iPhone speaker." schema:@{@"type": @"object", @"properties": @{@"wav_base64": @{@"type": @"string"}, @"volume": @{@"type": @"integer", @"default": @50}}, @"required": @[@"wav_base64"]}],
|
||||
[self tool:@"play_audio" description:@"Play a WAV file from the app's Documents/MCPFiles directory." schema:@{@"type": @"object", @"properties": @{@"filename": @{@"type": @"string"}, @"volume": @{@"type": @"integer", @"default": @50}}, @"required": @[@"filename"]}],
|
||||
[self tool:@"hermes_voice_turn" description:@"Record a voice command, send it to the Hermes voice gateway, play the returned WAV, and return transcript/response diagnostics." schema:@{@"type": @"object", @"properties": @{@"url": @{@"type": @"string", @"default": @"http://192.168.68.126:8642/api/esp32/voice"}, @"api_token": @{@"type": @"string"}, @"device_id": @{@"type": @"string", @"default": @"iphone6"}, @"duration_sec": @{@"type": @"integer", @"default": @4}, @"volume": @{@"type": @"integer", @"default": @80}, @"instructions": @{@"type": @"string"}, @"reply_mode": @{@"type": @"string", @"enum": @[@"sync", @"ack"], @"default": @"sync"}, @"screen_url": @{@"type": @"string"}}}],
|
||||
[self tool:@"record_voice" description:@"Record microphone audio to a WAV file inside Documents/MCPFiles." schema:@{@"type": @"object", @"properties": @{@"duration_sec": @{@"type": @"integer", @"default": @4}, @"filename": @{@"type": @"string", @"default": @"recording.wav"}}}],
|
||||
[self tool:@"capture_selfie" description:@"Capture a still image from the front camera and return it as PNG image content." schema:@{@"type": @"object", @"properties": @{}}],
|
||||
[self tool:@"write_file" description:@"Write a UTF-8 file inside the app's Documents/MCPFiles directory." schema:@{@"type": @"object", @"properties": @{@"path": @{@"type": @"string"}, @"content": @{@"type": @"string"}}, @"required": @[@"path", @"content"]}],
|
||||
[self tool:@"read_file" description:@"Read a UTF-8 file from the app's Documents/MCPFiles directory." schema:@{@"type": @"object", @"properties": @{@"path": @{@"type": @"string"}}, @"required": @[@"path"]}],
|
||||
[self tool:@"download_file" description:@"Download a URL into the app's Documents/MCPFiles directory. Accepts ESP32-compatible use_sd, but stores locally on iPhone." schema:@{@"type": @"object", @"properties": @{@"url": @{@"type": @"string"}, @"filename": @{@"type": @"string"}, @"use_sd": @{@"type": @"boolean", @"default": @NO}}, @"required": @[@"url", @"filename"]}],
|
||||
[self tool:@"get_video_streaming_instructions" description:@"Get details for monochrome TCP/UDP streaming and RGB565 color TCP streaming." schema:@{@"type": @"object", @"properties": @{@"protocol": @{@"type": @"string", @"enum": @[@"tcp", @"udp", @"color", @"both", @"all"], @"default": @"all"}}}],
|
||||
[self tool:@"get_stream_stats" description:@"Return TCP/UDP streaming counters, ports, uptime, last protocol, and debug state." schema:@{@"type": @"object", @"properties": @{}}],
|
||||
[self tool:@"reset_stream_stats" description:@"Reset TCP/UDP streaming counters and last-frame state." schema:@{@"type": @"object", @"properties": @{}}],
|
||||
[self tool:@"set_stream_debug" description:@"Enable or disable stream debug logging on the iPhone." schema:@{@"type": @"object", @"properties": @{@"enabled": @{@"type": @"boolean", @"default": @NO}}}],
|
||||
[self tool:@"execute_python" description:@"ESP32 compatibility stub; arbitrary Python execution is intentionally not implemented in the native iPhone app." schema:@{@"type": @"object", @"properties": @{@"code": @{@"type": @"string"}}, @"required": @[@"code"]}],
|
||||
[self tool:@"set_led" description:@"ESP32 compatibility stub; the iPhone 6 has no NeoPixel LED." schema:@{@"type": @"object", @"properties": @{}}],
|
||||
[self tool:@"get_sensors" description:@"ESP32 compatibility stub; SHTC3 temperature/humidity is not present." schema:@{@"type": @"object", @"properties": @{}}],
|
||||
[self tool:@"scan_ble" description:@"ESP32 compatibility stub; BLE scanning is not implemented in this first build." schema:@{@"type": @"object", @"properties": @{}}]
|
||||
];
|
||||
}
|
||||
|
||||
- (NSDictionary *)tool:(NSString *)name description:(NSString *)description schema:(NSDictionary *)schema {
|
||||
return @{@"name": name, @"description": description, @"inputSchema": schema};
|
||||
}
|
||||
|
||||
- (NSDictionary *)textResult:(NSString *)text rpcID:(id)rpcID {
|
||||
return [self result:@{@"content": @[@{@"type": @"text", @"text": text ?: @""}]} rpcID:rpcID];
|
||||
}
|
||||
|
||||
- (NSDictionary *)result:(NSDictionary *)result rpcID:(id)rpcID {
|
||||
return @{@"jsonrpc": @"2.0", @"result": result ?: @{}, @"id": rpcID ?: [NSNull null]};
|
||||
}
|
||||
|
||||
- (NSDictionary *)errorWithCode:(NSInteger)code message:(NSString *)message rpcID:(id)rpcID {
|
||||
return @{@"jsonrpc": @"2.0",
|
||||
@"error": @{@"code": @(code), @"message": message ?: @"Error"},
|
||||
@"id": rpcID ?: [NSNull null]};
|
||||
}
|
||||
|
||||
- (NSString *)jsonString:(id)object {
|
||||
NSData *data = [NSJSONSerialization dataWithJSONObject:object options:0 error:nil];
|
||||
return [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] ?: @"{}";
|
||||
}
|
||||
|
||||
- (NSString *)base64PayloadFromString:(NSString *)string {
|
||||
NSRange marker = [string rangeOfString:@";base64,"];
|
||||
if (marker.location != NSNotFound) {
|
||||
return [string substringFromIndex:marker.location + marker.length];
|
||||
}
|
||||
return string ?: @"";
|
||||
}
|
||||
|
||||
- (NSDictionary *)playAudioData:(NSData *)audioData volume:(NSInteger)volume {
|
||||
NSError *sessionError = nil;
|
||||
[[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayback error:&sessionError];
|
||||
[[AVAudioSession sharedInstance] setActive:YES error:&sessionError];
|
||||
|
||||
NSError *playerError = nil;
|
||||
self.audioPlayer = [[AVAudioPlayer alloc] initWithData:audioData error:&playerError];
|
||||
if (!self.audioPlayer || playerError) {
|
||||
return @{@"code": @-32000,
|
||||
@"message": playerError.localizedDescription ?: @"Failed to create audio player."};
|
||||
}
|
||||
|
||||
self.audioPlayer.volume = MAX(0.0, MIN(1.0, volume / 100.0));
|
||||
[self.audioPlayer prepareToPlay];
|
||||
[self.audioPlayer play];
|
||||
return nil;
|
||||
}
|
||||
|
||||
- (NSString *)headerValue:(NSString *)name fromResponse:(NSHTTPURLResponse *)response {
|
||||
for (id key in response.allHeaderFields) {
|
||||
if ([[NSString stringWithFormat:@"%@", key] caseInsensitiveCompare:name] == NSOrderedSame) {
|
||||
return [NSString stringWithFormat:@"%@", response.allHeaderFields[key]];
|
||||
}
|
||||
}
|
||||
return @"";
|
||||
}
|
||||
|
||||
- (NSDictionary *)performHermesVoiceTurn:(NSDictionary *)arguments {
|
||||
if (![self ensureMicrophonePermission]) {
|
||||
return @{@"error": @"Microphone permission is not granted for iPhone MCP."};
|
||||
}
|
||||
|
||||
NSInteger duration = arguments[@"duration_sec"] ? [arguments[@"duration_sec"] integerValue] : 4;
|
||||
duration = MAX(1, MIN(20, duration));
|
||||
NSInteger volume = arguments[@"volume"] ? [arguments[@"volume"] integerValue] : 80;
|
||||
volume = MAX(0, MIN(100, volume));
|
||||
NSString *urlString = [NSString stringWithFormat:@"%@", arguments[@"url"] ?: @"http://192.168.68.126:8642/api/esp32/voice"];
|
||||
NSURL *gatewayURL = [NSURL URLWithString:urlString];
|
||||
if (!gatewayURL || ![@[@"http", @"https"] containsObject:gatewayURL.scheme.lowercaseString]) {
|
||||
return @{@"error": @"Hermes url must be an http or https URL."};
|
||||
}
|
||||
|
||||
NSURL *recordingURL = [self safeFileURLForPath:@"hermes/request.wav"];
|
||||
NSDictionary *recording = [self recordVoiceToURL:recordingURL duration:duration];
|
||||
if (recording[@"error"]) {
|
||||
return recording;
|
||||
}
|
||||
NSData *wavData = [NSData dataWithContentsOfURL:recordingURL];
|
||||
if (!wavData.length) {
|
||||
return @{@"error": @"Hermes recording produced no audio data."};
|
||||
}
|
||||
|
||||
NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:gatewayURL
|
||||
cachePolicy:NSURLRequestReloadIgnoringLocalCacheData
|
||||
timeoutInterval:90.0];
|
||||
request.HTTPMethod = @"POST";
|
||||
request.HTTPBody = wavData;
|
||||
[request setValue:@"audio/wav" forHTTPHeaderField:@"Content-Type"];
|
||||
[request setValue:[NSString stringWithFormat:@"%@", arguments[@"device_id"] ?: @"iphone6"]
|
||||
forHTTPHeaderField:@"X-Device-ID"];
|
||||
[request setValue:[NSString stringWithFormat:@"%@", arguments[@"reply_mode"] ?: @"sync"]
|
||||
forHTTPHeaderField:@"X-Hermes-Reply-Mode"];
|
||||
|
||||
NSString *instructions = [NSString stringWithFormat:@"%@", arguments[@"instructions"] ?: @""];
|
||||
if (instructions.length) {
|
||||
[request setValue:instructions forHTTPHeaderField:@"X-Hermes-Instructions"];
|
||||
}
|
||||
NSString *screenURL = [NSString stringWithFormat:@"%@", arguments[@"screen_url"] ?: @""];
|
||||
if (screenURL.length) {
|
||||
[request setValue:screenURL forHTTPHeaderField:@"X-Hermes-Screen-Url"];
|
||||
}
|
||||
NSString *token = [NSString stringWithFormat:@"%@", arguments[@"api_token"] ?: @""];
|
||||
if (token.length) {
|
||||
if (![token.lowercaseString hasPrefix:@"bearer "]) {
|
||||
token = [@"Bearer " stringByAppendingString:token];
|
||||
}
|
||||
[request setValue:token forHTTPHeaderField:@"Authorization"];
|
||||
}
|
||||
|
||||
NSURLResponse *rawResponse = nil;
|
||||
NSError *requestError = nil;
|
||||
NSData *responseData = [NSURLConnection sendSynchronousRequest:request
|
||||
returningResponse:&rawResponse
|
||||
error:&requestError];
|
||||
if (requestError || !responseData) {
|
||||
if ([requestError.domain isEqualToString:NSURLErrorDomain] &&
|
||||
(requestError.code == NSURLErrorUserCancelledAuthentication ||
|
||||
requestError.code == NSURLErrorUserAuthenticationRequired)) {
|
||||
return @{@"error": @"Hermes authorization failed. Provide a valid api_token to hermes_voice_turn."};
|
||||
}
|
||||
return @{@"error": requestError.localizedDescription ?: @"Hermes request failed."};
|
||||
}
|
||||
NSHTTPURLResponse *response = (NSHTTPURLResponse *)rawResponse;
|
||||
if (response.statusCode < 200 || response.statusCode >= 300) {
|
||||
NSString *body = [[NSString alloc] initWithData:responseData encoding:NSUTF8StringEncoding] ?: @"";
|
||||
return @{@"error": [NSString stringWithFormat:@"Hermes returned HTTP %ld: %@",
|
||||
(long)response.statusCode,
|
||||
[body substringToIndex:MIN(body.length, 500)]]};
|
||||
}
|
||||
|
||||
NSDictionary *playError = [self playAudioData:responseData volume:volume];
|
||||
if (playError) {
|
||||
return @{@"error": playError[@"message"] ?: @"Hermes audio playback failed."};
|
||||
}
|
||||
NSURL *responseURL = [self safeFileURLForPath:@"hermes/response.wav"];
|
||||
[responseData writeToURL:responseURL atomically:YES];
|
||||
|
||||
return @{
|
||||
@"status": @"playing",
|
||||
@"http_status": @(response.statusCode),
|
||||
@"request_bytes": @(wavData.length),
|
||||
@"response_bytes": @(responseData.length),
|
||||
@"transcript": [self headerValue:@"X-Hermes-Transcript" fromResponse:response],
|
||||
@"response_text": [self headerValue:@"X-Hermes-Text-Response" fromResponse:response],
|
||||
@"response_id": [self headerValue:@"X-Hermes-Response-Id" fromResponse:response],
|
||||
@"conversation": [self headerValue:@"X-Hermes-Conversation" fromResponse:response],
|
||||
@"reply_mode": [NSString stringWithFormat:@"%@", arguments[@"reply_mode"] ?: @"sync"],
|
||||
@"saved_response": @"hermes/response.wav"
|
||||
};
|
||||
}
|
||||
|
||||
- (NSDictionary *)recordVoiceToURL:(NSURL *)url duration:(NSInteger)duration {
|
||||
NSError *directoryError = nil;
|
||||
[[NSFileManager defaultManager] createDirectoryAtURL:[url URLByDeletingLastPathComponent]
|
||||
withIntermediateDirectories:YES
|
||||
attributes:nil
|
||||
error:&directoryError];
|
||||
if (directoryError) {
|
||||
return @{@"error": directoryError.localizedDescription};
|
||||
}
|
||||
|
||||
NSError *sessionError = nil;
|
||||
AVAudioSession *session = [AVAudioSession sharedInstance];
|
||||
[session setCategory:AVAudioSessionCategoryPlayAndRecord error:&sessionError];
|
||||
[session setActive:YES error:&sessionError];
|
||||
if (sessionError) {
|
||||
return @{@"error": sessionError.localizedDescription};
|
||||
}
|
||||
|
||||
NSDictionary *settings = @{
|
||||
AVFormatIDKey: @(kAudioFormatLinearPCM),
|
||||
AVSampleRateKey: @16000,
|
||||
AVNumberOfChannelsKey: @1,
|
||||
AVLinearPCMBitDepthKey: @16,
|
||||
AVLinearPCMIsFloatKey: @NO,
|
||||
AVLinearPCMIsBigEndianKey: @NO
|
||||
};
|
||||
|
||||
NSError *recordError = nil;
|
||||
self.audioRecorder = [[AVAudioRecorder alloc] initWithURL:url
|
||||
settings:settings
|
||||
error:&recordError];
|
||||
if (!self.audioRecorder || recordError) {
|
||||
return @{@"error": recordError.localizedDescription ?: @"Failed to create audio recorder."};
|
||||
}
|
||||
|
||||
[self.audioRecorder prepareToRecord];
|
||||
[self.audioRecorder recordForDuration:duration];
|
||||
NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:duration + 0.25];
|
||||
while (self.audioRecorder.recording && [deadline timeIntervalSinceNow] > 0) {
|
||||
[[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode
|
||||
beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.05]];
|
||||
}
|
||||
[self.audioRecorder stop];
|
||||
|
||||
NSDictionary *attributes = [[NSFileManager defaultManager] attributesOfItemAtPath:url.path error:nil];
|
||||
NSNumber *bytes = attributes[NSFileSize] ?: @0;
|
||||
return @{@"filename": url.lastPathComponent ?: @"recording.wav",
|
||||
@"bytes": bytes,
|
||||
@"duration_sec": @(duration)};
|
||||
}
|
||||
|
||||
- (NSDictionary *)captureSelfiePNGBase64 {
|
||||
AVCaptureDeviceDiscoverySession *discovery = [AVCaptureDeviceDiscoverySession discoverySessionWithDeviceTypes:@[
|
||||
AVCaptureDeviceTypeBuiltInWideAngleCamera
|
||||
]
|
||||
mediaType:AVMediaTypeVideo
|
||||
position:AVCaptureDevicePositionFront];
|
||||
AVCaptureDevice *camera = discovery.devices.firstObject;
|
||||
if (!camera) {
|
||||
return @{@"error": @"Front camera not found."};
|
||||
}
|
||||
|
||||
NSError *inputError = nil;
|
||||
AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:camera error:&inputError];
|
||||
if (!input || inputError) {
|
||||
return @{@"error": inputError.localizedDescription ?: @"Failed to open front camera."};
|
||||
}
|
||||
|
||||
AVCaptureSession *session = [[AVCaptureSession alloc] init];
|
||||
session.sessionPreset = AVCaptureSessionPresetPhoto;
|
||||
if (![session canAddInput:input]) {
|
||||
return @{@"error": @"Could not add front camera input."};
|
||||
}
|
||||
[session addInput:input];
|
||||
|
||||
AVCaptureStillImageOutput *output = [[AVCaptureStillImageOutput alloc] init];
|
||||
output.outputSettings = @{AVVideoCodecKey: AVVideoCodecJPEG};
|
||||
if (![session canAddOutput:output]) {
|
||||
return @{@"error": @"Could not add still image output."};
|
||||
}
|
||||
[session addOutput:output];
|
||||
[session startRunning];
|
||||
|
||||
AVCaptureConnection *connection = [output connectionWithMediaType:AVMediaTypeVideo];
|
||||
if (connection.isVideoOrientationSupported) {
|
||||
connection.videoOrientation = AVCaptureVideoOrientationLandscapeRight;
|
||||
}
|
||||
|
||||
__block NSData *jpegData = nil;
|
||||
__block NSError *captureError = nil;
|
||||
__block BOOL finished = NO;
|
||||
|
||||
[output captureStillImageAsynchronouslyFromConnection:connection
|
||||
completionHandler:^(CMSampleBufferRef imageDataSampleBuffer, NSError *error) {
|
||||
if (imageDataSampleBuffer) {
|
||||
jpegData = [AVCaptureStillImageOutput jpegStillImageNSDataRepresentation:imageDataSampleBuffer];
|
||||
}
|
||||
captureError = error;
|
||||
finished = YES;
|
||||
}];
|
||||
|
||||
NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:5.0];
|
||||
while (!finished && [deadline timeIntervalSinceNow] > 0) {
|
||||
[[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode
|
||||
beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.05]];
|
||||
}
|
||||
[session stopRunning];
|
||||
|
||||
if (captureError || !jpegData) {
|
||||
return @{@"error": captureError.localizedDescription ?: @"Selfie capture timed out."};
|
||||
}
|
||||
|
||||
UIImage *image = [UIImage imageWithData:jpegData];
|
||||
NSData *pngData = UIImagePNGRepresentation(image);
|
||||
if (!pngData) {
|
||||
return @{@"error": @"Failed to convert selfie to PNG."};
|
||||
}
|
||||
return @{@"png_base64": [pngData base64EncodedStringWithOptions:0]};
|
||||
}
|
||||
|
||||
- (BOOL)ensureMicrophonePermission {
|
||||
AVAudioSession *session = [AVAudioSession sharedInstance];
|
||||
if (session.recordPermission == AVAudioSessionRecordPermissionGranted) {
|
||||
return YES;
|
||||
}
|
||||
if (session.recordPermission == AVAudioSessionRecordPermissionDenied) {
|
||||
return NO;
|
||||
}
|
||||
|
||||
__block BOOL granted = NO;
|
||||
__block BOOL finished = NO;
|
||||
[session requestRecordPermission:^(BOOL didGrant) {
|
||||
granted = didGrant;
|
||||
finished = YES;
|
||||
}];
|
||||
return [self waitForPermissionResult:&finished granted:&granted];
|
||||
}
|
||||
|
||||
- (BOOL)ensureCameraPermission {
|
||||
AVAuthorizationStatus status = [AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeVideo];
|
||||
if (status == AVAuthorizationStatusAuthorized) {
|
||||
return YES;
|
||||
}
|
||||
if (status == AVAuthorizationStatusDenied || status == AVAuthorizationStatusRestricted) {
|
||||
return NO;
|
||||
}
|
||||
|
||||
__block BOOL granted = NO;
|
||||
__block BOOL finished = NO;
|
||||
[AVCaptureDevice requestAccessForMediaType:AVMediaTypeVideo
|
||||
completionHandler:^(BOOL didGrant) {
|
||||
granted = didGrant;
|
||||
finished = YES;
|
||||
}];
|
||||
return [self waitForPermissionResult:&finished granted:&granted];
|
||||
}
|
||||
|
||||
- (BOOL)waitForPermissionResult:(BOOL *)finished granted:(BOOL *)granted {
|
||||
NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:30.0];
|
||||
while (!*finished && [deadline timeIntervalSinceNow] > 0) {
|
||||
[[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode
|
||||
beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.05]];
|
||||
}
|
||||
return *finished && *granted;
|
||||
}
|
||||
|
||||
- (NSData *)wavDataForToneFrequency:(NSInteger)frequency durationMS:(NSInteger)durationMS volume:(NSInteger)volume {
|
||||
const uint32_t sampleRate = 22050;
|
||||
const uint16_t channels = 1;
|
||||
const uint16_t bitsPerSample = 16;
|
||||
const uint16_t blockAlign = channels * bitsPerSample / 8;
|
||||
const uint32_t byteRate = sampleRate * blockAlign;
|
||||
uint32_t sampleCount = (uint32_t)((sampleRate * durationMS) / 1000);
|
||||
uint32_t dataSize = sampleCount * blockAlign;
|
||||
uint32_t riffSize = 36 + dataSize;
|
||||
|
||||
NSMutableData *data = [NSMutableData dataWithCapacity:44 + dataSize];
|
||||
[self appendASCII:"RIFF" toData:data];
|
||||
[self appendUInt32LE:riffSize toData:data];
|
||||
[self appendASCII:"WAVE" toData:data];
|
||||
[self appendASCII:"fmt " toData:data];
|
||||
[self appendUInt32LE:16 toData:data];
|
||||
[self appendUInt16LE:1 toData:data];
|
||||
[self appendUInt16LE:channels toData:data];
|
||||
[self appendUInt32LE:sampleRate toData:data];
|
||||
[self appendUInt32LE:byteRate toData:data];
|
||||
[self appendUInt16LE:blockAlign toData:data];
|
||||
[self appendUInt16LE:bitsPerSample toData:data];
|
||||
[self appendASCII:"data" toData:data];
|
||||
[self appendUInt32LE:dataSize toData:data];
|
||||
|
||||
double amplitude = 32767.0 * MAX(0.0, MIN(1.0, volume / 100.0));
|
||||
double phaseStep = 2.0 * M_PI * frequency / sampleRate;
|
||||
for (uint32_t i = 0; i < sampleCount; i++) {
|
||||
double fadeIn = MIN(1.0, i / (sampleRate * 0.01));
|
||||
double fadeOut = MIN(1.0, (sampleCount - i) / (sampleRate * 0.01));
|
||||
double envelope = MIN(fadeIn, fadeOut);
|
||||
int16_t sample = (int16_t)(sin(i * phaseStep) * amplitude * envelope);
|
||||
[self appendUInt16LE:(uint16_t)sample toData:data];
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
- (void)appendASCII:(const char *)string toData:(NSMutableData *)data {
|
||||
[data appendBytes:string length:4];
|
||||
}
|
||||
|
||||
- (void)appendUInt16LE:(uint16_t)value toData:(NSMutableData *)data {
|
||||
uint8_t bytes[] = {value & 0xff, (value >> 8) & 0xff};
|
||||
[data appendBytes:bytes length:sizeof(bytes)];
|
||||
}
|
||||
|
||||
- (void)appendUInt32LE:(uint32_t)value toData:(NSMutableData *)data {
|
||||
uint8_t bytes[] = {
|
||||
value & 0xff,
|
||||
(value >> 8) & 0xff,
|
||||
(value >> 16) & 0xff,
|
||||
(value >> 24) & 0xff
|
||||
};
|
||||
[data appendBytes:bytes length:sizeof(bytes)];
|
||||
}
|
||||
|
||||
- (NSString *)videoStreamingInstructionsForProtocol:(NSString *)protocol {
|
||||
NSMutableArray *lines = [NSMutableArray arrayWithArray:@[
|
||||
@"### iPhone MCP Video Streaming Instructions",
|
||||
@"Dimensions: 400x300, 1-bit monochrome, using the same 15,000-byte RLCD frame layout as the ESP32 server.",
|
||||
@"The iPhone stretches each received frame over the full landscape screen.",
|
||||
@"UDP discovery: send DISCOVER_SCREEN to port 5000; the iPhone replies SCREEN_IP_8080 because its JSON-RPC endpoint is on port 8080.",
|
||||
@"Mapping logic (Python):",
|
||||
@" def map_to_rlcd(pil_img):",
|
||||
@" img_1bit = pil_img.convert('1', dither=1)",
|
||||
@" px = img_1bit.load()",
|
||||
@" buf = bytearray(15000)",
|
||||
@" for y in range(300):",
|
||||
@" for x in range(400):",
|
||||
@" if px[x, y]:",
|
||||
@" inv_y = 299 - y",
|
||||
@" bx = x // 2",
|
||||
@" by = inv_y // 4",
|
||||
@" idx = bx * 75 + by",
|
||||
@" lx, ly = x % 2, inv_y % 4",
|
||||
@" bit = 7 - (ly * 2 + lx)",
|
||||
@" buf[idx] |= (1 << bit)",
|
||||
@" return buf"
|
||||
]];
|
||||
|
||||
if ([protocol isEqualToString:@"tcp"] || [protocol isEqualToString:@"both"] || [protocol isEqualToString:@"all"]) {
|
||||
[lines addObject:@"\n**TCP Streaming (Port 8081):**"];
|
||||
[lines addObject:@"Open a TCP connection to the iPhone IP on port 8081 and send consecutive 15,000-byte frame blocks."];
|
||||
}
|
||||
|
||||
if ([protocol isEqualToString:@"udp"] || [protocol isEqualToString:@"both"] || [protocol isEqualToString:@"all"]) {
|
||||
[lines addObject:@"\n**UDP Streaming / Broadcast (Port 8082):**"];
|
||||
[lines addObject:@"Split each 15,000-byte frame into 15 chunks of 1,000 bytes. Send each chunk as a 1002-byte packet: byte 0 = frame_id (0-255), byte 1 = chunk_idx (0-14), bytes 2..1001 = chunk payload."];
|
||||
}
|
||||
|
||||
if ([protocol isEqualToString:@"color"] || [protocol isEqualToString:@"all"]) {
|
||||
[lines addObject:@"\n**RGB565 Color TCP Streaming (Port 8083):**"];
|
||||
[lines addObject:@"Send each frame as a 16-byte header followed by width*height*2 bytes of big-endian RGB565 pixels. Header: bytes 0..3='IMCR', byte 4=1 (version), byte 5=1 (RGB565BE), bytes 6..7=width big-endian, bytes 8..9=height big-endian, bytes 10..13=payload length big-endian, bytes 14..15=flags (zero). Frames may use any dimensions up to 2048x2048 and are scaled to the full iPhone canvas."];
|
||||
}
|
||||
|
||||
return [lines componentsJoinedByString:@"\n"];
|
||||
}
|
||||
|
||||
- (NSString *)batteryStateName:(UIDeviceBatteryState)state {
|
||||
switch (state) {
|
||||
case UIDeviceBatteryStateUnplugged:
|
||||
return @"unplugged";
|
||||
case UIDeviceBatteryStateCharging:
|
||||
return @"charging";
|
||||
case UIDeviceBatteryStateFull:
|
||||
return @"full";
|
||||
default:
|
||||
return @"unknown";
|
||||
}
|
||||
}
|
||||
|
||||
- (NSURL *)safeFileURLForPath:(NSString *)path {
|
||||
if (path.length == 0 || [path hasPrefix:@"/"]) {
|
||||
return nil;
|
||||
}
|
||||
if ([[path pathComponents] containsObject:@".."]) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
NSArray *directories = NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES);
|
||||
NSString *documents = directories.firstObject;
|
||||
NSString *root = [documents stringByAppendingPathComponent:@"MCPFiles"];
|
||||
return [NSURL fileURLWithPath:[root stringByAppendingPathComponent:path]];
|
||||
}
|
||||
|
||||
- (BOOL)prefersStatusBarHidden {
|
||||
return YES;
|
||||
}
|
||||
|
||||
- (UIInterfaceOrientationMask)supportedInterfaceOrientations {
|
||||
return UIInterfaceOrientationMaskLandscape;
|
||||
}
|
||||
|
||||
- (UIInterfaceOrientation)preferredInterfaceOrientationForPresentation {
|
||||
return UIInterfaceOrientationLandscapeRight;
|
||||
}
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,13 @@
|
||||
TARGET := iphone:clang:12.4:12.0
|
||||
ARCHS = arm64
|
||||
INSTALL_TARGET_PROCESSES = IPhoneMCP
|
||||
|
||||
include $(THEOS)/makefiles/common.mk
|
||||
|
||||
APPLICATION_NAME = IPhoneMCP
|
||||
|
||||
IPhoneMCP_FILES = main.m IMCAppDelegate.m IMCViewController.m IMCCanvasView.m IMCMCPServer.m IMCVideoStreamServer.m
|
||||
IPhoneMCP_FRAMEWORKS = UIKit Foundation AVFoundation CoreGraphics
|
||||
IPhoneMCP_CFLAGS = -fobjc-arc -Wno-deprecated-declarations
|
||||
|
||||
include $(THEOS_MAKE_PATH)/application.mk
|
||||
@@ -0,0 +1,174 @@
|
||||
# iPhone MCP
|
||||
|
||||
Native UIKit MCP screen server for a jailbroken arm64 iPhone running iOS 12. It shares the ESP32 screen's MCP tool shape while using the iPhone display, speaker, microphone, battery, and front camera.
|
||||
|
||||
## Features
|
||||
|
||||
- HTTP JSON-RPC MCP endpoint: `POST http://<iphone-ip>:8080/api/mcp`
|
||||
- Full-screen landscape canvas with idle sleep disabled while the app is active
|
||||
- ESP32-compatible 400x300 monochrome streaming: TCP `8081`, UDP `8082`
|
||||
- Framed RGB565 color streaming: TCP `8083`
|
||||
- UDP discovery on `5000`: `DISCOVER_SCREEN` -> `SCREEN_IP_8080`
|
||||
- True pitched tones, WAV playback, microphone recording, and selfie capture
|
||||
- Hermes voice gateway turn: record -> transcribe/agent/TTS -> play response
|
||||
- Stream statistics and MCP screenshots
|
||||
|
||||
## Requirements
|
||||
|
||||
- Jailbroken arm64 iPhone (tested on iPhone 6 / iOS 12.5.8)
|
||||
- OpenSSH and `ldid` installed on the phone
|
||||
- Theos at `/var/mobile/theos` with `iPhoneOS12.4.sdk`
|
||||
- The phone and build computer on the same LAN
|
||||
|
||||
## Copy To The Phone
|
||||
|
||||
From the repository root:
|
||||
|
||||
```sh
|
||||
rsync -av iphone_app/ root@<iphone-ip>:/var/mobile/IPhoneMCP/
|
||||
```
|
||||
|
||||
Keep the phone unlocked while launching the app. On iOS 12, `uiopen` may refuse to launch a locked device.
|
||||
|
||||
## Build
|
||||
|
||||
The included direct build script is the most reliable path on the iPhone 6:
|
||||
|
||||
```sh
|
||||
ssh root@<iphone-ip>
|
||||
su - mobile -c 'chmod +x /var/mobile/IPhoneMCP/manual_build_iphone.sh && /var/mobile/IPhoneMCP/manual_build_iphone.sh /var/mobile/IPhoneMCP'
|
||||
```
|
||||
|
||||
This creates and signs:
|
||||
|
||||
```text
|
||||
/var/mobile/IPhoneMCP/.manual_build/IPhoneMCP.app
|
||||
```
|
||||
|
||||
The regular Theos build is also supported:
|
||||
|
||||
```sh
|
||||
su - mobile -c 'env THEOS=/var/mobile/theos PATH=/usr/bin:/bin:/usr/sbin:/sbin make -C /var/mobile/IPhoneMCP clean package SDKVERSION=12.4 INCLUDE_SDKVERSION=12.4 FAKEROOT="bash /var/mobile/theos/bin/fakeroot.sh -p /var/mobile/IPhoneMCP/.theos/fakeroot" _THEOS_PLATFORM_DPKG_DEB=dpkg-deb THEOS_PLATFORM_DEB_COMPRESSION_TYPE=gzip'
|
||||
```
|
||||
|
||||
## Package And Install
|
||||
|
||||
For a direct-build bundle:
|
||||
|
||||
```sh
|
||||
PKGROOT=/var/mobile/IPhoneMCP/.manual_pkg
|
||||
DEB=/var/mobile/IPhoneMCP/packages/com.reynafamily.iphonemcp_manual_iphoneos-arm.deb
|
||||
rm -rf "$PKGROOT"
|
||||
mkdir -p "$PKGROOT/DEBIAN" "$PKGROOT/Applications/IPhoneMCP.app" /var/mobile/IPhoneMCP/packages
|
||||
cp /var/mobile/IPhoneMCP/control "$PKGROOT/DEBIAN/control"
|
||||
cp /var/mobile/IPhoneMCP/.manual_build/IPhoneMCP.app/IPhoneMCP "$PKGROOT/Applications/IPhoneMCP.app/IPhoneMCP"
|
||||
cp /var/mobile/IPhoneMCP/.manual_build/IPhoneMCP.app/Info.plist "$PKGROOT/Applications/IPhoneMCP.app/Info.plist"
|
||||
chmod 755 "$PKGROOT/Applications/IPhoneMCP.app/IPhoneMCP"
|
||||
chown -R root:wheel "$PKGROOT"
|
||||
dpkg-deb -b "$PKGROOT" "$DEB"
|
||||
dpkg -i "$DEB"
|
||||
uicache -p /Applications/IPhoneMCP.app
|
||||
uiopen com.reynafamily.iphonemcp
|
||||
```
|
||||
|
||||
Verify listeners:
|
||||
|
||||
```sh
|
||||
netstat -an | grep -E '\.8080|\.8081|\.8082|\.8083|\.5000'
|
||||
```
|
||||
|
||||
## MCP Bridge
|
||||
|
||||
```sh
|
||||
python3 iphone_app/iphone_mcp_bridge.py --ip <iphone-ip>
|
||||
```
|
||||
|
||||
Example client configuration:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"iphone6-mcp": {
|
||||
"command": "python3",
|
||||
"args": ["/absolute/path/to/mcp_screen/iphone_app/iphone_mcp_bridge.py", "--ip", "192.168.68.150"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Monochrome Streaming
|
||||
|
||||
- TCP `8081`: consecutive 15,000-byte ESP32 RLCD frame buffers
|
||||
- UDP `8082`: fifteen 1002-byte packets per frame
|
||||
- byte 0: frame id
|
||||
- byte 1: chunk index 0-14
|
||||
- bytes 2-1001: 1000-byte payload
|
||||
|
||||
The mapping is identical to `test_stream.py` in the repository.
|
||||
|
||||
## RGB565 Color Streaming
|
||||
|
||||
Color frames use TCP port `8083`. Every frame has a 16-byte big-endian header followed by RGB565 data:
|
||||
|
||||
| Offset | Size | Meaning |
|
||||
|---|---:|---|
|
||||
| 0 | 4 | ASCII `IMCR` |
|
||||
| 4 | 1 | Version `1` |
|
||||
| 5 | 1 | Format `1` = RGB565 big-endian |
|
||||
| 6 | 2 | Width |
|
||||
| 8 | 2 | Height |
|
||||
| 10 | 4 | Payload length (`width * height * 2`) |
|
||||
| 14 | 2 | Flags, currently `0` |
|
||||
|
||||
Run the included example:
|
||||
|
||||
```sh
|
||||
python3 -m pip install Pillow
|
||||
python3 iphone_app/tools/stream_color.py --ip <iphone-ip>
|
||||
```
|
||||
|
||||
## Hermes Voice
|
||||
|
||||
The `hermes_voice_turn` MCP tool records a mono 16 kHz, 16-bit WAV, posts it to Hermes, plays the returned WAV, and returns the transcript and answer headers.
|
||||
|
||||
Default gateway:
|
||||
|
||||
```text
|
||||
http://192.168.68.126:8642/api/esp32/voice
|
||||
```
|
||||
|
||||
Example JSON-RPC call:
|
||||
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "tools/call",
|
||||
"params": {
|
||||
"name": "hermes_voice_turn",
|
||||
"arguments": {
|
||||
"url": "http://192.168.68.126:8642/api/esp32/voice",
|
||||
"api_token": "YOUR_HERMES_API_TOKEN",
|
||||
"device_id": "iphone6",
|
||||
"duration_sec": 4,
|
||||
"volume": 80,
|
||||
"reply_mode": "sync",
|
||||
"screen_url": "http://192.168.68.150:8080/api/mcp"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Do not commit the Hermes API token. It is accepted per call and is not persisted by the app.
|
||||
|
||||
## Implemented Tools
|
||||
|
||||
`clear_screen`, `draw_text`, `draw_image`, `get_screenshot`, `get_battery`, `play_tone`, `play_audio`, `play_audio_base64`, `record_voice`, `hermes_voice_turn`, `capture_selfie`, `write_file`, `read_file`, `download_file`, `get_video_streaming_instructions`, `get_stream_stats`, `reset_stream_stats`, and `set_stream_debug`.
|
||||
|
||||
Compatibility stubs remain for `set_led`, `get_sensors`, `scan_ble`, and `execute_python`.
|
||||
|
||||
## Operational Notes
|
||||
|
||||
- Keep the app foregrounded for reliable network service on iOS 12.
|
||||
- The idle timer is disabled only while the app is active.
|
||||
- Give each device a unique static DHCP reservation; duplicate IPs can make traffic reach the wrong host.
|
||||
@@ -0,0 +1,57 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
|
||||
"http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>CFBundleDisplayName</key>
|
||||
<string>iPhone MCP</string>
|
||||
<key>CFBundleExecutable</key>
|
||||
<string>IPhoneMCP</string>
|
||||
<key>CFBundleIdentifier</key>
|
||||
<string>com.reynafamily.iphonemcp</string>
|
||||
<key>CFBundleInfoDictionaryVersion</key>
|
||||
<string>6.0</string>
|
||||
<key>CFBundleName</key>
|
||||
<string>IPhoneMCP</string>
|
||||
<key>CFBundlePackageType</key>
|
||||
<string>APPL</string>
|
||||
<key>CFBundleSupportedPlatforms</key>
|
||||
<array>
|
||||
<string>iPhoneOS</string>
|
||||
</array>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>0.0.4</string>
|
||||
<key>CFBundleVersion</key>
|
||||
<string>4</string>
|
||||
<key>MinimumOSVersion</key>
|
||||
<string>12.0</string>
|
||||
<key>LSRequiresIPhoneOS</key>
|
||||
<true/>
|
||||
<key>NSAppTransportSecurity</key>
|
||||
<dict>
|
||||
<key>NSAllowsArbitraryLoads</key>
|
||||
<true/>
|
||||
</dict>
|
||||
<key>NSCameraUsageDescription</key>
|
||||
<string>iPhone MCP uses the camera when an MCP client calls the selfie capture tool.</string>
|
||||
<key>NSMicrophoneUsageDescription</key>
|
||||
<string>iPhone MCP uses the microphone when an MCP client calls the voice recording tool.</string>
|
||||
<key>UIDeviceFamily</key>
|
||||
<array>
|
||||
<integer>1</integer>
|
||||
</array>
|
||||
<key>UIRequiredDeviceCapabilities</key>
|
||||
<array>
|
||||
<string>arm64</string>
|
||||
</array>
|
||||
<key>UIStatusBarHidden</key>
|
||||
<true/>
|
||||
<key>UIViewControllerBasedStatusBarAppearance</key>
|
||||
<true/>
|
||||
<key>UISupportedInterfaceOrientations</key>
|
||||
<array>
|
||||
<string>UIInterfaceOrientationLandscapeLeft</string>
|
||||
<string>UIInterfaceOrientationLandscapeRight</string>
|
||||
</array>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -0,0 +1,8 @@
|
||||
Package: com.reynafamily.iphonemcp
|
||||
Name: iPhone MCP
|
||||
Version: 0.0.4
|
||||
Architecture: iphoneos-arm
|
||||
Description: A native UIKit MCP-style HTTP server for a jailbroken iPhone 6.
|
||||
Maintainer: Adolfo Reyna
|
||||
Author: Adolfo Reyna
|
||||
Section: Utilities
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="MCP stdio bridge for the jailbroken iPhone MCP app")
|
||||
parser.add_argument("--ip", required=True, help="iPhone IP address")
|
||||
parser.add_argument("--port", type=int, default=8080, help="iPhone MCP HTTP port")
|
||||
args = parser.parse_args()
|
||||
|
||||
url = f"http://{args.ip}:{args.port}/api/mcp"
|
||||
sys.stderr.write(f"iPhone MCP bridge routing stdio to {url}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
for line in sys.stdin:
|
||||
try:
|
||||
request = json.loads(line)
|
||||
http_request = urllib.request.Request(
|
||||
url,
|
||||
data=json.dumps(request).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(http_request, timeout=15.0) as response:
|
||||
sys.stdout.write(response.read().decode("utf-8") + "\n")
|
||||
sys.stdout.flush()
|
||||
except urllib.error.URLError as exc:
|
||||
write_error(-32000, f"Bridge failed to reach iPhone MCP app: {exc}", request if "request" in locals() else {})
|
||||
except Exception as exc:
|
||||
write_error(-32603, f"Bridge error: {exc}", request if "request" in locals() else {})
|
||||
|
||||
|
||||
def write_error(code, message, request):
|
||||
sys.stdout.write(json.dumps({
|
||||
"jsonrpc": "2.0",
|
||||
"error": {"code": code, "message": message},
|
||||
"id": request.get("id"),
|
||||
}) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,9 @@
|
||||
#import <UIKit/UIKit.h>
|
||||
#import "IMCAppDelegate.h"
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
@autoreleasepool {
|
||||
return UIApplicationMain(argc, argv, nil,
|
||||
NSStringFromClass(IMCAppDelegate.class));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
PROJECT_DIR="${1:-/var/mobile/IPhoneMCP}"
|
||||
SDK="${SDK:-/var/mobile/theos/sdks/iPhoneOS12.4.sdk}"
|
||||
BUILD_DIR="$PROJECT_DIR/.manual_build"
|
||||
OBJ_DIR="$BUILD_DIR/obj"
|
||||
APP_DIR="$BUILD_DIR/IPhoneMCP.app"
|
||||
|
||||
mkdir -p "$OBJ_DIR" "$APP_DIR"
|
||||
cp "$PROJECT_DIR/Resources/Info.plist" "$APP_DIR/Info.plist"
|
||||
|
||||
CFLAGS="-arch arm64 -isysroot $SDK -miphoneos-version-min=12.0 -fobjc-arc -Wno-deprecated-declarations -I$PROJECT_DIR"
|
||||
SOURCES="main.m IMCAppDelegate.m IMCViewController.m IMCCanvasView.m IMCMCPServer.m IMCVideoStreamServer.m"
|
||||
OBJECTS=""
|
||||
|
||||
for src in $SOURCES; do
|
||||
obj="$OBJ_DIR/${src%.m}.o"
|
||||
echo "Compiling $src"
|
||||
clang $CFLAGS -c "$PROJECT_DIR/$src" -o "$obj"
|
||||
OBJECTS="$OBJECTS $obj"
|
||||
done
|
||||
|
||||
echo "Linking IPhoneMCP"
|
||||
clang -arch arm64 \
|
||||
-isysroot "$SDK" \
|
||||
-miphoneos-version-min=12.0 \
|
||||
-o "$APP_DIR/IPhoneMCP" \
|
||||
$OBJECTS \
|
||||
-framework UIKit \
|
||||
-framework Foundation \
|
||||
-framework AVFoundation \
|
||||
-framework CoreGraphics
|
||||
|
||||
echo "Signing IPhoneMCP"
|
||||
ldid -S "$APP_DIR/IPhoneMCP"
|
||||
|
||||
echo "$APP_DIR"
|
||||
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Minimal Hermes-compatible voice endpoint for iPhone integration testing."""
|
||||
|
||||
import argparse
|
||||
import io
|
||||
import math
|
||||
import struct
|
||||
import wave
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
|
||||
def response_wav() -> bytes:
|
||||
output = io.BytesIO()
|
||||
with wave.open(output, "wb") as wav:
|
||||
wav.setnchannels(1)
|
||||
wav.setsampwidth(2)
|
||||
wav.setframerate(16000)
|
||||
frames = bytearray()
|
||||
for index in range(int(16000 * 0.3)):
|
||||
sample = int(7000 * math.sin(2 * math.pi * 660 * index / 16000))
|
||||
frames.extend(struct.pack("<h", sample))
|
||||
wav.writeframes(frames)
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
if self.path != "/api/esp32/voice":
|
||||
self.send_error(404)
|
||||
return
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
received = self.rfile.read(length)
|
||||
print(f"received {len(received)} bytes from {self.headers.get('X-Device-ID', 'unknown')}", flush=True)
|
||||
payload = response_wav()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "audio/wav")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.send_header("X-Hermes-Transcript", "Mock voice request received")
|
||||
self.send_header("X-Hermes-Text-Response", "The iPhone Hermes integration is working")
|
||||
self.send_header("X-Hermes-Response-Id", "mock-response")
|
||||
self.send_header("X-Hermes-Conversation", "iphone-test")
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
def log_message(self, message, *args):
|
||||
print(message % args, flush=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--host", default="0.0.0.0")
|
||||
parser.add_argument("--port", type=int, default=8765)
|
||||
args = parser.parse_args()
|
||||
server = ThreadingHTTPServer((args.host, args.port), Handler)
|
||||
print(f"mock Hermes gateway listening on {args.host}:{args.port}", flush=True)
|
||||
server.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Stream a generated RGB565 color animation to iPhone MCP TCP port 8083."""
|
||||
|
||||
import argparse
|
||||
import socket
|
||||
import struct
|
||||
import time
|
||||
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
|
||||
def rgb565be(image: Image.Image) -> bytes:
|
||||
image = image.convert("RGB")
|
||||
output = bytearray(image.width * image.height * 2)
|
||||
for index, (red, green, blue) in enumerate(image.getdata()):
|
||||
value = ((red >> 3) << 11) | ((green >> 2) << 5) | (blue >> 3)
|
||||
output[index * 2] = value >> 8
|
||||
output[index * 2 + 1] = value & 0xFF
|
||||
return bytes(output)
|
||||
|
||||
|
||||
def send_frame(sock: socket.socket, image: Image.Image) -> None:
|
||||
payload = rgb565be(image)
|
||||
header = struct.pack(">4sBBHHIH", b"IMCR", 1, 1, image.width, image.height, len(payload), 0)
|
||||
sock.sendall(header + payload)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--ip", required=True, help="iPhone Wi-Fi IP")
|
||||
parser.add_argument("--port", type=int, default=8083)
|
||||
parser.add_argument("--width", type=int, default=400)
|
||||
parser.add_argument("--height", type=int, default=300)
|
||||
args = parser.parse_args()
|
||||
|
||||
with socket.create_connection((args.ip, args.port), timeout=8) as sock:
|
||||
frame = 0
|
||||
while True:
|
||||
hue = (frame * 4) % 256
|
||||
image = Image.new("RGB", (args.width, args.height), (12, 18, 35))
|
||||
draw = ImageDraw.Draw(image)
|
||||
for x in range(args.width):
|
||||
red = (x * 255 // max(1, args.width - 1) + hue) % 256
|
||||
blue = 255 - (x * 255 // max(1, args.width - 1))
|
||||
draw.line((x, 0, x, args.height - 1), fill=(red, 70, blue))
|
||||
draw.rounded_rectangle((20, 20, args.width - 20, args.height - 20), radius=22,
|
||||
outline=(255, 255, 255), width=4)
|
||||
draw.text((40, 45), "iPhone MCP RGB565", fill=(255, 255, 255))
|
||||
draw.text((40, 72), f"Frame {frame}", fill=(255, 240, 80))
|
||||
send_frame(sock, image)
|
||||
frame += 1
|
||||
time.sleep(1 / 20)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user