adolforeyna/mcp_screen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
main
mcp_screen/iphone_app/IMCViewController.m
T
Adolfo Reyna ad30662a48 Add jailbroken iPhone MCP screen app
2026-06-21 21:27:06 -04:00

847 lines
43 KiB
Objective-C
Raw Permalink Blame History

#import "IMCViewController.h"
#import "IMCCanvasView.h"
#import "IMCMCPServer.h"
#import "IMCVideoStreamServer.h"
#import <AVFoundation/AVFoundation.h>
#import <math.h>
@interface IMCPhotoCaptureDelegate : NSObject <AVCapturePhotoCaptureDelegate>
@property (nonatomic, copy) void (^completion)(NSData *jpegData, NSError *error);
@end
@implementation IMCPhotoCaptureDelegate
- (void)captureOutput:(AVCapturePhotoOutput *)output
didFinishProcessingPhoto:(AVCapturePhoto *)photo
error:(NSError *)error {
NSData *data = error ? nil : [photo fileDataRepresentation];
if (self.completion) {
self.completion(data, error);
}
}
@end
@interface IMCViewController ()
@property (nonatomic, strong) IMCCanvasView *canvasView;
@property (nonatomic, strong) IMCMCPServer *server;
@property (nonatomic, strong) IMCVideoStreamServer *videoStreamServer;
@property (nonatomic, strong) AVAudioPlayer *audioPlayer;
@property (nonatomic, strong) AVAudioRecorder *audioRecorder;
@end
@implementation IMCViewController
- (void)viewDidLoad {
[super viewDidLoad];
self.view.backgroundColor = [UIColor colorWithRed:0.03 green:0.06 blue:0.12 alpha:1.0];
self.canvasView = [[IMCCanvasView alloc] initWithFrame:CGRectZero];
self.canvasView.translatesAutoresizingMaskIntoConstraints = NO;
[self.view addSubview:self.canvasView];
[NSLayoutConstraint activateConstraints:@[
[self.canvasView.topAnchor constraintEqualToAnchor:self.view.topAnchor],
[self.canvasView.bottomAnchor constraintEqualToAnchor:self.view.bottomAnchor],
[self.canvasView.leadingAnchor constraintEqualToAnchor:self.view.leadingAnchor],
[self.canvasView.trailingAnchor constraintEqualToAnchor:self.view.trailingAnchor]
]];
[self.canvasView clearWithColor:0];
[self.canvasView drawText:@"iPhone MCP landscape server ready" x:18 y:18 size:2];
[self.canvasView drawText:@"POST /api/mcp on port 8080" x:18 y:58 size:1];
self.server = [[IMCMCPServer alloc] initWithViewController:self port:8080];
[self.server start];
self.videoStreamServer = [[IMCVideoStreamServer alloc] initWithCanvasView:self.canvasView];
[self.videoStreamServer start];
}
- (NSDictionary *)handleRPCRequest:(NSDictionary *)request {
__block NSDictionary *response = nil;
dispatch_sync(dispatch_get_main_queue(), ^{
response = [self handleRPCRequestOnMainThread:request];
});
return response;
}
- (NSDictionary *)handleRPCRequestOnMainThread:(NSDictionary *)request {
id rpcID = request[@"id"] ?: [NSNull null];
NSString *method = request[@"method"];
if ([method isEqualToString:@"initialize"]) {
return [self result:@{
@"protocolVersion": @"2024-11-05",
@"capabilities": @{@"tools": @{}},
@"serverInfo": @{@"name": @"iphone6-mcp", @"version": @"0.0.4"}
} rpcID:rpcID];
}
if ([method isEqualToString:@"notifications/initialized"]) {
return [self result:@{} rpcID:rpcID];
}
if ([method isEqualToString:@"tools/list"]) {
return [self result:@{@"tools": [self tools]} rpcID:rpcID];
}
if ([method isEqualToString:@"tools/call"]) {
NSDictionary *params = request[@"params"] ?: @{};
NSString *name = params[@"name"];
NSDictionary *arguments = params[@"arguments"] ?: @{};
return [self callTool:name arguments:arguments rpcID:rpcID];
}
return [self errorWithCode:-32601 message:[NSString stringWithFormat:@"Method %@ not found", method] rpcID:rpcID];
}
- (NSDictionary *)callTool:(NSString *)name arguments:(NSDictionary *)arguments rpcID:(id)rpcID {
@try {
if ([name isEqualToString:@"clear_screen"]) {
NSInteger color = [arguments[@"color"] integerValue];
[self.canvasView clearWithColor:color];
return [self textResult:@"Screen cleared." rpcID:rpcID];
}
if ([name isEqualToString:@"draw_text"]) {
NSString *text = [NSString stringWithFormat:@"%@", arguments[@"text"] ?: @""];
CGFloat x = [arguments[@"x"] doubleValue];
CGFloat y = [arguments[@"y"] doubleValue];
NSInteger size = arguments[@"size"] ? [arguments[@"size"] integerValue] : 1;
[self.canvasView drawText:text x:x y:y size:size];
return [self textResult:[NSString stringWithFormat:@"Drew text '%@' at (%.0f, %.0f).", text, x, y] rpcID:rpcID];
}
if ([name isEqualToString:@"draw_image"]) {
NSString *base64 = [NSString stringWithFormat:@"%@", arguments[@"image_base64"] ?: @""];
NSRange marker = [base64 rangeOfString:@";base64,"];
if (marker.location != NSNotFound) {
base64 = [base64 substringFromIndex:marker.location + marker.length];
}
NSData *data = [[NSData alloc] initWithBase64EncodedString:base64 options:NSDataBase64DecodingIgnoreUnknownCharacters];
if (!data) {
return [self errorWithCode:-32602 message:@"Invalid image_base64." rpcID:rpcID];
}
[self.canvasView drawImageData:data
x:[arguments[@"x"] doubleValue]
y:[arguments[@"y"] doubleValue]];
return [self textResult:@"Image displayed." rpcID:rpcID];
}
if ([name isEqualToString:@"get_screenshot"]) {
NSString *png = [self.canvasView snapshotPNGBase64];
return [self result:@{@"content": @[@{@"type": @"image", @"data": png, @"mimeType": @"image/png"}]} rpcID:rpcID];
}
if ([name isEqualToString:@"get_battery"]) {
UIDevice *device = [UIDevice currentDevice];
device.batteryMonitoringEnabled = YES;
NSDictionary *battery = @{
@"level_pct": @(MAX(device.batteryLevel, 0.0) * 100.0),
@"state": [self batteryStateName:device.batteryState]
};
return [self textResult:[self jsonString:battery] rpcID:rpcID];
}
if ([name isEqualToString:@"get_sensors"] || [name isEqualToString:@"scan_ble"] || [name isEqualToString:@"set_led"]) {
return [self textResult:[NSString stringWithFormat:@"%@ is not implemented in this first iPhone build.", name] rpcID:rpcID];
}
if ([name isEqualToString:@"play_tone"]) {
NSInteger frequency = arguments[@"frequency"] ? [arguments[@"frequency"] integerValue] : 440;
NSInteger duration = arguments[@"duration_ms"] ? [arguments[@"duration_ms"] integerValue] : 1000;
NSInteger volume = arguments[@"volume"] ? [arguments[@"volume"] integerValue] : 50;
frequency = MAX(50, MIN(10000, frequency));
duration = MAX(50, MIN(5000, duration));
volume = MAX(0, MIN(100, volume));
NSData *toneData = [self wavDataForToneFrequency:frequency
durationMS:duration
volume:volume];
NSDictionary *error = [self playAudioData:toneData volume:volume];
if (error) {
return [self errorWithCode:[error[@"code"] integerValue]
message:error[@"message"]
rpcID:rpcID];
}
return [self textResult:[NSString stringWithFormat:@"Played tone of %ldHz for %ldms at volume %ld.",
(long)frequency,
(long)duration,
(long)volume]
rpcID:rpcID];
}
if ([name isEqualToString:@"play_audio_base64"]) {
NSString *base64 = [self base64PayloadFromString:[NSString stringWithFormat:@"%@", arguments[@"wav_base64"] ?: @""]];
NSData *audioData = [[NSData alloc] initWithBase64EncodedString:base64
options:NSDataBase64DecodingIgnoreUnknownCharacters];
if (!audioData) {
return [self errorWithCode:-32602 message:@"Invalid wav_base64." rpcID:rpcID];
}
NSInteger volumeArg = arguments[@"volume"] ? [arguments[@"volume"] integerValue] : 50;
volumeArg = MAX(0, MIN(100, volumeArg));
NSDictionary *error = [self playAudioData:audioData volume:volumeArg];
if (error) {
return [self errorWithCode:[error[@"code"] integerValue]
message:error[@"message"]
rpcID:rpcID];
}
return [self textResult:[NSString stringWithFormat:@"Playing base64 WAV audio (%lu bytes) at volume %ld.",
(unsigned long)audioData.length,
(long)volumeArg]
rpcID:rpcID];
}
if ([name isEqualToString:@"play_audio"]) {
NSString *filename = [NSString stringWithFormat:@"%@", arguments[@"filename"] ?: @""];
NSURL *url = [self safeFileURLForPath:filename];
if (!url) {
return [self errorWithCode:-32602
message:@"filename must be relative and cannot contain '..'."
rpcID:rpcID];
}
NSData *audioData = [NSData dataWithContentsOfURL:url];
if (!audioData) {
return [self errorWithCode:-32000
message:[NSString stringWithFormat:@"Could not read audio file '%@' from Documents/MCPFiles.", filename]
rpcID:rpcID];
}
NSInteger volumeArg = arguments[@"volume"] ? [arguments[@"volume"] integerValue] : 50;
volumeArg = MAX(0, MIN(100, volumeArg));
NSDictionary *error = [self playAudioData:audioData volume:volumeArg];
if (error) {
return [self errorWithCode:[error[@"code"] integerValue]
message:error[@"message"]
rpcID:rpcID];
}
return [self textResult:[NSString stringWithFormat:@"Playing WAV file '%@' (%lu bytes) at volume %ld.",
filename,
(unsigned long)audioData.length,
(long)volumeArg]
rpcID:rpcID];
}
if ([name isEqualToString:@"hermes_voice_turn"]) {
NSDictionary *voiceResult = [self performHermesVoiceTurn:arguments];
if (voiceResult[@"error"]) {
return [self errorWithCode:-32000 message:voiceResult[@"error"] rpcID:rpcID];
}
return [self textResult:[self jsonString:voiceResult] rpcID:rpcID];
}
if ([name isEqualToString:@"record_voice"]) {
if (![self ensureMicrophonePermission]) {
return [self errorWithCode:-32000
message:@"Microphone permission is not granted for iPhone MCP."
rpcID:rpcID];
}
NSInteger duration = arguments[@"duration_sec"] ? [arguments[@"duration_sec"] integerValue] : 4;
duration = MAX(1, MIN(15, duration));
NSString *filename = [NSString stringWithFormat:@"%@", arguments[@"filename"] ?: @"recording.wav"];
NSURL *url = [self safeFileURLForPath:filename];
if (!url) {
return [self errorWithCode:-32602
message:@"filename must be relative and cannot contain '..'."
rpcID:rpcID];
}
NSDictionary *result = [self recordVoiceToURL:url duration:duration];
if (result[@"error"]) {
return [self errorWithCode:-32000 message:result[@"error"] rpcID:rpcID];
}
return [self textResult:[self jsonString:result] rpcID:rpcID];
}
if ([name isEqualToString:@"capture_selfie"]) {
if (![self ensureCameraPermission]) {
return [self errorWithCode:-32000
message:@"Camera permission is not granted for iPhone MCP."
rpcID:rpcID];
}
NSDictionary *result = [self captureSelfiePNGBase64];
if (result[@"error"]) {
return [self errorWithCode:-32000 message:result[@"error"] rpcID:rpcID];
}
return [self result:@{@"content": @[@{@"type": @"image",
@"data": result[@"png_base64"],
@"mimeType": @"image/png"}]} rpcID:rpcID];
}
if ([name isEqualToString:@"write_file"]) {
NSString *path = [NSString stringWithFormat:@"%@", arguments[@"path"] ?: @""];
NSString *content = [NSString stringWithFormat:@"%@", arguments[@"content"] ?: @""];
NSURL *url = [self safeFileURLForPath:path];
if (!url) {
return [self errorWithCode:-32602 message:@"Path must be relative and cannot contain '..'." rpcID:rpcID];
}
[[NSFileManager defaultManager] createDirectoryAtURL:[url URLByDeletingLastPathComponent]
withIntermediateDirectories:YES
attributes:nil
error:nil];
NSError *writeError = nil;
[content writeToURL:url atomically:YES encoding:NSUTF8StringEncoding error:&writeError];
if (writeError) {
return [self errorWithCode:-32000 message:writeError.localizedDescription rpcID:rpcID];
}
return [self textResult:[NSString stringWithFormat:@"Wrote %lu characters to %@.", (unsigned long)content.length, path] rpcID:rpcID];
}
if ([name isEqualToString:@"read_file"]) {
NSString *path = [NSString stringWithFormat:@"%@", arguments[@"path"] ?: @""];
NSURL *url = [self safeFileURLForPath:path];
if (!url) {
return [self errorWithCode:-32602 message:@"Path must be relative and cannot contain '..'." rpcID:rpcID];
}
NSError *readError = nil;
NSString *content = [NSString stringWithContentsOfURL:url encoding:NSUTF8StringEncoding error:&readError];
if (readError) {
return [self errorWithCode:-32000 message:readError.localizedDescription rpcID:rpcID];
}
return [self textResult:content rpcID:rpcID];
}
if ([name isEqualToString:@"download_file"]) {
NSString *urlString = [NSString stringWithFormat:@"%@", arguments[@"url"] ?: @""];
NSString *filename = [NSString stringWithFormat:@"%@", arguments[@"filename"] ?: @""];
BOOL useSD = arguments[@"use_sd"] ? [arguments[@"use_sd"] boolValue] : NO;
NSURL *sourceURL = [NSURL URLWithString:urlString];
NSURL *destinationURL = [self safeFileURLForPath:filename];
if (!sourceURL || !destinationURL) {
return [self errorWithCode:-32602 message:@"Invalid url or filename." rpcID:rpcID];
}
NSData *data = [NSData dataWithContentsOfURL:sourceURL];
if (!data) {
return [self errorWithCode:-32000 message:@"Download failed." rpcID:rpcID];
}
[[NSFileManager defaultManager] createDirectoryAtURL:[destinationURL URLByDeletingLastPathComponent]
withIntermediateDirectories:YES
attributes:nil
error:nil];
[data writeToURL:destinationURL atomically:YES];
NSString *storageNote = useSD ? @" The iPhone build has no SD card, so use_sd was accepted and stored in app Documents instead." : @"";
return [self textResult:[NSString stringWithFormat:@"Downloaded %lu bytes to %@.%@", (unsigned long)data.length, filename, storageNote] rpcID:rpcID];
}
if ([name isEqualToString:@"get_video_streaming_instructions"]) {
NSString *protocol = [[NSString stringWithFormat:@"%@", arguments[@"protocol"] ?: @"both"] lowercaseString];
return [self textResult:[self videoStreamingInstructionsForProtocol:protocol] rpcID:rpcID];
}
if ([name isEqualToString:@"get_stream_stats"]) {
return [self textResult:[self jsonString:[self.videoStreamServer streamStats]] rpcID:rpcID];
}
if ([name isEqualToString:@"reset_stream_stats"]) {
[self.videoStreamServer resetStreamStats];
return [self textResult:[self jsonString:[self.videoStreamServer streamStats]] rpcID:rpcID];
}
if ([name isEqualToString:@"set_stream_debug"]) {
BOOL enabled = arguments[@"enabled"] ? [arguments[@"enabled"] boolValue] : NO;
[self.videoStreamServer setDebugEnabled:enabled];
return [self textResult:[self jsonString:@{@"debug_enabled": @(enabled)}] rpcID:rpcID];
}
if ([name isEqualToString:@"execute_python"]) {
return [self textResult:@"execute_python is available on the MicroPython ESP32 target, but the native iPhone app does not execute arbitrary Python code. Use write_file/read_file/download_file for files and the dedicated drawing/audio/camera tools for app actions." rpcID:rpcID];
}
return [self errorWithCode:-32602 message:[NSString stringWithFormat:@"Unknown tool: %@", name] rpcID:rpcID];
} @catch (NSException *exception) {
return [self errorWithCode:-32000 message:exception.reason ?: @"Tool failed." rpcID:rpcID];
}
}
- (NSArray *)tools {
return @[
[self tool:@"clear_screen" description:@"Clear the iPhone canvas to white (0) or black (1)." schema:@{@"type": @"object", @"properties": @{@"color": @{@"type": @"integer", @"enum": @[@0, @1]}}, @"required": @[@"color"]}],
[self tool:@"draw_text" description:@"Draw text on the iPhone canvas using 400x300 logical coordinates." schema:@{@"type": @"object", @"properties": @{@"text": @{@"type": @"string"}, @"x": @{@"type": @"integer"}, @"y": @{@"type": @"integer"}, @"size": @{@"type": @"integer", @"enum": @[@1, @2]}}, @"required": @[@"text", @"x", @"y"]}],
[self tool:@"draw_image" description:@"Draw a base64 PNG/JPEG/GIF image on the iPhone canvas." schema:@{@"type": @"object", @"properties": @{@"image_base64": @{@"type": @"string"}, @"x": @{@"type": @"integer", @"default": @0}, @"y": @{@"type": @"integer", @"default": @0}}, @"required": @[@"image_base64"]}],
[self tool:@"get_screenshot" description:@"Return the current iPhone canvas as a PNG image." schema:@{@"type": @"object", @"properties": @{}}],
[self tool:@"get_battery" description:@"Read the iPhone battery percentage and charge state." schema:@{@"type": @"object", @"properties": @{}}],
[self tool:@"play_tone" description:@"Generate and play a sine-wave tone with the requested frequency, duration, and volume." schema:@{@"type": @"object", @"properties": @{@"frequency": @{@"type": @"integer", @"default": @440}, @"duration_ms": @{@"type": @"integer", @"default": @1000}, @"volume": @{@"type": @"integer", @"default": @50}}}],
[self tool:@"play_audio_base64" description:@"Decode a base64 WAV payload and play it through the iPhone speaker." schema:@{@"type": @"object", @"properties": @{@"wav_base64": @{@"type": @"string"}, @"volume": @{@"type": @"integer", @"default": @50}}, @"required": @[@"wav_base64"]}],
[self tool:@"play_audio" description:@"Play a WAV file from the app's Documents/MCPFiles directory." schema:@{@"type": @"object", @"properties": @{@"filename": @{@"type": @"string"}, @"volume": @{@"type": @"integer", @"default": @50}}, @"required": @[@"filename"]}],
[self tool:@"hermes_voice_turn" description:@"Record a voice command, send it to the Hermes voice gateway, play the returned WAV, and return transcript/response diagnostics." schema:@{@"type": @"object", @"properties": @{@"url": @{@"type": @"string", @"default": @"http://192.168.68.126:8642/api/esp32/voice"}, @"api_token": @{@"type": @"string"}, @"device_id": @{@"type": @"string", @"default": @"iphone6"}, @"duration_sec": @{@"type": @"integer", @"default": @4}, @"volume": @{@"type": @"integer", @"default": @80}, @"instructions": @{@"type": @"string"}, @"reply_mode": @{@"type": @"string", @"enum": @[@"sync", @"ack"], @"default": @"sync"}, @"screen_url": @{@"type": @"string"}}}],
[self tool:@"record_voice" description:@"Record microphone audio to a WAV file inside Documents/MCPFiles." schema:@{@"type": @"object", @"properties": @{@"duration_sec": @{@"type": @"integer", @"default": @4}, @"filename": @{@"type": @"string", @"default": @"recording.wav"}}}],
[self tool:@"capture_selfie" description:@"Capture a still image from the front camera and return it as PNG image content." schema:@{@"type": @"object", @"properties": @{}}],
[self tool:@"write_file" description:@"Write a UTF-8 file inside the app's Documents/MCPFiles directory." schema:@{@"type": @"object", @"properties": @{@"path": @{@"type": @"string"}, @"content": @{@"type": @"string"}}, @"required": @[@"path", @"content"]}],
[self tool:@"read_file" description:@"Read a UTF-8 file from the app's Documents/MCPFiles directory." schema:@{@"type": @"object", @"properties": @{@"path": @{@"type": @"string"}}, @"required": @[@"path"]}],
[self tool:@"download_file" description:@"Download a URL into the app's Documents/MCPFiles directory. Accepts ESP32-compatible use_sd, but stores locally on iPhone." schema:@{@"type": @"object", @"properties": @{@"url": @{@"type": @"string"}, @"filename": @{@"type": @"string"}, @"use_sd": @{@"type": @"boolean", @"default": @NO}}, @"required": @[@"url", @"filename"]}],
[self tool:@"get_video_streaming_instructions" description:@"Get details for monochrome TCP/UDP streaming and RGB565 color TCP streaming." schema:@{@"type": @"object", @"properties": @{@"protocol": @{@"type": @"string", @"enum": @[@"tcp", @"udp", @"color", @"both", @"all"], @"default": @"all"}}}],
[self tool:@"get_stream_stats" description:@"Return TCP/UDP streaming counters, ports, uptime, last protocol, and debug state." schema:@{@"type": @"object", @"properties": @{}}],
[self tool:@"reset_stream_stats" description:@"Reset TCP/UDP streaming counters and last-frame state." schema:@{@"type": @"object", @"properties": @{}}],
[self tool:@"set_stream_debug" description:@"Enable or disable stream debug logging on the iPhone." schema:@{@"type": @"object", @"properties": @{@"enabled": @{@"type": @"boolean", @"default": @NO}}}],
[self tool:@"execute_python" description:@"ESP32 compatibility stub; arbitrary Python execution is intentionally not implemented in the native iPhone app." schema:@{@"type": @"object", @"properties": @{@"code": @{@"type": @"string"}}, @"required": @[@"code"]}],
[self tool:@"set_led" description:@"ESP32 compatibility stub; the iPhone 6 has no NeoPixel LED." schema:@{@"type": @"object", @"properties": @{}}],
[self tool:@"get_sensors" description:@"ESP32 compatibility stub; SHTC3 temperature/humidity is not present." schema:@{@"type": @"object", @"properties": @{}}],
[self tool:@"scan_ble" description:@"ESP32 compatibility stub; BLE scanning is not implemented in this first build." schema:@{@"type": @"object", @"properties": @{}}]
];
}
- (NSDictionary *)tool:(NSString *)name description:(NSString *)description schema:(NSDictionary *)schema {
return @{@"name": name, @"description": description, @"inputSchema": schema};
}
- (NSDictionary *)textResult:(NSString *)text rpcID:(id)rpcID {
return [self result:@{@"content": @[@{@"type": @"text", @"text": text ?: @""}]} rpcID:rpcID];
}
- (NSDictionary *)result:(NSDictionary *)result rpcID:(id)rpcID {
return @{@"jsonrpc": @"2.0", @"result": result ?: @{}, @"id": rpcID ?: [NSNull null]};
}
- (NSDictionary *)errorWithCode:(NSInteger)code message:(NSString *)message rpcID:(id)rpcID {
return @{@"jsonrpc": @"2.0",
@"error": @{@"code": @(code), @"message": message ?: @"Error"},
@"id": rpcID ?: [NSNull null]};
}
- (NSString *)jsonString:(id)object {
NSData *data = [NSJSONSerialization dataWithJSONObject:object options:0 error:nil];
return [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] ?: @"{}";
}
- (NSString *)base64PayloadFromString:(NSString *)string {
NSRange marker = [string rangeOfString:@";base64,"];
if (marker.location != NSNotFound) {
return [string substringFromIndex:marker.location + marker.length];
}
return string ?: @"";
}
- (NSDictionary *)playAudioData:(NSData *)audioData volume:(NSInteger)volume {
NSError *sessionError = nil;
[[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayback error:&sessionError];
[[AVAudioSession sharedInstance] setActive:YES error:&sessionError];
NSError *playerError = nil;
self.audioPlayer = [[AVAudioPlayer alloc] initWithData:audioData error:&playerError];
if (!self.audioPlayer || playerError) {
return @{@"code": @-32000,
@"message": playerError.localizedDescription ?: @"Failed to create audio player."};
}
self.audioPlayer.volume = MAX(0.0, MIN(1.0, volume / 100.0));
[self.audioPlayer prepareToPlay];
[self.audioPlayer play];
return nil;
}
- (NSString *)headerValue:(NSString *)name fromResponse:(NSHTTPURLResponse *)response {
for (id key in response.allHeaderFields) {
if ([[NSString stringWithFormat:@"%@", key] caseInsensitiveCompare:name] == NSOrderedSame) {
return [NSString stringWithFormat:@"%@", response.allHeaderFields[key]];
}
}
return @"";
}
- (NSDictionary *)performHermesVoiceTurn:(NSDictionary *)arguments {
if (![self ensureMicrophonePermission]) {
return @{@"error": @"Microphone permission is not granted for iPhone MCP."};
}
NSInteger duration = arguments[@"duration_sec"] ? [arguments[@"duration_sec"] integerValue] : 4;
duration = MAX(1, MIN(20, duration));
NSInteger volume = arguments[@"volume"] ? [arguments[@"volume"] integerValue] : 80;
volume = MAX(0, MIN(100, volume));
NSString *urlString = [NSString stringWithFormat:@"%@", arguments[@"url"] ?: @"http://192.168.68.126:8642/api/esp32/voice"];
NSURL *gatewayURL = [NSURL URLWithString:urlString];
if (!gatewayURL || ![@[@"http", @"https"] containsObject:gatewayURL.scheme.lowercaseString]) {
return @{@"error": @"Hermes url must be an http or https URL."};
}
NSURL *recordingURL = [self safeFileURLForPath:@"hermes/request.wav"];
NSDictionary *recording = [self recordVoiceToURL:recordingURL duration:duration];
if (recording[@"error"]) {
return recording;
}
NSData *wavData = [NSData dataWithContentsOfURL:recordingURL];
if (!wavData.length) {
return @{@"error": @"Hermes recording produced no audio data."};
}
NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:gatewayURL
cachePolicy:NSURLRequestReloadIgnoringLocalCacheData
timeoutInterval:90.0];
request.HTTPMethod = @"POST";
request.HTTPBody = wavData;
[request setValue:@"audio/wav" forHTTPHeaderField:@"Content-Type"];
[request setValue:[NSString stringWithFormat:@"%@", arguments[@"device_id"] ?: @"iphone6"]
forHTTPHeaderField:@"X-Device-ID"];
[request setValue:[NSString stringWithFormat:@"%@", arguments[@"reply_mode"] ?: @"sync"]
forHTTPHeaderField:@"X-Hermes-Reply-Mode"];
NSString *instructions = [NSString stringWithFormat:@"%@", arguments[@"instructions"] ?: @""];
if (instructions.length) {
[request setValue:instructions forHTTPHeaderField:@"X-Hermes-Instructions"];
}
NSString *screenURL = [NSString stringWithFormat:@"%@", arguments[@"screen_url"] ?: @""];
if (screenURL.length) {
[request setValue:screenURL forHTTPHeaderField:@"X-Hermes-Screen-Url"];
}
NSString *token = [NSString stringWithFormat:@"%@", arguments[@"api_token"] ?: @""];
if (token.length) {
if (![token.lowercaseString hasPrefix:@"bearer "]) {
token = [@"Bearer " stringByAppendingString:token];
}
[request setValue:token forHTTPHeaderField:@"Authorization"];
}
NSURLResponse *rawResponse = nil;
NSError *requestError = nil;
NSData *responseData = [NSURLConnection sendSynchronousRequest:request
returningResponse:&rawResponse
error:&requestError];
if (requestError || !responseData) {
if ([requestError.domain isEqualToString:NSURLErrorDomain] &&
(requestError.code == NSURLErrorUserCancelledAuthentication ||
requestError.code == NSURLErrorUserAuthenticationRequired)) {
return @{@"error": @"Hermes authorization failed. Provide a valid api_token to hermes_voice_turn."};
}
return @{@"error": requestError.localizedDescription ?: @"Hermes request failed."};
}
NSHTTPURLResponse *response = (NSHTTPURLResponse *)rawResponse;
if (response.statusCode < 200 || response.statusCode >= 300) {
NSString *body = [[NSString alloc] initWithData:responseData encoding:NSUTF8StringEncoding] ?: @"";
return @{@"error": [NSString stringWithFormat:@"Hermes returned HTTP %ld: %@",
(long)response.statusCode,
[body substringToIndex:MIN(body.length, 500)]]};
}
NSDictionary *playError = [self playAudioData:responseData volume:volume];
if (playError) {
return @{@"error": playError[@"message"] ?: @"Hermes audio playback failed."};
}
NSURL *responseURL = [self safeFileURLForPath:@"hermes/response.wav"];
[responseData writeToURL:responseURL atomically:YES];
return @{
@"status": @"playing",
@"http_status": @(response.statusCode),
@"request_bytes": @(wavData.length),
@"response_bytes": @(responseData.length),
@"transcript": [self headerValue:@"X-Hermes-Transcript" fromResponse:response],
@"response_text": [self headerValue:@"X-Hermes-Text-Response" fromResponse:response],
@"response_id": [self headerValue:@"X-Hermes-Response-Id" fromResponse:response],
@"conversation": [self headerValue:@"X-Hermes-Conversation" fromResponse:response],
@"reply_mode": [NSString stringWithFormat:@"%@", arguments[@"reply_mode"] ?: @"sync"],
@"saved_response": @"hermes/response.wav"
};
}
- (NSDictionary *)recordVoiceToURL:(NSURL *)url duration:(NSInteger)duration {
NSError *directoryError = nil;
[[NSFileManager defaultManager] createDirectoryAtURL:[url URLByDeletingLastPathComponent]
withIntermediateDirectories:YES
attributes:nil
error:&directoryError];
if (directoryError) {
return @{@"error": directoryError.localizedDescription};
}
NSError *sessionError = nil;
AVAudioSession *session = [AVAudioSession sharedInstance];
[session setCategory:AVAudioSessionCategoryPlayAndRecord error:&sessionError];
[session setActive:YES error:&sessionError];
if (sessionError) {
return @{@"error": sessionError.localizedDescription};
}
NSDictionary *settings = @{
AVFormatIDKey: @(kAudioFormatLinearPCM),
AVSampleRateKey: @16000,
AVNumberOfChannelsKey: @1,
AVLinearPCMBitDepthKey: @16,
AVLinearPCMIsFloatKey: @NO,
AVLinearPCMIsBigEndianKey: @NO
};
NSError *recordError = nil;
self.audioRecorder = [[AVAudioRecorder alloc] initWithURL:url
settings:settings
error:&recordError];
if (!self.audioRecorder || recordError) {
return @{@"error": recordError.localizedDescription ?: @"Failed to create audio recorder."};
}
[self.audioRecorder prepareToRecord];
[self.audioRecorder recordForDuration:duration];
NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:duration + 0.25];
while (self.audioRecorder.recording && [deadline timeIntervalSinceNow] > 0) {
[[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode
beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.05]];
}
[self.audioRecorder stop];
NSDictionary *attributes = [[NSFileManager defaultManager] attributesOfItemAtPath:url.path error:nil];
NSNumber *bytes = attributes[NSFileSize] ?: @0;
return @{@"filename": url.lastPathComponent ?: @"recording.wav",
@"bytes": bytes,
@"duration_sec": @(duration)};
}
- (NSDictionary *)captureSelfiePNGBase64 {
AVCaptureDeviceDiscoverySession *discovery = [AVCaptureDeviceDiscoverySession discoverySessionWithDeviceTypes:@[
AVCaptureDeviceTypeBuiltInWideAngleCamera
]
mediaType:AVMediaTypeVideo
position:AVCaptureDevicePositionFront];
AVCaptureDevice *camera = discovery.devices.firstObject;
if (!camera) {
return @{@"error": @"Front camera not found."};
}
NSError *inputError = nil;
AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:camera error:&inputError];
if (!input || inputError) {
return @{@"error": inputError.localizedDescription ?: @"Failed to open front camera."};
}
AVCaptureSession *session = [[AVCaptureSession alloc] init];
session.sessionPreset = AVCaptureSessionPresetPhoto;
if (![session canAddInput:input]) {
return @{@"error": @"Could not add front camera input."};
}
[session addInput:input];
AVCaptureStillImageOutput *output = [[AVCaptureStillImageOutput alloc] init];
output.outputSettings = @{AVVideoCodecKey: AVVideoCodecJPEG};
if (![session canAddOutput:output]) {
return @{@"error": @"Could not add still image output."};
}
[session addOutput:output];
[session startRunning];
AVCaptureConnection *connection = [output connectionWithMediaType:AVMediaTypeVideo];
if (connection.isVideoOrientationSupported) {
connection.videoOrientation = AVCaptureVideoOrientationLandscapeRight;
}
__block NSData *jpegData = nil;
__block NSError *captureError = nil;
__block BOOL finished = NO;
[output captureStillImageAsynchronouslyFromConnection:connection
completionHandler:^(CMSampleBufferRef imageDataSampleBuffer, NSError *error) {
if (imageDataSampleBuffer) {
jpegData = [AVCaptureStillImageOutput jpegStillImageNSDataRepresentation:imageDataSampleBuffer];
}
captureError = error;
finished = YES;
}];
NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:5.0];
while (!finished && [deadline timeIntervalSinceNow] > 0) {
[[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode
beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.05]];
}
[session stopRunning];
if (captureError || !jpegData) {
return @{@"error": captureError.localizedDescription ?: @"Selfie capture timed out."};
}
UIImage *image = [UIImage imageWithData:jpegData];
NSData *pngData = UIImagePNGRepresentation(image);
if (!pngData) {
return @{@"error": @"Failed to convert selfie to PNG."};
}
return @{@"png_base64": [pngData base64EncodedStringWithOptions:0]};
}
- (BOOL)ensureMicrophonePermission {
AVAudioSession *session = [AVAudioSession sharedInstance];
if (session.recordPermission == AVAudioSessionRecordPermissionGranted) {
return YES;
}
if (session.recordPermission == AVAudioSessionRecordPermissionDenied) {
return NO;
}
__block BOOL granted = NO;
__block BOOL finished = NO;
[session requestRecordPermission:^(BOOL didGrant) {
granted = didGrant;
finished = YES;
}];
return [self waitForPermissionResult:&finished granted:&granted];
}
- (BOOL)ensureCameraPermission {
AVAuthorizationStatus status = [AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeVideo];
if (status == AVAuthorizationStatusAuthorized) {
return YES;
}
if (status == AVAuthorizationStatusDenied || status == AVAuthorizationStatusRestricted) {
return NO;
}
__block BOOL granted = NO;
__block BOOL finished = NO;
[AVCaptureDevice requestAccessForMediaType:AVMediaTypeVideo
completionHandler:^(BOOL didGrant) {
granted = didGrant;
finished = YES;
}];
return [self waitForPermissionResult:&finished granted:&granted];
}
- (BOOL)waitForPermissionResult:(BOOL *)finished granted:(BOOL *)granted {
NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:30.0];
while (!*finished && [deadline timeIntervalSinceNow] > 0) {
[[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode
beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.05]];
}
return *finished && *granted;
}
- (NSData *)wavDataForToneFrequency:(NSInteger)frequency durationMS:(NSInteger)durationMS volume:(NSInteger)volume {
const uint32_t sampleRate = 22050;
const uint16_t channels = 1;
const uint16_t bitsPerSample = 16;
const uint16_t blockAlign = channels * bitsPerSample / 8;
const uint32_t byteRate = sampleRate * blockAlign;
uint32_t sampleCount = (uint32_t)((sampleRate * durationMS) / 1000);
uint32_t dataSize = sampleCount * blockAlign;
uint32_t riffSize = 36 + dataSize;
NSMutableData *data = [NSMutableData dataWithCapacity:44 + dataSize];
[self appendASCII:"RIFF" toData:data];
[self appendUInt32LE:riffSize toData:data];
[self appendASCII:"WAVE" toData:data];
[self appendASCII:"fmt " toData:data];
[self appendUInt32LE:16 toData:data];
[self appendUInt16LE:1 toData:data];
[self appendUInt16LE:channels toData:data];
[self appendUInt32LE:sampleRate toData:data];
[self appendUInt32LE:byteRate toData:data];
[self appendUInt16LE:blockAlign toData:data];
[self appendUInt16LE:bitsPerSample toData:data];
[self appendASCII:"data" toData:data];
[self appendUInt32LE:dataSize toData:data];
double amplitude = 32767.0 * MAX(0.0, MIN(1.0, volume / 100.0));
double phaseStep = 2.0 * M_PI * frequency / sampleRate;
for (uint32_t i = 0; i < sampleCount; i++) {
double fadeIn = MIN(1.0, i / (sampleRate * 0.01));
double fadeOut = MIN(1.0, (sampleCount - i) / (sampleRate * 0.01));
double envelope = MIN(fadeIn, fadeOut);
int16_t sample = (int16_t)(sin(i * phaseStep) * amplitude * envelope);
[self appendUInt16LE:(uint16_t)sample toData:data];
}
return data;
}
- (void)appendASCII:(const char *)string toData:(NSMutableData *)data {
[data appendBytes:string length:4];
}
- (void)appendUInt16LE:(uint16_t)value toData:(NSMutableData *)data {
uint8_t bytes[] = {value & 0xff, (value >> 8) & 0xff};
[data appendBytes:bytes length:sizeof(bytes)];
}
- (void)appendUInt32LE:(uint32_t)value toData:(NSMutableData *)data {
uint8_t bytes[] = {
value & 0xff,
(value >> 8) & 0xff,
(value >> 16) & 0xff,
(value >> 24) & 0xff
};
[data appendBytes:bytes length:sizeof(bytes)];
}
- (NSString *)videoStreamingInstructionsForProtocol:(NSString *)protocol {
NSMutableArray *lines = [NSMutableArray arrayWithArray:@[
@"### iPhone MCP Video Streaming Instructions",
@"Dimensions: 400x300, 1-bit monochrome, using the same 15,000-byte RLCD frame layout as the ESP32 server.",
@"The iPhone stretches each received frame over the full landscape screen.",
@"UDP discovery: send DISCOVER_SCREEN to port 5000; the iPhone replies SCREEN_IP_8080 because its JSON-RPC endpoint is on port 8080.",
@"Mapping logic (Python):",
@" def map_to_rlcd(pil_img):",
@" img_1bit = pil_img.convert('1', dither=1)",
@" px = img_1bit.load()",
@" buf = bytearray(15000)",
@" for y in range(300):",
@" for x in range(400):",
@" if px[x, y]:",
@" inv_y = 299 - y",
@" bx = x // 2",
@" by = inv_y // 4",
@" idx = bx * 75 + by",
@" lx, ly = x % 2, inv_y % 4",
@" bit = 7 - (ly * 2 + lx)",
@" buf[idx] |= (1 << bit)",
@" return buf"
]];
if ([protocol isEqualToString:@"tcp"] || [protocol isEqualToString:@"both"] || [protocol isEqualToString:@"all"]) {
[lines addObject:@"\n**TCP Streaming (Port 8081):**"];
[lines addObject:@"Open a TCP connection to the iPhone IP on port 8081 and send consecutive 15,000-byte frame blocks."];
}
if ([protocol isEqualToString:@"udp"] || [protocol isEqualToString:@"both"] || [protocol isEqualToString:@"all"]) {
[lines addObject:@"\n**UDP Streaming / Broadcast (Port 8082):**"];
[lines addObject:@"Split each 15,000-byte frame into 15 chunks of 1,000 bytes. Send each chunk as a 1002-byte packet: byte 0 = frame_id (0-255), byte 1 = chunk_idx (0-14), bytes 2..1001 = chunk payload."];
}
if ([protocol isEqualToString:@"color"] || [protocol isEqualToString:@"all"]) {
[lines addObject:@"\n**RGB565 Color TCP Streaming (Port 8083):**"];
[lines addObject:@"Send each frame as a 16-byte header followed by width*height*2 bytes of big-endian RGB565 pixels. Header: bytes 0..3='IMCR', byte 4=1 (version), byte 5=1 (RGB565BE), bytes 6..7=width big-endian, bytes 8..9=height big-endian, bytes 10..13=payload length big-endian, bytes 14..15=flags (zero). Frames may use any dimensions up to 2048x2048 and are scaled to the full iPhone canvas."];
}
return [lines componentsJoinedByString:@"\n"];
}
- (NSString *)batteryStateName:(UIDeviceBatteryState)state {
switch (state) {
case UIDeviceBatteryStateUnplugged:
return @"unplugged";
case UIDeviceBatteryStateCharging:
return @"charging";
case UIDeviceBatteryStateFull:
return @"full";
default:
return @"unknown";
}
}
- (NSURL *)safeFileURLForPath:(NSString *)path {
if (path.length == 0 || [path hasPrefix:@"/"]) {
return nil;
}
if ([[path pathComponents] containsObject:@".."]) {
return nil;
}
NSArray *directories = NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES);
NSString *documents = directories.firstObject;
NSString *root = [documents stringByAppendingPathComponent:@"MCPFiles"];
return [NSURL fileURLWithPath:[root stringByAppendingPathComponent:path]];
}
- (BOOL)prefersStatusBarHidden {
return YES;
}
- (UIInterfaceOrientationMask)supportedInterfaceOrientations {
return UIInterfaceOrientationMaskLandscape;
}
- (UIInterfaceOrientation)preferredInterfaceOrientationForPresentation {
return UIInterfaceOrientationLandscapeRight;
}
@end
Reference in New Issue View Git Blame Copy Permalink