#import "IMCViewController.h" #import "IMCCanvasView.h" #import "IMCMCPServer.h" #import "IMCVideoStreamServer.h" #import #import @interface IMCPhotoCaptureDelegate : NSObject @property (nonatomic, copy) void (^completion)(NSData *jpegData, NSError *error); @end @implementation IMCPhotoCaptureDelegate - (void)captureOutput:(AVCapturePhotoOutput *)output didFinishProcessingPhoto:(AVCapturePhoto *)photo error:(NSError *)error { NSData *data = error ? nil : [photo fileDataRepresentation]; if (self.completion) { self.completion(data, error); } } @end @interface IMCViewController () @property (nonatomic, strong) IMCCanvasView *canvasView; @property (nonatomic, strong) IMCMCPServer *server; @property (nonatomic, strong) IMCVideoStreamServer *videoStreamServer; @property (nonatomic, strong) AVAudioPlayer *audioPlayer; @property (nonatomic, strong) AVAudioRecorder *audioRecorder; @end @implementation IMCViewController - (void)viewDidLoad { [super viewDidLoad]; self.view.backgroundColor = [UIColor colorWithRed:0.03 green:0.06 blue:0.12 alpha:1.0]; self.canvasView = [[IMCCanvasView alloc] initWithFrame:CGRectZero]; self.canvasView.translatesAutoresizingMaskIntoConstraints = NO; [self.view addSubview:self.canvasView]; [NSLayoutConstraint activateConstraints:@[ [self.canvasView.topAnchor constraintEqualToAnchor:self.view.topAnchor], [self.canvasView.bottomAnchor constraintEqualToAnchor:self.view.bottomAnchor], [self.canvasView.leadingAnchor constraintEqualToAnchor:self.view.leadingAnchor], [self.canvasView.trailingAnchor constraintEqualToAnchor:self.view.trailingAnchor] ]]; [self.canvasView clearWithColor:0]; [self.canvasView drawText:@"iPhone MCP landscape server ready" x:18 y:18 size:2]; [self.canvasView drawText:@"POST /api/mcp on port 8080" x:18 y:58 size:1]; self.server = [[IMCMCPServer alloc] initWithViewController:self port:8080]; [self.server start]; self.videoStreamServer = [[IMCVideoStreamServer alloc] initWithCanvasView:self.canvasView]; [self.videoStreamServer start]; } - (NSDictionary *)handleRPCRequest:(NSDictionary *)request { __block NSDictionary *response = nil; dispatch_sync(dispatch_get_main_queue(), ^{ response = [self handleRPCRequestOnMainThread:request]; }); return response; } - (NSDictionary *)handleRPCRequestOnMainThread:(NSDictionary *)request { id rpcID = request[@"id"] ?: [NSNull null]; NSString *method = request[@"method"]; if ([method isEqualToString:@"initialize"]) { return [self result:@{ @"protocolVersion": @"2024-11-05", @"capabilities": @{@"tools": @{}}, @"serverInfo": @{@"name": @"iphone6-mcp", @"version": @"0.0.4"} } rpcID:rpcID]; } if ([method isEqualToString:@"notifications/initialized"]) { return [self result:@{} rpcID:rpcID]; } if ([method isEqualToString:@"tools/list"]) { return [self result:@{@"tools": [self tools]} rpcID:rpcID]; } if ([method isEqualToString:@"tools/call"]) { NSDictionary *params = request[@"params"] ?: @{}; NSString *name = params[@"name"]; NSDictionary *arguments = params[@"arguments"] ?: @{}; return [self callTool:name arguments:arguments rpcID:rpcID]; } return [self errorWithCode:-32601 message:[NSString stringWithFormat:@"Method %@ not found", method] rpcID:rpcID]; } - (NSDictionary *)callTool:(NSString *)name arguments:(NSDictionary *)arguments rpcID:(id)rpcID { @try { if ([name isEqualToString:@"clear_screen"]) { NSInteger color = [arguments[@"color"] integerValue]; [self.canvasView clearWithColor:color]; return [self textResult:@"Screen cleared." rpcID:rpcID]; } if ([name isEqualToString:@"draw_text"]) { NSString *text = [NSString stringWithFormat:@"%@", arguments[@"text"] ?: @""]; CGFloat x = [arguments[@"x"] doubleValue]; CGFloat y = [arguments[@"y"] doubleValue]; NSInteger size = arguments[@"size"] ? [arguments[@"size"] integerValue] : 1; [self.canvasView drawText:text x:x y:y size:size]; return [self textResult:[NSString stringWithFormat:@"Drew text '%@' at (%.0f, %.0f).", text, x, y] rpcID:rpcID]; } if ([name isEqualToString:@"draw_image"]) { NSString *base64 = [NSString stringWithFormat:@"%@", arguments[@"image_base64"] ?: @""]; NSRange marker = [base64 rangeOfString:@";base64,"]; if (marker.location != NSNotFound) { base64 = [base64 substringFromIndex:marker.location + marker.length]; } NSData *data = [[NSData alloc] initWithBase64EncodedString:base64 options:NSDataBase64DecodingIgnoreUnknownCharacters]; if (!data) { return [self errorWithCode:-32602 message:@"Invalid image_base64." rpcID:rpcID]; } [self.canvasView drawImageData:data x:[arguments[@"x"] doubleValue] y:[arguments[@"y"] doubleValue]]; return [self textResult:@"Image displayed." rpcID:rpcID]; } if ([name isEqualToString:@"get_screenshot"]) { NSString *png = [self.canvasView snapshotPNGBase64]; return [self result:@{@"content": @[@{@"type": @"image", @"data": png, @"mimeType": @"image/png"}]} rpcID:rpcID]; } if ([name isEqualToString:@"get_battery"]) { UIDevice *device = [UIDevice currentDevice]; device.batteryMonitoringEnabled = YES; NSDictionary *battery = @{ @"level_pct": @(MAX(device.batteryLevel, 0.0) * 100.0), @"state": [self batteryStateName:device.batteryState] }; return [self textResult:[self jsonString:battery] rpcID:rpcID]; } if ([name isEqualToString:@"get_sensors"] || [name isEqualToString:@"scan_ble"] || [name isEqualToString:@"set_led"]) { return [self textResult:[NSString stringWithFormat:@"%@ is not implemented in this first iPhone build.", name] rpcID:rpcID]; } if ([name isEqualToString:@"play_tone"]) { NSInteger frequency = arguments[@"frequency"] ? [arguments[@"frequency"] integerValue] : 440; NSInteger duration = arguments[@"duration_ms"] ? [arguments[@"duration_ms"] integerValue] : 1000; NSInteger volume = arguments[@"volume"] ? [arguments[@"volume"] integerValue] : 50; frequency = MAX(50, MIN(10000, frequency)); duration = MAX(50, MIN(5000, duration)); volume = MAX(0, MIN(100, volume)); NSData *toneData = [self wavDataForToneFrequency:frequency durationMS:duration volume:volume]; NSDictionary *error = [self playAudioData:toneData volume:volume]; if (error) { return [self errorWithCode:[error[@"code"] integerValue] message:error[@"message"] rpcID:rpcID]; } return [self textResult:[NSString stringWithFormat:@"Played tone of %ldHz for %ldms at volume %ld.", (long)frequency, (long)duration, (long)volume] rpcID:rpcID]; } if ([name isEqualToString:@"play_audio_base64"]) { NSString *base64 = [self base64PayloadFromString:[NSString stringWithFormat:@"%@", arguments[@"wav_base64"] ?: @""]]; NSData *audioData = [[NSData alloc] initWithBase64EncodedString:base64 options:NSDataBase64DecodingIgnoreUnknownCharacters]; if (!audioData) { return [self errorWithCode:-32602 message:@"Invalid wav_base64." rpcID:rpcID]; } NSInteger volumeArg = arguments[@"volume"] ? [arguments[@"volume"] integerValue] : 50; volumeArg = MAX(0, MIN(100, volumeArg)); NSDictionary *error = [self playAudioData:audioData volume:volumeArg]; if (error) { return [self errorWithCode:[error[@"code"] integerValue] message:error[@"message"] rpcID:rpcID]; } return [self textResult:[NSString stringWithFormat:@"Playing base64 WAV audio (%lu bytes) at volume %ld.", (unsigned long)audioData.length, (long)volumeArg] rpcID:rpcID]; } if ([name isEqualToString:@"play_audio"]) { NSString *filename = [NSString stringWithFormat:@"%@", arguments[@"filename"] ?: @""]; NSURL *url = [self safeFileURLForPath:filename]; if (!url) { return [self errorWithCode:-32602 message:@"filename must be relative and cannot contain '..'." rpcID:rpcID]; } NSData *audioData = [NSData dataWithContentsOfURL:url]; if (!audioData) { return [self errorWithCode:-32000 message:[NSString stringWithFormat:@"Could not read audio file '%@' from Documents/MCPFiles.", filename] rpcID:rpcID]; } NSInteger volumeArg = arguments[@"volume"] ? [arguments[@"volume"] integerValue] : 50; volumeArg = MAX(0, MIN(100, volumeArg)); NSDictionary *error = [self playAudioData:audioData volume:volumeArg]; if (error) { return [self errorWithCode:[error[@"code"] integerValue] message:error[@"message"] rpcID:rpcID]; } return [self textResult:[NSString stringWithFormat:@"Playing WAV file '%@' (%lu bytes) at volume %ld.", filename, (unsigned long)audioData.length, (long)volumeArg] rpcID:rpcID]; } if ([name isEqualToString:@"hermes_voice_turn"]) { NSDictionary *voiceResult = [self performHermesVoiceTurn:arguments]; if (voiceResult[@"error"]) { return [self errorWithCode:-32000 message:voiceResult[@"error"] rpcID:rpcID]; } return [self textResult:[self jsonString:voiceResult] rpcID:rpcID]; } if ([name isEqualToString:@"record_voice"]) { if (![self ensureMicrophonePermission]) { return [self errorWithCode:-32000 message:@"Microphone permission is not granted for iPhone MCP." rpcID:rpcID]; } NSInteger duration = arguments[@"duration_sec"] ? [arguments[@"duration_sec"] integerValue] : 4; duration = MAX(1, MIN(15, duration)); NSString *filename = [NSString stringWithFormat:@"%@", arguments[@"filename"] ?: @"recording.wav"]; NSURL *url = [self safeFileURLForPath:filename]; if (!url) { return [self errorWithCode:-32602 message:@"filename must be relative and cannot contain '..'." rpcID:rpcID]; } NSDictionary *result = [self recordVoiceToURL:url duration:duration]; if (result[@"error"]) { return [self errorWithCode:-32000 message:result[@"error"] rpcID:rpcID]; } return [self textResult:[self jsonString:result] rpcID:rpcID]; } if ([name isEqualToString:@"capture_selfie"]) { if (![self ensureCameraPermission]) { return [self errorWithCode:-32000 message:@"Camera permission is not granted for iPhone MCP." rpcID:rpcID]; } NSDictionary *result = [self captureSelfiePNGBase64]; if (result[@"error"]) { return [self errorWithCode:-32000 message:result[@"error"] rpcID:rpcID]; } return [self result:@{@"content": @[@{@"type": @"image", @"data": result[@"png_base64"], @"mimeType": @"image/png"}]} rpcID:rpcID]; } if ([name isEqualToString:@"write_file"]) { NSString *path = [NSString stringWithFormat:@"%@", arguments[@"path"] ?: @""]; NSString *content = [NSString stringWithFormat:@"%@", arguments[@"content"] ?: @""]; NSURL *url = [self safeFileURLForPath:path]; if (!url) { return [self errorWithCode:-32602 message:@"Path must be relative and cannot contain '..'." rpcID:rpcID]; } [[NSFileManager defaultManager] createDirectoryAtURL:[url URLByDeletingLastPathComponent] withIntermediateDirectories:YES attributes:nil error:nil]; NSError *writeError = nil; [content writeToURL:url atomically:YES encoding:NSUTF8StringEncoding error:&writeError]; if (writeError) { return [self errorWithCode:-32000 message:writeError.localizedDescription rpcID:rpcID]; } return [self textResult:[NSString stringWithFormat:@"Wrote %lu characters to %@.", (unsigned long)content.length, path] rpcID:rpcID]; } if ([name isEqualToString:@"read_file"]) { NSString *path = [NSString stringWithFormat:@"%@", arguments[@"path"] ?: @""]; NSURL *url = [self safeFileURLForPath:path]; if (!url) { return [self errorWithCode:-32602 message:@"Path must be relative and cannot contain '..'." rpcID:rpcID]; } NSError *readError = nil; NSString *content = [NSString stringWithContentsOfURL:url encoding:NSUTF8StringEncoding error:&readError]; if (readError) { return [self errorWithCode:-32000 message:readError.localizedDescription rpcID:rpcID]; } return [self textResult:content rpcID:rpcID]; } if ([name isEqualToString:@"download_file"]) { NSString *urlString = [NSString stringWithFormat:@"%@", arguments[@"url"] ?: @""]; NSString *filename = [NSString stringWithFormat:@"%@", arguments[@"filename"] ?: @""]; BOOL useSD = arguments[@"use_sd"] ? [arguments[@"use_sd"] boolValue] : NO; NSURL *sourceURL = [NSURL URLWithString:urlString]; NSURL *destinationURL = [self safeFileURLForPath:filename]; if (!sourceURL || !destinationURL) { return [self errorWithCode:-32602 message:@"Invalid url or filename." rpcID:rpcID]; } NSData *data = [NSData dataWithContentsOfURL:sourceURL]; if (!data) { return [self errorWithCode:-32000 message:@"Download failed." rpcID:rpcID]; } [[NSFileManager defaultManager] createDirectoryAtURL:[destinationURL URLByDeletingLastPathComponent] withIntermediateDirectories:YES attributes:nil error:nil]; [data writeToURL:destinationURL atomically:YES]; NSString *storageNote = useSD ? @" The iPhone build has no SD card, so use_sd was accepted and stored in app Documents instead." : @""; return [self textResult:[NSString stringWithFormat:@"Downloaded %lu bytes to %@.%@", (unsigned long)data.length, filename, storageNote] rpcID:rpcID]; } if ([name isEqualToString:@"get_video_streaming_instructions"]) { NSString *protocol = [[NSString stringWithFormat:@"%@", arguments[@"protocol"] ?: @"both"] lowercaseString]; return [self textResult:[self videoStreamingInstructionsForProtocol:protocol] rpcID:rpcID]; } if ([name isEqualToString:@"get_stream_stats"]) { return [self textResult:[self jsonString:[self.videoStreamServer streamStats]] rpcID:rpcID]; } if ([name isEqualToString:@"reset_stream_stats"]) { [self.videoStreamServer resetStreamStats]; return [self textResult:[self jsonString:[self.videoStreamServer streamStats]] rpcID:rpcID]; } if ([name isEqualToString:@"set_stream_debug"]) { BOOL enabled = arguments[@"enabled"] ? [arguments[@"enabled"] boolValue] : NO; [self.videoStreamServer setDebugEnabled:enabled]; return [self textResult:[self jsonString:@{@"debug_enabled": @(enabled)}] rpcID:rpcID]; } if ([name isEqualToString:@"execute_python"]) { return [self textResult:@"execute_python is available on the MicroPython ESP32 target, but the native iPhone app does not execute arbitrary Python code. Use write_file/read_file/download_file for files and the dedicated drawing/audio/camera tools for app actions." rpcID:rpcID]; } return [self errorWithCode:-32602 message:[NSString stringWithFormat:@"Unknown tool: %@", name] rpcID:rpcID]; } @catch (NSException *exception) { return [self errorWithCode:-32000 message:exception.reason ?: @"Tool failed." rpcID:rpcID]; } } - (NSArray *)tools { return @[ [self tool:@"clear_screen" description:@"Clear the iPhone canvas to white (0) or black (1)." schema:@{@"type": @"object", @"properties": @{@"color": @{@"type": @"integer", @"enum": @[@0, @1]}}, @"required": @[@"color"]}], [self tool:@"draw_text" description:@"Draw text on the iPhone canvas using 400x300 logical coordinates." schema:@{@"type": @"object", @"properties": @{@"text": @{@"type": @"string"}, @"x": @{@"type": @"integer"}, @"y": @{@"type": @"integer"}, @"size": @{@"type": @"integer", @"enum": @[@1, @2]}}, @"required": @[@"text", @"x", @"y"]}], [self tool:@"draw_image" description:@"Draw a base64 PNG/JPEG/GIF image on the iPhone canvas." schema:@{@"type": @"object", @"properties": @{@"image_base64": @{@"type": @"string"}, @"x": @{@"type": @"integer", @"default": @0}, @"y": @{@"type": @"integer", @"default": @0}}, @"required": @[@"image_base64"]}], [self tool:@"get_screenshot" description:@"Return the current iPhone canvas as a PNG image." schema:@{@"type": @"object", @"properties": @{}}], [self tool:@"get_battery" description:@"Read the iPhone battery percentage and charge state." schema:@{@"type": @"object", @"properties": @{}}], [self tool:@"play_tone" description:@"Generate and play a sine-wave tone with the requested frequency, duration, and volume." schema:@{@"type": @"object", @"properties": @{@"frequency": @{@"type": @"integer", @"default": @440}, @"duration_ms": @{@"type": @"integer", @"default": @1000}, @"volume": @{@"type": @"integer", @"default": @50}}}], [self tool:@"play_audio_base64" description:@"Decode a base64 WAV payload and play it through the iPhone speaker." schema:@{@"type": @"object", @"properties": @{@"wav_base64": @{@"type": @"string"}, @"volume": @{@"type": @"integer", @"default": @50}}, @"required": @[@"wav_base64"]}], [self tool:@"play_audio" description:@"Play a WAV file from the app's Documents/MCPFiles directory." schema:@{@"type": @"object", @"properties": @{@"filename": @{@"type": @"string"}, @"volume": @{@"type": @"integer", @"default": @50}}, @"required": @[@"filename"]}], [self tool:@"hermes_voice_turn" description:@"Record a voice command, send it to the Hermes voice gateway, play the returned WAV, and return transcript/response diagnostics." schema:@{@"type": @"object", @"properties": @{@"url": @{@"type": @"string", @"default": @"http://192.168.68.126:8642/api/esp32/voice"}, @"api_token": @{@"type": @"string"}, @"device_id": @{@"type": @"string", @"default": @"iphone6"}, @"duration_sec": @{@"type": @"integer", @"default": @4}, @"volume": @{@"type": @"integer", @"default": @80}, @"instructions": @{@"type": @"string"}, @"reply_mode": @{@"type": @"string", @"enum": @[@"sync", @"ack"], @"default": @"sync"}, @"screen_url": @{@"type": @"string"}}}], [self tool:@"record_voice" description:@"Record microphone audio to a WAV file inside Documents/MCPFiles." schema:@{@"type": @"object", @"properties": @{@"duration_sec": @{@"type": @"integer", @"default": @4}, @"filename": @{@"type": @"string", @"default": @"recording.wav"}}}], [self tool:@"capture_selfie" description:@"Capture a still image from the front camera and return it as PNG image content." schema:@{@"type": @"object", @"properties": @{}}], [self tool:@"write_file" description:@"Write a UTF-8 file inside the app's Documents/MCPFiles directory." schema:@{@"type": @"object", @"properties": @{@"path": @{@"type": @"string"}, @"content": @{@"type": @"string"}}, @"required": @[@"path", @"content"]}], [self tool:@"read_file" description:@"Read a UTF-8 file from the app's Documents/MCPFiles directory." schema:@{@"type": @"object", @"properties": @{@"path": @{@"type": @"string"}}, @"required": @[@"path"]}], [self tool:@"download_file" description:@"Download a URL into the app's Documents/MCPFiles directory. Accepts ESP32-compatible use_sd, but stores locally on iPhone." schema:@{@"type": @"object", @"properties": @{@"url": @{@"type": @"string"}, @"filename": @{@"type": @"string"}, @"use_sd": @{@"type": @"boolean", @"default": @NO}}, @"required": @[@"url", @"filename"]}], [self tool:@"get_video_streaming_instructions" description:@"Get details for monochrome TCP/UDP streaming and RGB565 color TCP streaming." schema:@{@"type": @"object", @"properties": @{@"protocol": @{@"type": @"string", @"enum": @[@"tcp", @"udp", @"color", @"both", @"all"], @"default": @"all"}}}], [self tool:@"get_stream_stats" description:@"Return TCP/UDP streaming counters, ports, uptime, last protocol, and debug state." schema:@{@"type": @"object", @"properties": @{}}], [self tool:@"reset_stream_stats" description:@"Reset TCP/UDP streaming counters and last-frame state." schema:@{@"type": @"object", @"properties": @{}}], [self tool:@"set_stream_debug" description:@"Enable or disable stream debug logging on the iPhone." schema:@{@"type": @"object", @"properties": @{@"enabled": @{@"type": @"boolean", @"default": @NO}}}], [self tool:@"execute_python" description:@"ESP32 compatibility stub; arbitrary Python execution is intentionally not implemented in the native iPhone app." schema:@{@"type": @"object", @"properties": @{@"code": @{@"type": @"string"}}, @"required": @[@"code"]}], [self tool:@"set_led" description:@"ESP32 compatibility stub; the iPhone 6 has no NeoPixel LED." schema:@{@"type": @"object", @"properties": @{}}], [self tool:@"get_sensors" description:@"ESP32 compatibility stub; SHTC3 temperature/humidity is not present." schema:@{@"type": @"object", @"properties": @{}}], [self tool:@"scan_ble" description:@"ESP32 compatibility stub; BLE scanning is not implemented in this first build." schema:@{@"type": @"object", @"properties": @{}}] ]; } - (NSDictionary *)tool:(NSString *)name description:(NSString *)description schema:(NSDictionary *)schema { return @{@"name": name, @"description": description, @"inputSchema": schema}; } - (NSDictionary *)textResult:(NSString *)text rpcID:(id)rpcID { return [self result:@{@"content": @[@{@"type": @"text", @"text": text ?: @""}]} rpcID:rpcID]; } - (NSDictionary *)result:(NSDictionary *)result rpcID:(id)rpcID { return @{@"jsonrpc": @"2.0", @"result": result ?: @{}, @"id": rpcID ?: [NSNull null]}; } - (NSDictionary *)errorWithCode:(NSInteger)code message:(NSString *)message rpcID:(id)rpcID { return @{@"jsonrpc": @"2.0", @"error": @{@"code": @(code), @"message": message ?: @"Error"}, @"id": rpcID ?: [NSNull null]}; } - (NSString *)jsonString:(id)object { NSData *data = [NSJSONSerialization dataWithJSONObject:object options:0 error:nil]; return [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] ?: @"{}"; } - (NSString *)base64PayloadFromString:(NSString *)string { NSRange marker = [string rangeOfString:@";base64,"]; if (marker.location != NSNotFound) { return [string substringFromIndex:marker.location + marker.length]; } return string ?: @""; } - (NSDictionary *)playAudioData:(NSData *)audioData volume:(NSInteger)volume { NSError *sessionError = nil; [[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayback error:&sessionError]; [[AVAudioSession sharedInstance] setActive:YES error:&sessionError]; NSError *playerError = nil; self.audioPlayer = [[AVAudioPlayer alloc] initWithData:audioData error:&playerError]; if (!self.audioPlayer || playerError) { return @{@"code": @-32000, @"message": playerError.localizedDescription ?: @"Failed to create audio player."}; } self.audioPlayer.volume = MAX(0.0, MIN(1.0, volume / 100.0)); [self.audioPlayer prepareToPlay]; [self.audioPlayer play]; return nil; } - (NSString *)headerValue:(NSString *)name fromResponse:(NSHTTPURLResponse *)response { for (id key in response.allHeaderFields) { if ([[NSString stringWithFormat:@"%@", key] caseInsensitiveCompare:name] == NSOrderedSame) { return [NSString stringWithFormat:@"%@", response.allHeaderFields[key]]; } } return @""; } - (NSDictionary *)performHermesVoiceTurn:(NSDictionary *)arguments { if (![self ensureMicrophonePermission]) { return @{@"error": @"Microphone permission is not granted for iPhone MCP."}; } NSInteger duration = arguments[@"duration_sec"] ? [arguments[@"duration_sec"] integerValue] : 4; duration = MAX(1, MIN(20, duration)); NSInteger volume = arguments[@"volume"] ? [arguments[@"volume"] integerValue] : 80; volume = MAX(0, MIN(100, volume)); NSString *urlString = [NSString stringWithFormat:@"%@", arguments[@"url"] ?: @"http://192.168.68.126:8642/api/esp32/voice"]; NSURL *gatewayURL = [NSURL URLWithString:urlString]; if (!gatewayURL || ![@[@"http", @"https"] containsObject:gatewayURL.scheme.lowercaseString]) { return @{@"error": @"Hermes url must be an http or https URL."}; } NSURL *recordingURL = [self safeFileURLForPath:@"hermes/request.wav"]; NSDictionary *recording = [self recordVoiceToURL:recordingURL duration:duration]; if (recording[@"error"]) { return recording; } NSData *wavData = [NSData dataWithContentsOfURL:recordingURL]; if (!wavData.length) { return @{@"error": @"Hermes recording produced no audio data."}; } NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:gatewayURL cachePolicy:NSURLRequestReloadIgnoringLocalCacheData timeoutInterval:90.0]; request.HTTPMethod = @"POST"; request.HTTPBody = wavData; [request setValue:@"audio/wav" forHTTPHeaderField:@"Content-Type"]; [request setValue:[NSString stringWithFormat:@"%@", arguments[@"device_id"] ?: @"iphone6"] forHTTPHeaderField:@"X-Device-ID"]; [request setValue:[NSString stringWithFormat:@"%@", arguments[@"reply_mode"] ?: @"sync"] forHTTPHeaderField:@"X-Hermes-Reply-Mode"]; NSString *instructions = [NSString stringWithFormat:@"%@", arguments[@"instructions"] ?: @""]; if (instructions.length) { [request setValue:instructions forHTTPHeaderField:@"X-Hermes-Instructions"]; } NSString *screenURL = [NSString stringWithFormat:@"%@", arguments[@"screen_url"] ?: @""]; if (screenURL.length) { [request setValue:screenURL forHTTPHeaderField:@"X-Hermes-Screen-Url"]; } NSString *token = [NSString stringWithFormat:@"%@", arguments[@"api_token"] ?: @""]; if (token.length) { if (![token.lowercaseString hasPrefix:@"bearer "]) { token = [@"Bearer " stringByAppendingString:token]; } [request setValue:token forHTTPHeaderField:@"Authorization"]; } NSURLResponse *rawResponse = nil; NSError *requestError = nil; NSData *responseData = [NSURLConnection sendSynchronousRequest:request returningResponse:&rawResponse error:&requestError]; if (requestError || !responseData) { if ([requestError.domain isEqualToString:NSURLErrorDomain] && (requestError.code == NSURLErrorUserCancelledAuthentication || requestError.code == NSURLErrorUserAuthenticationRequired)) { return @{@"error": @"Hermes authorization failed. Provide a valid api_token to hermes_voice_turn."}; } return @{@"error": requestError.localizedDescription ?: @"Hermes request failed."}; } NSHTTPURLResponse *response = (NSHTTPURLResponse *)rawResponse; if (response.statusCode < 200 || response.statusCode >= 300) { NSString *body = [[NSString alloc] initWithData:responseData encoding:NSUTF8StringEncoding] ?: @""; return @{@"error": [NSString stringWithFormat:@"Hermes returned HTTP %ld: %@", (long)response.statusCode, [body substringToIndex:MIN(body.length, 500)]]}; } NSDictionary *playError = [self playAudioData:responseData volume:volume]; if (playError) { return @{@"error": playError[@"message"] ?: @"Hermes audio playback failed."}; } NSURL *responseURL = [self safeFileURLForPath:@"hermes/response.wav"]; [responseData writeToURL:responseURL atomically:YES]; return @{ @"status": @"playing", @"http_status": @(response.statusCode), @"request_bytes": @(wavData.length), @"response_bytes": @(responseData.length), @"transcript": [self headerValue:@"X-Hermes-Transcript" fromResponse:response], @"response_text": [self headerValue:@"X-Hermes-Text-Response" fromResponse:response], @"response_id": [self headerValue:@"X-Hermes-Response-Id" fromResponse:response], @"conversation": [self headerValue:@"X-Hermes-Conversation" fromResponse:response], @"reply_mode": [NSString stringWithFormat:@"%@", arguments[@"reply_mode"] ?: @"sync"], @"saved_response": @"hermes/response.wav" }; } - (NSDictionary *)recordVoiceToURL:(NSURL *)url duration:(NSInteger)duration { NSError *directoryError = nil; [[NSFileManager defaultManager] createDirectoryAtURL:[url URLByDeletingLastPathComponent] withIntermediateDirectories:YES attributes:nil error:&directoryError]; if (directoryError) { return @{@"error": directoryError.localizedDescription}; } NSError *sessionError = nil; AVAudioSession *session = [AVAudioSession sharedInstance]; [session setCategory:AVAudioSessionCategoryPlayAndRecord error:&sessionError]; [session setActive:YES error:&sessionError]; if (sessionError) { return @{@"error": sessionError.localizedDescription}; } NSDictionary *settings = @{ AVFormatIDKey: @(kAudioFormatLinearPCM), AVSampleRateKey: @16000, AVNumberOfChannelsKey: @1, AVLinearPCMBitDepthKey: @16, AVLinearPCMIsFloatKey: @NO, AVLinearPCMIsBigEndianKey: @NO }; NSError *recordError = nil; self.audioRecorder = [[AVAudioRecorder alloc] initWithURL:url settings:settings error:&recordError]; if (!self.audioRecorder || recordError) { return @{@"error": recordError.localizedDescription ?: @"Failed to create audio recorder."}; } [self.audioRecorder prepareToRecord]; [self.audioRecorder recordForDuration:duration]; NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:duration + 0.25]; while (self.audioRecorder.recording && [deadline timeIntervalSinceNow] > 0) { [[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.05]]; } [self.audioRecorder stop]; NSDictionary *attributes = [[NSFileManager defaultManager] attributesOfItemAtPath:url.path error:nil]; NSNumber *bytes = attributes[NSFileSize] ?: @0; return @{@"filename": url.lastPathComponent ?: @"recording.wav", @"bytes": bytes, @"duration_sec": @(duration)}; } - (NSDictionary *)captureSelfiePNGBase64 { AVCaptureDeviceDiscoverySession *discovery = [AVCaptureDeviceDiscoverySession discoverySessionWithDeviceTypes:@[ AVCaptureDeviceTypeBuiltInWideAngleCamera ] mediaType:AVMediaTypeVideo position:AVCaptureDevicePositionFront]; AVCaptureDevice *camera = discovery.devices.firstObject; if (!camera) { return @{@"error": @"Front camera not found."}; } NSError *inputError = nil; AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:camera error:&inputError]; if (!input || inputError) { return @{@"error": inputError.localizedDescription ?: @"Failed to open front camera."}; } AVCaptureSession *session = [[AVCaptureSession alloc] init]; session.sessionPreset = AVCaptureSessionPresetPhoto; if (![session canAddInput:input]) { return @{@"error": @"Could not add front camera input."}; } [session addInput:input]; AVCaptureStillImageOutput *output = [[AVCaptureStillImageOutput alloc] init]; output.outputSettings = @{AVVideoCodecKey: AVVideoCodecJPEG}; if (![session canAddOutput:output]) { return @{@"error": @"Could not add still image output."}; } [session addOutput:output]; [session startRunning]; AVCaptureConnection *connection = [output connectionWithMediaType:AVMediaTypeVideo]; if (connection.isVideoOrientationSupported) { connection.videoOrientation = AVCaptureVideoOrientationLandscapeRight; } __block NSData *jpegData = nil; __block NSError *captureError = nil; __block BOOL finished = NO; [output captureStillImageAsynchronouslyFromConnection:connection completionHandler:^(CMSampleBufferRef imageDataSampleBuffer, NSError *error) { if (imageDataSampleBuffer) { jpegData = [AVCaptureStillImageOutput jpegStillImageNSDataRepresentation:imageDataSampleBuffer]; } captureError = error; finished = YES; }]; NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:5.0]; while (!finished && [deadline timeIntervalSinceNow] > 0) { [[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.05]]; } [session stopRunning]; if (captureError || !jpegData) { return @{@"error": captureError.localizedDescription ?: @"Selfie capture timed out."}; } UIImage *image = [UIImage imageWithData:jpegData]; NSData *pngData = UIImagePNGRepresentation(image); if (!pngData) { return @{@"error": @"Failed to convert selfie to PNG."}; } return @{@"png_base64": [pngData base64EncodedStringWithOptions:0]}; } - (BOOL)ensureMicrophonePermission { AVAudioSession *session = [AVAudioSession sharedInstance]; if (session.recordPermission == AVAudioSessionRecordPermissionGranted) { return YES; } if (session.recordPermission == AVAudioSessionRecordPermissionDenied) { return NO; } __block BOOL granted = NO; __block BOOL finished = NO; [session requestRecordPermission:^(BOOL didGrant) { granted = didGrant; finished = YES; }]; return [self waitForPermissionResult:&finished granted:&granted]; } - (BOOL)ensureCameraPermission { AVAuthorizationStatus status = [AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeVideo]; if (status == AVAuthorizationStatusAuthorized) { return YES; } if (status == AVAuthorizationStatusDenied || status == AVAuthorizationStatusRestricted) { return NO; } __block BOOL granted = NO; __block BOOL finished = NO; [AVCaptureDevice requestAccessForMediaType:AVMediaTypeVideo completionHandler:^(BOOL didGrant) { granted = didGrant; finished = YES; }]; return [self waitForPermissionResult:&finished granted:&granted]; } - (BOOL)waitForPermissionResult:(BOOL *)finished granted:(BOOL *)granted { NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:30.0]; while (!*finished && [deadline timeIntervalSinceNow] > 0) { [[NSRunLoop currentRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.05]]; } return *finished && *granted; } - (NSData *)wavDataForToneFrequency:(NSInteger)frequency durationMS:(NSInteger)durationMS volume:(NSInteger)volume { const uint32_t sampleRate = 22050; const uint16_t channels = 1; const uint16_t bitsPerSample = 16; const uint16_t blockAlign = channels * bitsPerSample / 8; const uint32_t byteRate = sampleRate * blockAlign; uint32_t sampleCount = (uint32_t)((sampleRate * durationMS) / 1000); uint32_t dataSize = sampleCount * blockAlign; uint32_t riffSize = 36 + dataSize; NSMutableData *data = [NSMutableData dataWithCapacity:44 + dataSize]; [self appendASCII:"RIFF" toData:data]; [self appendUInt32LE:riffSize toData:data]; [self appendASCII:"WAVE" toData:data]; [self appendASCII:"fmt " toData:data]; [self appendUInt32LE:16 toData:data]; [self appendUInt16LE:1 toData:data]; [self appendUInt16LE:channels toData:data]; [self appendUInt32LE:sampleRate toData:data]; [self appendUInt32LE:byteRate toData:data]; [self appendUInt16LE:blockAlign toData:data]; [self appendUInt16LE:bitsPerSample toData:data]; [self appendASCII:"data" toData:data]; [self appendUInt32LE:dataSize toData:data]; double amplitude = 32767.0 * MAX(0.0, MIN(1.0, volume / 100.0)); double phaseStep = 2.0 * M_PI * frequency / sampleRate; for (uint32_t i = 0; i < sampleCount; i++) { double fadeIn = MIN(1.0, i / (sampleRate * 0.01)); double fadeOut = MIN(1.0, (sampleCount - i) / (sampleRate * 0.01)); double envelope = MIN(fadeIn, fadeOut); int16_t sample = (int16_t)(sin(i * phaseStep) * amplitude * envelope); [self appendUInt16LE:(uint16_t)sample toData:data]; } return data; } - (void)appendASCII:(const char *)string toData:(NSMutableData *)data { [data appendBytes:string length:4]; } - (void)appendUInt16LE:(uint16_t)value toData:(NSMutableData *)data { uint8_t bytes[] = {value & 0xff, (value >> 8) & 0xff}; [data appendBytes:bytes length:sizeof(bytes)]; } - (void)appendUInt32LE:(uint32_t)value toData:(NSMutableData *)data { uint8_t bytes[] = { value & 0xff, (value >> 8) & 0xff, (value >> 16) & 0xff, (value >> 24) & 0xff }; [data appendBytes:bytes length:sizeof(bytes)]; } - (NSString *)videoStreamingInstructionsForProtocol:(NSString *)protocol { NSMutableArray *lines = [NSMutableArray arrayWithArray:@[ @"### iPhone MCP Video Streaming Instructions", @"Dimensions: 400x300, 1-bit monochrome, using the same 15,000-byte RLCD frame layout as the ESP32 server.", @"The iPhone stretches each received frame over the full landscape screen.", @"UDP discovery: send DISCOVER_SCREEN to port 5000; the iPhone replies SCREEN_IP_8080 because its JSON-RPC endpoint is on port 8080.", @"Mapping logic (Python):", @" def map_to_rlcd(pil_img):", @" img_1bit = pil_img.convert('1', dither=1)", @" px = img_1bit.load()", @" buf = bytearray(15000)", @" for y in range(300):", @" for x in range(400):", @" if px[x, y]:", @" inv_y = 299 - y", @" bx = x // 2", @" by = inv_y // 4", @" idx = bx * 75 + by", @" lx, ly = x % 2, inv_y % 4", @" bit = 7 - (ly * 2 + lx)", @" buf[idx] |= (1 << bit)", @" return buf" ]]; if ([protocol isEqualToString:@"tcp"] || [protocol isEqualToString:@"both"] || [protocol isEqualToString:@"all"]) { [lines addObject:@"\n**TCP Streaming (Port 8081):**"]; [lines addObject:@"Open a TCP connection to the iPhone IP on port 8081 and send consecutive 15,000-byte frame blocks."]; } if ([protocol isEqualToString:@"udp"] || [protocol isEqualToString:@"both"] || [protocol isEqualToString:@"all"]) { [lines addObject:@"\n**UDP Streaming / Broadcast (Port 8082):**"]; [lines addObject:@"Split each 15,000-byte frame into 15 chunks of 1,000 bytes. Send each chunk as a 1002-byte packet: byte 0 = frame_id (0-255), byte 1 = chunk_idx (0-14), bytes 2..1001 = chunk payload."]; } if ([protocol isEqualToString:@"color"] || [protocol isEqualToString:@"all"]) { [lines addObject:@"\n**RGB565 Color TCP Streaming (Port 8083):**"]; [lines addObject:@"Send each frame as a 16-byte header followed by width*height*2 bytes of big-endian RGB565 pixels. Header: bytes 0..3='IMCR', byte 4=1 (version), byte 5=1 (RGB565BE), bytes 6..7=width big-endian, bytes 8..9=height big-endian, bytes 10..13=payload length big-endian, bytes 14..15=flags (zero). Frames may use any dimensions up to 2048x2048 and are scaled to the full iPhone canvas."]; } return [lines componentsJoinedByString:@"\n"]; } - (NSString *)batteryStateName:(UIDeviceBatteryState)state { switch (state) { case UIDeviceBatteryStateUnplugged: return @"unplugged"; case UIDeviceBatteryStateCharging: return @"charging"; case UIDeviceBatteryStateFull: return @"full"; default: return @"unknown"; } } - (NSURL *)safeFileURLForPath:(NSString *)path { if (path.length == 0 || [path hasPrefix:@"/"]) { return nil; } if ([[path pathComponents] containsObject:@".."]) { return nil; } NSArray *directories = NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES); NSString *documents = directories.firstObject; NSString *root = [documents stringByAppendingPathComponent:@"MCPFiles"]; return [NSURL fileURLWithPath:[root stringByAppendingPathComponent:path]]; } - (BOOL)prefersStatusBarHidden { return YES; } - (UIInterfaceOrientationMask)supportedInterfaceOrientations { return UIInterfaceOrientationMaskLandscape; } - (UIInterfaceOrientation)preferredInterfaceOrientationForPresentation { return UIInterfaceOrientationLandscapeRight; } @end