diff --git a/Apps/McpScreen/IMPLEMENTATION_PLAN.md b/Apps/McpScreen/IMPLEMENTATION_PLAN.md index a037305..bcb3e05 100644 --- a/Apps/McpScreen/IMPLEMENTATION_PLAN.md +++ b/Apps/McpScreen/IMPLEMENTATION_PLAN.md @@ -64,9 +64,12 @@ Pending verification: Tactility unloads external-app ELF memory immediately after `onDestroy` returns. An app task blocked in `accept()` therefore cannot be allowed to -survive destruction. The HTTP listener is non-blocking, client reads have -short timeouts, and `onDestroy` closes sockets, waits for the worker to signal -exit, then deletes the suspended worker task before returning. +survive destruction. The HTTP listener is non-blocking and client reads have +short timeouts. `onHide` only clears the run flag and returns immediately so +it does not block Tactility's GUI lifecycle or close an lwIP descriptor from +the wrong task. The worker owns the socket shutdown, then suspends itself +asynchronously. The next `onShow` reaps it before starting a new worker, while +`onDestroy` waits for and deletes it before returning. ## Design principles @@ -124,20 +127,20 @@ implementation is easier to validate. - `onCreate` - Allocate application state. - - Start UDP discovery and HTTP MCP service tasks. - `onShow` - Build the dashboard and MCP-controlled display canvas. - Attach the current UI objects to the shared application state. + - Start the HTTP MCP service. - `onHide` + - Signal the HTTP MCP service to stop without blocking the GUI lifecycle. - Detach and invalidate UI object pointers. - - Keep network services running while the application is merely hidden, if - Tactility preserves the app instance. - `onDestroy` - - Stop tasks and sockets. + - Join/delete any remaining worker and close sockets as a safety fallback. - Release buffers, devices, and application state. -The first implementation slice must explicitly verify whether Tactility keeps -the app process and `onCreate` state alive while another app is visible. +The MCP server is deliberately foreground-only. Tactility does not need to +keep the listener or display-control task active while another app owns the +screen. ### FreeRTOS responsibilities diff --git a/Apps/McpScreen/README.md b/Apps/McpScreen/README.md index 746ff33..05cdbab 100644 --- a/Apps/McpScreen/README.md +++ b/Apps/McpScreen/README.md @@ -2,7 +2,7 @@ Native Tactility port of the MicroPython MCP Screen firmware. -The current Phase 1 slice provides: +Phase 1 provides: - JSON-RPC 2.0 over `POST /api/mcp` on port 80 - `tools/list` @@ -10,6 +10,16 @@ The current Phase 1 slice provides: - `clear_screen` - `draw_text` +Phase 2 adds: + +- `draw_raw_rgb565` +- `POST /api/screen/raw` +- `draw_color_bmp` +- bridge-assisted PBM `draw_image` +- PBM `get_screenshot` + +Backlight and screen-power controls are intentionally excluded. + See `IMPLEMENTATION_PLAN.md` for the full roadmap. UDP discovery is temporarily unavailable to an external app because the @@ -17,9 +27,14 @@ Tactility 0.7.0-dev firmware does not export `lwip_recvfrom` and `lwip_sendto`. The existing MicroPython bridge remains compatible through its HTTP subnet-scan fallback. -The HTTP worker uses a non-blocking listener and is explicitly stopped and -deleted during `onDestroy`. This is required because Tactility unloads the -external ELF after the app closes. +The HTTP worker uses a non-blocking listener and short client timeouts. +`onHide` signals it and returns immediately; the worker owns the socket +shutdown and then suspends itself outside the GUI lifecycle. It is reaped by +the next `onShow`, or by `onDestroy` before Tactility unloads the external ELF. + +The MCP server is foreground-only: it starts in `onShow`, and port 80 closes +within one short worker poll after `onHide`. It is intentionally unavailable +whenever McpScreen is not the active screen. ## Build @@ -51,7 +66,7 @@ The existing MicroPython host bridge can also be used unchanged: python3 /Users/adolforeyna/Projects/MicroPython/test1/Screen/mcp_bridge.py ``` -Or run the included Phase 1 smoke test: +Or run the included Phase 2 smoke test: ```bash python3 Apps/McpScreen/tools/smoke_test.py --ip DEVICE_IP --draw diff --git a/Apps/McpScreen/main/Source/DisplayTools.c b/Apps/McpScreen/main/Source/DisplayTools.c new file mode 100644 index 0000000..f86be10 --- /dev/null +++ b/Apps/McpScreen/main/Source/DisplayTools.c @@ -0,0 +1,337 @@ +#include "McpScreen.h" + +#include +#include +#include + +#include +#include + +static const char* TAG = "DisplayTools"; + +static bool ascii_space(uint8_t value) { + return value == ' ' || value == '\t' || value == '\r' || + value == '\n' || value == '\f' || value == '\v'; +} + +static bool ascii_digit(uint8_t value) { + return value >= '0' && value <= '9'; +} + +static uint16_t read_le16(const uint8_t* value) { + return (uint16_t)value[0] | ((uint16_t)value[1] << 8); +} + +static uint32_t read_le32(const uint8_t* value) { + return (uint32_t)value[0] | + ((uint32_t)value[1] << 8) | + ((uint32_t)value[2] << 16) | + ((uint32_t)value[3] << 24); +} + +static bool display_ready(McpScreenState* state) { + return state != NULL && + state->visible && + state->draw_area != NULL && + state->framebuffer != NULL && + state->draw_width > 0 && + state->draw_height > 0; +} + +static uint16_t rgb888_to_rgb565(uint8_t red, uint8_t green, uint8_t blue) { + return (uint16_t)(((red & 0xF8) << 8) | + ((green & 0xFC) << 3) | + (blue >> 3)); +} + +static void put_pixel(McpScreenState* state, int x, int y, uint16_t color) { + if (x >= 0 && y >= 0 && x < state->draw_width && y < state->draw_height) { + state->framebuffer[(size_t)y * state->draw_width + x] = color; + } +} + +bool mcp_ui_draw_rgb565_be( + McpScreenState* state, + const uint8_t* data, + size_t data_size, + int x, + int y, + int width, + int height +) { + if (data == NULL || width <= 0 || height <= 0 || + data_size < (size_t)width * height * 2 || !display_ready(state)) { + return false; + } + + bool success = false; + if (tt_lvgl_lock(pdMS_TO_TICKS(2000))) { + if (display_ready(state)) { + for (int source_y = 0; source_y < height; ++source_y) { + int destination_y = y + source_y; + if (destination_y < 0 || destination_y >= state->draw_height) { + continue; + } + for (int source_x = 0; source_x < width; ++source_x) { + int destination_x = x + source_x; + if (destination_x < 0 || destination_x >= state->draw_width) { + continue; + } + size_t offset = ((size_t)source_y * width + source_x) * 2; + uint16_t color = ((uint16_t)data[offset] << 8) | data[offset + 1]; + put_pixel(state, destination_x, destination_y, color); + } + } + lv_obj_invalidate(state->draw_area); + state->override_active = true; + success = true; + } + tt_lvgl_unlock(); + } + return success; +} + +bool mcp_ui_draw_bmp( + McpScreenState* state, + const uint8_t* data, + size_t data_size, + int x, + int y +) { + if (data == NULL || data_size < 54 || !display_ready(state) || + data[0] != 'B' || data[1] != 'M') { + return false; + } + + uint32_t pixel_offset = read_le32(data + 10); + uint32_t dib_size = read_le32(data + 14); + int32_t width = (int32_t)read_le32(data + 18); + int32_t signed_height = (int32_t)read_le32(data + 22); + uint16_t planes = read_le16(data + 26); + uint16_t bits_per_pixel = read_le16(data + 28); + uint32_t compression = read_le32(data + 30); + if (dib_size < 40 || width <= 0 || signed_height == 0 || planes != 1 || + (bits_per_pixel != 24 && bits_per_pixel != 32) || compression != 0) { + return false; + } + + int height = signed_height < 0 ? -signed_height : signed_height; + bool top_down = signed_height < 0; + size_t row_stride = (((size_t)width * bits_per_pixel + 31) / 32) * 4; + if (pixel_offset > data_size || + row_stride > data_size || + (size_t)height > (data_size - pixel_offset) / row_stride) { + return false; + } + + bool success = false; + if (tt_lvgl_lock(pdMS_TO_TICKS(2500))) { + if (display_ready(state)) { + size_t bytes_per_pixel = bits_per_pixel / 8; + for (int source_y = 0; source_y < height; ++source_y) { + int file_y = top_down ? source_y : (height - 1 - source_y); + const uint8_t* row = data + pixel_offset + (size_t)file_y * row_stride; + for (int source_x = 0; source_x < width; ++source_x) { + const uint8_t* pixel = row + (size_t)source_x * bytes_per_pixel; + put_pixel( + state, + x + source_x, + y + source_y, + rgb888_to_rgb565(pixel[2], pixel[1], pixel[0]) + ); + } + } + lv_obj_invalidate(state->draw_area); + state->override_active = true; + success = true; + } + tt_lvgl_unlock(); + } + return success; +} + +static bool pbm_next_number( + const uint8_t* data, + size_t data_size, + size_t* offset, + int* result +) { + while (*offset < data_size) { + if (data[*offset] == '#') { + while (*offset < data_size && data[*offset] != '\n') { + (*offset)++; + } + } else if (ascii_space(data[*offset])) { + (*offset)++; + } else { + break; + } + } + if (*offset >= data_size || !ascii_digit(data[*offset])) { + return false; + } + + int value = 0; + while (*offset < data_size && ascii_digit(data[*offset])) { + value = value * 10 + (data[*offset] - '0'); + (*offset)++; + } + *result = value; + return true; +} + +bool mcp_ui_draw_pbm( + McpScreenState* state, + const uint8_t* data, + size_t data_size, + int x, + int y +) { + if (data == NULL || data_size < 8 || !display_ready(state) || + data[0] != 'P' || data[1] != '4') { + ESP_LOGE( + TAG, + "PBM precondition failed data=%p size=%u ready=%d magic=%02x%02x", + data, + (unsigned)data_size, + display_ready(state), + data_size > 0 ? data[0] : 0, + data_size > 1 ? data[1] : 0 + ); + return false; + } + + size_t offset = 2; + int width = 0; + int height = 0; + if (!pbm_next_number(data, data_size, &offset, &width) || + !pbm_next_number(data, data_size, &offset, &height) || + width <= 0 || height <= 0) { + ESP_LOGE(TAG, "PBM dimension parse failed offset=%u w=%d h=%d", (unsigned)offset, width, height); + return false; + } + if (offset >= data_size || !ascii_space(data[offset])) { + ESP_LOGE( + TAG, + "PBM missing header separator offset=%u size=%u byte=%02x", + (unsigned)offset, + (unsigned)data_size, + offset < data_size ? data[offset] : 0 + ); + return false; + } + if (data[offset] == '\r' && offset + 1 < data_size && data[offset + 1] == '\n') { + offset += 2; + } else { + offset++; + } + + size_t row_bytes = ((size_t)width + 7) / 8; + if (offset > data_size || + (size_t)height > (data_size - offset) / row_bytes) { + ESP_LOGE( + TAG, + "PBM payload too small offset=%u size=%u row=%u h=%d", + (unsigned)offset, + (unsigned)data_size, + (unsigned)row_bytes, + height + ); + return false; + } + + bool success = false; + if (tt_lvgl_lock(pdMS_TO_TICKS(2000))) { + if (display_ready(state)) { + for (int source_y = 0; source_y < height; ++source_y) { + const uint8_t* row = data + offset + (size_t)source_y * row_bytes; + for (int source_x = 0; source_x < width; ++source_x) { + bool black = (row[source_x >> 3] & (0x80 >> (source_x & 7))) != 0; + put_pixel(state, x + source_x, y + source_y, black ? 0x0000 : 0xFFFF); + } + } + lv_obj_invalidate(state->draw_area); + state->override_active = true; + success = true; + } + tt_lvgl_unlock(); + } else { + ESP_LOGE(TAG, "PBM LVGL lock timed out"); + } + if (!success) ESP_LOGE(TAG, "PBM draw failed after lock ready=%d", display_ready(state)); + return success; +} + +static char* base64_encode(const uint8_t* input, size_t input_size) { + static const char alphabet[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + size_t output_size = ((input_size + 2) / 3) * 4; + char* output = malloc(output_size + 1); + if (output == NULL) { + return NULL; + } + + size_t in = 0; + size_t out = 0; + while (in < input_size) { + uint32_t value = (uint32_t)input[in++] << 16; + bool have_second = in < input_size; + if (have_second) value |= (uint32_t)input[in++] << 8; + bool have_third = in < input_size; + if (have_third) value |= input[in++]; + + output[out++] = alphabet[(value >> 18) & 0x3F]; + output[out++] = alphabet[(value >> 12) & 0x3F]; + output[out++] = have_second ? alphabet[(value >> 6) & 0x3F] : '='; + output[out++] = have_third ? alphabet[value & 0x3F] : '='; + } + output[out] = '\0'; + return output; +} + +char* mcp_ui_get_screenshot_pbm_base64(McpScreenState* state) { + if (!display_ready(state)) { + return NULL; + } + + size_t row_bytes = ((size_t)state->draw_width + 7) / 8; + char header[40]; + int header_size = snprintf( + header, + sizeof(header), + "P4\n%u %u\n", + state->draw_width, + state->draw_height + ); + size_t pbm_size = (size_t)header_size + row_bytes * state->draw_height; + uint8_t* pbm = calloc(1, pbm_size); + if (pbm == NULL) { + return NULL; + } + memcpy(pbm, header, header_size); + + bool success = false; + if (tt_lvgl_lock(pdMS_TO_TICKS(2000))) { + if (display_ready(state)) { + for (int y = 0; y < state->draw_height; ++y) { + uint8_t* row = pbm + header_size + (size_t)y * row_bytes; + for (int x = 0; x < state->draw_width; ++x) { + uint16_t color = state->framebuffer[(size_t)y * state->draw_width + x]; + int red = (color >> 11) & 0x1F; + int green = (color >> 5) & 0x3F; + int blue = color & 0x1F; + int luminance = red * 2 + green * 3 + blue; + if (luminance < 128) { + row[x >> 3] |= (0x80 >> (x & 7)); + } + } + } + success = true; + } + tt_lvgl_unlock(); + } + + char* encoded = success ? base64_encode(pbm, pbm_size) : NULL; + free(pbm); + return encoded; +} diff --git a/Apps/McpScreen/main/Source/McpScreen.c b/Apps/McpScreen/main/Source/McpScreen.c index 741cae4..b84871f 100644 --- a/Apps/McpScreen/main/Source/McpScreen.c +++ b/Apps/McpScreen/main/Source/McpScreen.c @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -44,7 +45,7 @@ void mcp_ui_set_last_tool(McpScreenState* state, const char* tool) { } bool mcp_ui_clear(McpScreenState* state, int color) { - if (state == NULL || !state->visible) { + if (state == NULL || !state->visible || state->framebuffer == NULL) { return false; } @@ -52,12 +53,12 @@ bool mcp_ui_clear(McpScreenState* state, int color) { if (tt_lvgl_lock(pdMS_TO_TICKS(1500))) { if (state->visible && state->draw_area != NULL && lv_obj_is_valid(state->draw_area)) { lv_obj_clean(state->draw_area); - lv_obj_set_style_bg_color( - state->draw_area, - color == 1 ? lv_color_black() : lv_color_white(), - LV_PART_MAIN - ); - lv_obj_set_style_bg_opa(state->draw_area, LV_OPA_COVER, LV_PART_MAIN); + uint16_t fill = color == 1 ? 0x0000 : 0xFFFF; + size_t pixel_count = (size_t)state->draw_width * state->draw_height; + for (size_t i = 0; i < pixel_count; ++i) { + state->framebuffer[i] = fill; + } + lv_obj_invalidate(state->draw_area); state->draw_color = color; state->override_active = true; success = true; @@ -127,15 +128,16 @@ static void* create_data(void) { } static void destroy_data(void* data) { - free(data); + McpScreenState* state = data; + if (state != NULL && state->framebuffer != NULL) { + heap_caps_free(state->framebuffer); + } + free(state); } static void on_create(AppHandle app, void* data) { McpScreenState* state = data; state->app = app; - if (!mcp_services_start(state)) { - ESP_LOGE(TAG, "One or more MCP services failed to start"); - } } static void on_destroy(AppHandle app, void* data) { @@ -145,6 +147,7 @@ static void on_destroy(AppHandle app, void* data) { static void on_show(AppHandle app, void* data, lv_obj_t* parent) { McpScreenState* state = data; + ESP_LOGI(TAG, "onShow: starting foreground UI and MCP service"); state->visible = true; lv_obj_remove_flag(parent, LV_OBJ_FLAG_SCROLLABLE); @@ -156,17 +159,27 @@ static void on_show(AppHandle app, void* data, lv_obj_t* parent) { lv_obj_align(toolbar, LV_ALIGN_TOP_MID, 0, 0); state->server_label = lv_label_create(toolbar); - lv_label_set_text(state->server_label, state->http_ready ? "MCP :80 ready" : "MCP starting"); + lv_label_set_text(state->server_label, "MCP starting"); lv_obj_set_style_text_color( state->server_label, - state->http_ready ? lv_palette_main(LV_PALETTE_GREEN) : lv_palette_main(LV_PALETTE_ORANGE), + lv_palette_main(LV_PALETTE_ORANGE), LV_PART_MAIN ); state->tool_label = lv_label_create(toolbar); lv_label_set_text(state->tool_label, "Last: none"); - state->draw_area = lv_obj_create(parent); + if (!mcp_services_start(state)) { + ESP_LOGE(TAG, "Failed to start foreground MCP service"); + lv_label_set_text(state->server_label, "MCP start failed"); + lv_obj_set_style_text_color( + state->server_label, + lv_palette_main(LV_PALETTE_RED), + LV_PART_MAIN + ); + } + + state->draw_area = lv_canvas_create(parent); lv_obj_set_width(state->draw_area, LV_PCT(100)); lv_obj_set_flex_grow(state->draw_area, 1); lv_obj_set_style_radius(state->draw_area, 0, LV_PART_MAIN); @@ -182,9 +195,39 @@ static void on_show(AppHandle app, void* data, lv_obj_t* parent) { state->draw_width = lv_obj_get_content_width(state->draw_area); state->draw_height = lv_obj_get_content_height(state->draw_area); + size_t required_size = (size_t)state->draw_width * state->draw_height * sizeof(uint16_t); + if (state->framebuffer == NULL || state->framebuffer_size != required_size) { + if (state->framebuffer != NULL) { + heap_caps_free(state->framebuffer); + state->framebuffer = NULL; + } + state->framebuffer = heap_caps_malloc(required_size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (state->framebuffer == NULL) { + state->framebuffer = heap_caps_malloc(required_size, MALLOC_CAP_8BIT); + } + state->framebuffer_size = state->framebuffer == NULL ? 0 : required_size; + } + + if (state->framebuffer == NULL) { + ESP_LOGE(TAG, "Failed to allocate %u-byte framebuffer", (unsigned)required_size); + lv_obj_t* error = lv_label_create(state->draw_area); + lv_label_set_text(error, "Framebuffer allocation failed"); + lv_obj_center(error); + return; + } + + lv_canvas_set_buffer( + state->draw_area, + state->framebuffer, + state->draw_width, + state->draw_height, + LV_COLOR_FORMAT_RGB565 + ); + if (!state->override_active) { - lv_obj_set_style_bg_color(state->draw_area, lv_color_hex(0x101820), LV_PART_MAIN); - lv_obj_set_style_bg_opa(state->draw_area, LV_OPA_COVER, LV_PART_MAIN); + for (size_t i = 0; i < (size_t)state->draw_width * state->draw_height; ++i) { + state->framebuffer[i] = 0x10C3; + } state->draw_color = 1; lv_obj_t* title = lv_label_create(state->draw_area); @@ -215,12 +258,15 @@ static void on_show(AppHandle app, void* data, lv_obj_t* parent) { lv_obj_set_style_text_color(wifi, lv_color_white(), LV_PART_MAIN); lv_obj_align(wifi, LV_ALIGN_CENTER, 0, 48); } + } static void on_hide(AppHandle app, void* data) { (void)app; McpScreenState* state = data; + ESP_LOGI(TAG, "onHide: stopping foreground MCP service"); state->visible = false; + mcp_services_hide(state); state->draw_area = NULL; state->server_label = NULL; state->tool_label = NULL; diff --git a/Apps/McpScreen/main/Source/McpScreen.h b/Apps/McpScreen/main/Source/McpScreen.h index 9ee5ff2..9c17414 100644 --- a/Apps/McpScreen/main/Source/McpScreen.h +++ b/Apps/McpScreen/main/Source/McpScreen.h @@ -27,6 +27,8 @@ typedef struct { lv_obj_t* server_label; lv_obj_t* tool_label; + uint16_t* framebuffer; + size_t framebuffer_size; uint16_t display_width; uint16_t display_height; uint16_t draw_width; @@ -35,6 +37,7 @@ typedef struct { } McpScreenState; bool mcp_services_start(McpScreenState* state); +void mcp_services_hide(McpScreenState* state); void mcp_services_stop(McpScreenState* state); void mcp_ui_set_server_status(McpScreenState* state, const char* status); @@ -47,3 +50,27 @@ bool mcp_ui_draw_text( int y, int size ); +bool mcp_ui_draw_rgb565_be( + McpScreenState* state, + const uint8_t* data, + size_t data_size, + int x, + int y, + int width, + int height +); +bool mcp_ui_draw_bmp( + McpScreenState* state, + const uint8_t* data, + size_t data_size, + int x, + int y +); +bool mcp_ui_draw_pbm( + McpScreenState* state, + const uint8_t* data, + size_t data_size, + int x, + int y +); +char* mcp_ui_get_screenshot_pbm_base64(McpScreenState* state); diff --git a/Apps/McpScreen/main/Source/McpServer.c b/Apps/McpScreen/main/Source/McpServer.c index 352dc83..344d448 100644 --- a/Apps/McpScreen/main/Source/McpServer.c +++ b/Apps/McpScreen/main/Source/McpServer.c @@ -15,8 +15,9 @@ #define MCP_HTTP_PORT 80 #define MCP_HEADER_LIMIT 2048 -#define MCP_BODY_LIMIT 8192 -#define MCP_RESPONSE_LIMIT 8192 +#define MCP_JSON_BODY_LIMIT (256 * 1024) +#define MCP_RAW_BODY_LIMIT (512 * 1024) +#define MCP_RESPONSE_LIMIT (64 * 1024) static const char* TAG = "McpServer"; @@ -49,9 +50,110 @@ static const char* TOOLS_JSON = "}," "\"required\":[\"text\",\"x\",\"y\"]" "}" + "}," + "{" + "\"name\":\"draw_image\"," + "\"description\":\"Draw a bridge-preprocessed binary PBM image.\"," + "\"inputSchema\":{" + "\"type\":\"object\"," + "\"properties\":{" + "\"pbm_base64\":{\"type\":\"string\"}," + "\"x\":{\"type\":\"integer\",\"default\":0}," + "\"y\":{\"type\":\"integer\",\"default\":0}," + "\"dither\":{\"type\":\"boolean\",\"default\":true}" + "}," + "\"required\":[\"pbm_base64\"]" + "}" + "}," + "{" + "\"name\":\"draw_color_bmp\"," + "\"description\":\"Draw an uncompressed 24-bit or 32-bit BMP image.\"," + "\"inputSchema\":{" + "\"type\":\"object\"," + "\"properties\":{" + "\"bmp_base64\":{\"type\":\"string\"}," + "\"x\":{\"type\":\"integer\",\"default\":0}," + "\"y\":{\"type\":\"integer\",\"default\":0}" + "}," + "\"required\":[\"bmp_base64\"]" + "}" + "}," + "{" + "\"name\":\"draw_raw_rgb565\"," + "\"description\":\"Draw big-endian raw RGB565 pixels.\"," + "\"inputSchema\":{" + "\"type\":\"object\"," + "\"properties\":{" + "\"rgb565_base64\":{\"type\":\"string\"}," + "\"x\":{\"type\":\"integer\"}," + "\"y\":{\"type\":\"integer\"}," + "\"w\":{\"type\":\"integer\"}," + "\"h\":{\"type\":\"integer\"}" + "}," + "\"required\":[\"rgb565_base64\",\"x\",\"y\",\"w\",\"h\"]" + "}" + "}," + "{" + "\"name\":\"get_screenshot\"," + "\"description\":\"Capture the MCP framebuffer as a PBM image.\"," + "\"inputSchema\":{\"type\":\"object\",\"properties\":{}}" "}" "]"; +static int base64_value(char character) { + if (character >= 'A' && character <= 'Z') return character - 'A'; + if (character >= 'a' && character <= 'z') return character - 'a' + 26; + if (character >= '0' && character <= '9') return character - '0' + 52; + if (character == '+') return 62; + if (character == '/') return 63; + return -1; +} + +static uint8_t* base64_decode(const char* input, size_t* output_size) { + if (input == NULL || output_size == NULL) { + return NULL; + } + + const char* payload = input; + const char* marker = strstr(input, ";base64,"); + if (marker != NULL) { + payload = marker + 8; + } + + size_t input_size = strlen(payload); + uint8_t* output = malloc((input_size / 4) * 3 + 3); + if (output == NULL) { + return NULL; + } + + uint32_t accumulator = 0; + int bits = 0; + size_t written = 0; + for (size_t index = 0; index < input_size; ++index) { + char character = payload[index]; + if (character == '=') { + break; + } + int value = base64_value(character); + if (value < 0) { + if (character == '\r' || character == '\n' || + character == ' ' || character == '\t') { + continue; + } + free(output); + return NULL; + } + accumulator = (accumulator << 6) | (uint32_t)value; + bits += 6; + if (bits >= 8) { + bits -= 8; + output[written++] = (uint8_t)((accumulator >> bits) & 0xFF); + } + } + *output_size = written; + return output; +} + static void close_socket(int* socket_fd) { if (socket_fd != NULL && *socket_fd >= 0) { close(*socket_fd); @@ -125,15 +227,17 @@ static cJSON* handle_tool_call(McpScreenState* state, cJSON* id, cJSON* params) if (strcmp(name, "get_capabilities") == 0) { char capabilities[320]; + uint16_t width = state->draw_width > 0 ? state->draw_width : state->display_width; + uint16_t height = state->draw_height > 0 ? state->draw_height : state->display_height; snprintf( capabilities, sizeof(capabilities), "{\"color\":true,\"width\":%u,\"height\":%u," - "\"formats\":[\"rgb565_base64\",\"bmp_base64\"]," + "\"formats\":[\"rgb565_base64\",\"bmp_base64\",\"pbm_base64\"]," "\"platform\":\"tactility\",\"appVersion\":\"0.1.0\"," "\"uiVisible\":%s}", - state->display_width, - state->display_height, + width, + height, state->visible ? "true" : "false" ); return make_tool_result(id, capabilities); @@ -191,6 +295,110 @@ static cJSON* handle_tool_call(McpScreenState* state, cJSON* id, cJSON* params) return make_tool_result(id, message); } + if (strcmp(name, "draw_raw_rgb565") == 0) { + cJSON* encoded_item = cJSON_GetObjectItemCaseSensitive(arguments, "rgb565_base64"); + cJSON* x_item = cJSON_GetObjectItemCaseSensitive(arguments, "x"); + cJSON* y_item = cJSON_GetObjectItemCaseSensitive(arguments, "y"); + cJSON* width_item = cJSON_GetObjectItemCaseSensitive(arguments, "w"); + cJSON* height_item = cJSON_GetObjectItemCaseSensitive(arguments, "h"); + if (!cJSON_IsString(encoded_item) || !cJSON_IsNumber(x_item) || + !cJSON_IsNumber(y_item) || !cJSON_IsNumber(width_item) || + !cJSON_IsNumber(height_item)) { + return make_error(id, -32602, "draw_raw_rgb565 requires rgb565_base64, x, y, w, and h"); + } + + size_t decoded_size = 0; + uint8_t* decoded = base64_decode(encoded_item->valuestring, &decoded_size); + if (decoded == NULL) { + return make_error(id, -32602, "Invalid RGB565 base64 data"); + } + bool drawn = mcp_ui_draw_rgb565_be( + state, + decoded, + decoded_size, + x_item->valueint, + y_item->valueint, + width_item->valueint, + height_item->valueint + ); + free(decoded); + return drawn + ? make_tool_result(id, "Raw RGB565 data displayed successfully.") + : make_error(id, -32010, "Failed to draw RGB565 data"); + } + + if (strcmp(name, "draw_color_bmp") == 0) { + cJSON* encoded_item = cJSON_GetObjectItemCaseSensitive(arguments, "bmp_base64"); + cJSON* x_item = cJSON_GetObjectItemCaseSensitive(arguments, "x"); + cJSON* y_item = cJSON_GetObjectItemCaseSensitive(arguments, "y"); + if (!cJSON_IsString(encoded_item)) { + return make_error(id, -32602, "Missing bmp_base64"); + } + size_t decoded_size = 0; + uint8_t* decoded = base64_decode(encoded_item->valuestring, &decoded_size); + if (decoded == NULL) { + return make_error(id, -32602, "Invalid BMP base64 data"); + } + bool drawn = mcp_ui_draw_bmp( + state, + decoded, + decoded_size, + cJSON_IsNumber(x_item) ? x_item->valueint : 0, + cJSON_IsNumber(y_item) ? y_item->valueint : 0 + ); + free(decoded); + return drawn + ? make_tool_result(id, "Color BMP image displayed successfully.") + : make_error(id, -32602, "Unsupported or invalid BMP image"); + } + + if (strcmp(name, "draw_image") == 0) { + cJSON* encoded_item = cJSON_GetObjectItemCaseSensitive(arguments, "pbm_base64"); + cJSON* x_item = cJSON_GetObjectItemCaseSensitive(arguments, "x"); + cJSON* y_item = cJSON_GetObjectItemCaseSensitive(arguments, "y"); + if (!cJSON_IsString(encoded_item)) { + return make_error( + id, + -32602, + "draw_image requires bridge-preprocessed pbm_base64" + ); + } + size_t decoded_size = 0; + uint8_t* decoded = base64_decode(encoded_item->valuestring, &decoded_size); + if (decoded == NULL) { + return make_error(id, -32602, "Invalid PBM base64 data"); + } + bool drawn = mcp_ui_draw_pbm( + state, + decoded, + decoded_size, + cJSON_IsNumber(x_item) ? x_item->valueint : 0, + cJSON_IsNumber(y_item) ? y_item->valueint : 0 + ); + free(decoded); + return drawn + ? make_tool_result(id, "Image displayed successfully.") + : make_error(id, -32602, "Unsupported or invalid PBM image"); + } + + if (strcmp(name, "get_screenshot") == 0) { + char* encoded = mcp_ui_get_screenshot_pbm_base64(state); + if (encoded == NULL) { + return make_error(id, -32010, "Screenshot capture failed"); + } + size_t result_size = strlen(encoded) + 17; + char* result = malloc(result_size); + if (result == NULL) { + free(encoded); + return make_error(id, -32603, "Out of memory"); + } + snprintf(result, result_size, "__PBM_BASE64__:%s", encoded); + cJSON* response = make_tool_result(id, result); + free(result); + free(encoded); + return response; + } + return make_error(id, -32602, "Unknown tool"); } @@ -289,94 +497,164 @@ static void send_http_response(int client, int status, const char* content_type, } } +static int query_integer(const char* path, const char* name, int default_value) { + const char* query = strchr(path, '?'); + if (query == NULL) { + return default_value; + } + query++; + size_t name_size = strlen(name); + while (*query != '\0') { + if (strncmp(query, name, name_size) == 0 && query[name_size] == '=') { + return atoi(query + name_size + 1); + } + query = strchr(query, '&'); + if (query == NULL) { + break; + } + query++; + } + return default_value; +} + static void handle_http_client(McpScreenState* state, int client) { - struct timeval timeout = { .tv_sec = 0, .tv_usec = 500000 }; + struct timeval timeout = { .tv_sec = 0, .tv_usec = 250000 }; setsockopt(client, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); setsockopt(client, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout)); - char* request = calloc(1, MCP_HEADER_LIMIT + MCP_BODY_LIMIT + 1); - if (request == NULL) { + char* headers = calloc(1, MCP_HEADER_LIMIT + 1); + if (headers == NULL) { send_http_response(client, 400, "text/plain", "Out of memory"); return; } int received = 0; int header_end = -1; - while (received < MCP_HEADER_LIMIT) { - int count = recv(client, request + received, MCP_HEADER_LIMIT - received, 0); + while (state->running && received < MCP_HEADER_LIMIT) { + int count = recv(client, headers + received, MCP_HEADER_LIMIT - received, 0); if (count <= 0) { - free(request); + free(headers); return; } received += count; - header_end = find_header_end(request, received); + header_end = find_header_end(headers, received); if (header_end >= 0) { break; } } - if (header_end < 0) { - send_http_response(client, 400, "text/plain", "Invalid HTTP headers"); - free(request); + if (!state->running) { + free(headers); return; } - request[header_end - 4] = '\0'; + if (header_end < 0) { + send_http_response(client, 400, "text/plain", "Invalid HTTP headers"); + free(headers); + return; + } + + headers[header_end - 4] = '\0'; char method[12] = {0}; char path[128] = {0}; - if (sscanf(request, "%11s %127s", method, path) != 2) { + if (sscanf(headers, "%11s %127s", method, path) != 2) { send_http_response(client, 400, "text/plain", "Invalid request line"); - free(request); + free(headers); return; } if (strcmp(method, "POST") != 0) { send_http_response(client, 405, "text/plain", "POST required"); - free(request); - return; - } - if (strcmp(path, "/api/mcp") != 0) { - send_http_response(client, 404, "text/plain", "Not found"); - free(request); + free(headers); return; } - int content_length = parse_content_length(request); - if (content_length <= 0 || content_length > MCP_BODY_LIMIT) { + bool is_mcp = strcmp(path, "/api/mcp") == 0; + bool is_raw = strncmp(path, "/api/screen/raw", 15) == 0; + if (!is_mcp && !is_raw) { + send_http_response(client, 404, "text/plain", "Not found"); + free(headers); + return; + } + + int body_limit = is_raw ? MCP_RAW_BODY_LIMIT : MCP_JSON_BODY_LIMIT; + int content_length = parse_content_length(headers); + if (content_length <= 0 || content_length > body_limit) { send_http_response( client, - content_length > MCP_BODY_LIMIT ? 413 : 400, + content_length > body_limit ? 413 : 400, "text/plain", "Invalid Content-Length" ); - free(request); + free(headers); + return; + } + + uint8_t* body = malloc((size_t)content_length + (is_mcp ? 1 : 0)); + if (body == NULL) { + send_http_response(client, 400, "text/plain", "Out of memory"); + free(headers); return; } int body_received = received - header_end; - memmove(request, request + header_end, body_received); - while (body_received < content_length) { - int count = recv(client, request + body_received, content_length - body_received, 0); + if (body_received > content_length) { + body_received = content_length; + } + memcpy(body, headers + header_end, body_received); + free(headers); + + while (state->running && body_received < content_length) { + int count = recv(client, body + body_received, content_length - body_received, 0); if (count <= 0) { send_http_response(client, 400, "text/plain", "Incomplete request body"); - free(request); + free(body); return; } body_received += count; } - request[content_length] = '\0'; - cJSON* response = handle_rpc(state, request); - char* response_text = cJSON_PrintUnformatted(response); - if (response_text != NULL && strlen(response_text) <= MCP_RESPONSE_LIMIT) { - send_http_response(client, 200, "application/json", response_text); - } else { - send_http_response(client, 400, "text/plain", "Response serialization failed"); + if (!state->running || body_received < content_length) { + free(body); + return; } - cJSON_free(response_text); - cJSON_Delete(response); - free(request); + if (is_raw) { + int x = query_integer(path, "x", 0); + int y = query_integer(path, "y", 0); + int width = query_integer(path, "w", state->draw_width); + int height = query_integer(path, "h", state->draw_height); + bool valid_size = width > 0 && height > 0 && + (size_t)width * height * 2 == (size_t)content_length; + bool drawn = valid_size && mcp_ui_draw_rgb565_be( + state, + body, + content_length, + x, + y, + width, + height + ); + send_http_response( + client, + drawn ? 200 : 400, + "text/plain", + drawn ? "OK" : "Invalid RGB565 payload" + ); + } else { + body[content_length] = '\0'; + cJSON* response = handle_rpc(state, (const char*)body); + char* response_text = cJSON_PrintUnformatted(response); + if (response_text != NULL && strlen(response_text) <= MCP_RESPONSE_LIMIT) { + send_http_response(client, 200, "application/json", response_text); + } else { + send_http_response(client, 400, "text/plain", "Response serialization failed"); + } + cJSON_free(response_text); + cJSON_Delete(response); + } + + free(body); } static void http_task(void* argument) { @@ -455,11 +733,28 @@ suspend_for_owner: bool mcp_services_start(McpScreenState* state) { if (state == NULL || state->running) { + ESP_LOGE(TAG, "Service start rejected state=%p running=%d", state, state != NULL && state->running); return false; } + /* + * onHide only requests shutdown because it runs in Tactility's GUI + * lifecycle. Reap the worker after it has reached its safe suspended + * state before starting a new foreground server. + */ + if (state->http_task != NULL) { + if (!state->http_exited) { + ESP_LOGE(TAG, "Previous HTTP worker is still stopping"); + return false; + } + vTaskDelete(state->http_task); + state->http_task = NULL; + } + state->running = true; state->http_exited = false; + state->http_socket = -1; + state->client_socket = -1; BaseType_t http_result = xTaskCreate( http_task, "mcp_http", @@ -485,7 +780,7 @@ bool mcp_services_start(McpScreenState* state) { return true; } -void mcp_services_stop(McpScreenState* state) { +void mcp_services_hide(McpScreenState* state) { if (state == NULL) { return; } @@ -493,12 +788,17 @@ void mcp_services_stop(McpScreenState* state) { state->running = false; state->http_ready = false; state->discovery_ready = false; - close_socket(&state->http_socket); - close_socket(&state->client_socket); - close_socket(&state->discovery_socket); +} + +void mcp_services_stop(McpScreenState* state) { + if (state == NULL) { + return; + } + + mcp_services_hide(state); for (int attempt = 0; attempt < 80; ++attempt) { - if (state->http_exited) { + if (state->http_task == NULL || state->http_exited) { break; } vTaskDelay(pdMS_TO_TICKS(10)); @@ -506,11 +806,15 @@ void mcp_services_stop(McpScreenState* state) { if (state->http_task != NULL) { /* - * Normal path: worker is suspended after closing its sockets. - * Fallback path: force-delete after 800 ms so loader_dispatch can - * never be held until the task watchdog fires. + * Normal path: the non-blocking worker has closed its sockets and + * suspended itself. The bounded fallback guarantees that no + * external-ELF instruction survives onDestroy. */ vTaskDelete(state->http_task); state->http_task = NULL; } + close_socket(&state->http_socket); + close_socket(&state->client_socket); + close_socket(&state->discovery_socket); + state->http_exited = true; } diff --git a/Apps/McpScreen/tools/smoke_test.py b/Apps/McpScreen/tools/smoke_test.py index d6acc7d..d1ce843 100644 --- a/Apps/McpScreen/tools/smoke_test.py +++ b/Apps/McpScreen/tools/smoke_test.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 -"""Host smoke test for the Tactility MCP Screen Phase 1 API.""" +"""Host smoke test for the Tactility MCP Screen Phase 2 API.""" import argparse +import base64 import json import socket +import struct import urllib.request @@ -49,6 +51,53 @@ def require_result(response: dict, label: str) -> None: raise RuntimeError(f"{label} returned no result: {response}") +def make_bmp() -> bytes: + width = 4 + height = 4 + row_size = width * 3 + pixels = bytearray() + colors = [ + (0, 0, 255), + (0, 255, 0), + (255, 0, 0), + (255, 255, 255), + ] + for y in range(height): + for x in range(width): + blue, green, red = colors[(x + y) % len(colors)] + pixels.extend((blue, green, red)) + pixels.extend(b"\0" * ((4 - row_size % 4) % 4)) + + offset = 54 + file_size = offset + len(pixels) + header = ( + b"BM" + + struct.pack(" None: + pixels = bytes( + [ + 0xF8, 0x00, + 0x07, 0xE0, + 0x00, 0x1F, + 0xFF, 0xFF, + ] + ) + request = urllib.request.Request( + f"http://{ip}/api/screen/raw?x=2&y=2&w=2&h=2", + data=pixels, + headers={"Content-Type": "application/octet-stream"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=5) as response: + if response.read() != b"OK": + raise RuntimeError("Raw endpoint returned an unexpected response") + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--ip", help="Device IP; omit to use UDP discovery") @@ -106,9 +155,73 @@ def main() -> None: }, ) require_result(drawn, "draw_text") - print("Visible draw test sent.") - print("Phase 1 smoke test passed.") + raw_rpc = rpc( + ip, + 5, + "tools/call", + { + "name": "draw_raw_rgb565", + "arguments": { + "rgb565_base64": base64.b64encode( + bytes([0xFF, 0xE0, 0xF8, 0x1F, 0x07, 0xFF, 0x00, 0x00]) + ).decode(), + "x": 20, + "y": 70, + "w": 4, + "h": 1, + }, + }, + ) + require_result(raw_rpc, "draw_raw_rgb565") + + bmp = rpc( + ip, + 6, + "tools/call", + { + "name": "draw_color_bmp", + "arguments": { + "bmp_base64": base64.b64encode(make_bmp()).decode(), + "x": 30, + "y": 90, + }, + }, + ) + require_result(bmp, "draw_color_bmp") + + pbm_bytes = b"P4\n8 2\n" + bytes([0b10101010, 0b01010101]) + pbm = rpc( + ip, + 7, + "tools/call", + { + "name": "draw_image", + "arguments": { + "pbm_base64": base64.b64encode(pbm_bytes).decode(), + "x": 50, + "y": 110, + }, + }, + ) + require_result(pbm, "draw_image") + + raw_endpoint(ip) + + screenshot = rpc( + ip, + 8, + "tools/call", + {"name": "get_screenshot", "arguments": {}}, + ) + require_result(screenshot, "get_screenshot") + screenshot_text = screenshot["result"]["content"][0]["text"] + if not screenshot_text.startswith("__PBM_BASE64__:"): + raise RuntimeError("Screenshot did not return PBM data") + base64.b64decode(screenshot_text.split(":", 1)[1], validate=True) + print("Visible Phase 2 draw and screenshot tests sent.") + + print("Phase 2 smoke test passed.") if __name__ == "__main__":