feat: add McpScreen phase 2 display tools
Main / Build (Brainfuck) (push) Has been cancelled
Main / Build (Breakout) (push) Has been cancelled
Main / Build (Calculator) (push) Has been cancelled
Main / Build (Diceware) (push) Has been cancelled
Main / Build (EpubReader) (push) Has been cancelled
Main / Build (GPIO) (push) Has been cancelled
Main / Build (GraphicsDemo) (push) Has been cancelled
Main / Build (HelloWorld) (push) Has been cancelled
Main / Build (M5UnitTest) (push) Has been cancelled
Main / Build (Magic8Ball) (push) Has been cancelled
Main / Build (MediaKeys) (push) Has been cancelled
Main / Build (MystifyDemo) (push) Has been cancelled
Main / Build (SerialConsole) (push) Has been cancelled
Main / Build (Snake) (push) Has been cancelled
Main / Build (TamaTac) (push) Has been cancelled
Main / Build (TodoList) (push) Has been cancelled
Main / Build (TwoEleven) (push) Has been cancelled
Main / Bundle (push) Has been cancelled

This commit is contained in:
Adolfo Reyna
2026-06-25 12:43:08 -04:00
parent ac9d6dc0d3
commit 0fb3c22f79
7 changed files with 928 additions and 83 deletions
+337
View File
@@ -0,0 +1,337 @@
#include "McpScreen.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <esp_log.h>
#include <tt_lvgl.h>
static const char* TAG = "DisplayTools";
static bool ascii_space(uint8_t value) {
return value == ' ' || value == '\t' || value == '\r' ||
value == '\n' || value == '\f' || value == '\v';
}
static bool ascii_digit(uint8_t value) {
return value >= '0' && value <= '9';
}
static uint16_t read_le16(const uint8_t* value) {
return (uint16_t)value[0] | ((uint16_t)value[1] << 8);
}
static uint32_t read_le32(const uint8_t* value) {
return (uint32_t)value[0] |
((uint32_t)value[1] << 8) |
((uint32_t)value[2] << 16) |
((uint32_t)value[3] << 24);
}
static bool display_ready(McpScreenState* state) {
return state != NULL &&
state->visible &&
state->draw_area != NULL &&
state->framebuffer != NULL &&
state->draw_width > 0 &&
state->draw_height > 0;
}
static uint16_t rgb888_to_rgb565(uint8_t red, uint8_t green, uint8_t blue) {
return (uint16_t)(((red & 0xF8) << 8) |
((green & 0xFC) << 3) |
(blue >> 3));
}
static void put_pixel(McpScreenState* state, int x, int y, uint16_t color) {
if (x >= 0 && y >= 0 && x < state->draw_width && y < state->draw_height) {
state->framebuffer[(size_t)y * state->draw_width + x] = color;
}
}
bool mcp_ui_draw_rgb565_be(
McpScreenState* state,
const uint8_t* data,
size_t data_size,
int x,
int y,
int width,
int height
) {
if (data == NULL || width <= 0 || height <= 0 ||
data_size < (size_t)width * height * 2 || !display_ready(state)) {
return false;
}
bool success = false;
if (tt_lvgl_lock(pdMS_TO_TICKS(2000))) {
if (display_ready(state)) {
for (int source_y = 0; source_y < height; ++source_y) {
int destination_y = y + source_y;
if (destination_y < 0 || destination_y >= state->draw_height) {
continue;
}
for (int source_x = 0; source_x < width; ++source_x) {
int destination_x = x + source_x;
if (destination_x < 0 || destination_x >= state->draw_width) {
continue;
}
size_t offset = ((size_t)source_y * width + source_x) * 2;
uint16_t color = ((uint16_t)data[offset] << 8) | data[offset + 1];
put_pixel(state, destination_x, destination_y, color);
}
}
lv_obj_invalidate(state->draw_area);
state->override_active = true;
success = true;
}
tt_lvgl_unlock();
}
return success;
}
bool mcp_ui_draw_bmp(
McpScreenState* state,
const uint8_t* data,
size_t data_size,
int x,
int y
) {
if (data == NULL || data_size < 54 || !display_ready(state) ||
data[0] != 'B' || data[1] != 'M') {
return false;
}
uint32_t pixel_offset = read_le32(data + 10);
uint32_t dib_size = read_le32(data + 14);
int32_t width = (int32_t)read_le32(data + 18);
int32_t signed_height = (int32_t)read_le32(data + 22);
uint16_t planes = read_le16(data + 26);
uint16_t bits_per_pixel = read_le16(data + 28);
uint32_t compression = read_le32(data + 30);
if (dib_size < 40 || width <= 0 || signed_height == 0 || planes != 1 ||
(bits_per_pixel != 24 && bits_per_pixel != 32) || compression != 0) {
return false;
}
int height = signed_height < 0 ? -signed_height : signed_height;
bool top_down = signed_height < 0;
size_t row_stride = (((size_t)width * bits_per_pixel + 31) / 32) * 4;
if (pixel_offset > data_size ||
row_stride > data_size ||
(size_t)height > (data_size - pixel_offset) / row_stride) {
return false;
}
bool success = false;
if (tt_lvgl_lock(pdMS_TO_TICKS(2500))) {
if (display_ready(state)) {
size_t bytes_per_pixel = bits_per_pixel / 8;
for (int source_y = 0; source_y < height; ++source_y) {
int file_y = top_down ? source_y : (height - 1 - source_y);
const uint8_t* row = data + pixel_offset + (size_t)file_y * row_stride;
for (int source_x = 0; source_x < width; ++source_x) {
const uint8_t* pixel = row + (size_t)source_x * bytes_per_pixel;
put_pixel(
state,
x + source_x,
y + source_y,
rgb888_to_rgb565(pixel[2], pixel[1], pixel[0])
);
}
}
lv_obj_invalidate(state->draw_area);
state->override_active = true;
success = true;
}
tt_lvgl_unlock();
}
return success;
}
static bool pbm_next_number(
const uint8_t* data,
size_t data_size,
size_t* offset,
int* result
) {
while (*offset < data_size) {
if (data[*offset] == '#') {
while (*offset < data_size && data[*offset] != '\n') {
(*offset)++;
}
} else if (ascii_space(data[*offset])) {
(*offset)++;
} else {
break;
}
}
if (*offset >= data_size || !ascii_digit(data[*offset])) {
return false;
}
int value = 0;
while (*offset < data_size && ascii_digit(data[*offset])) {
value = value * 10 + (data[*offset] - '0');
(*offset)++;
}
*result = value;
return true;
}
bool mcp_ui_draw_pbm(
McpScreenState* state,
const uint8_t* data,
size_t data_size,
int x,
int y
) {
if (data == NULL || data_size < 8 || !display_ready(state) ||
data[0] != 'P' || data[1] != '4') {
ESP_LOGE(
TAG,
"PBM precondition failed data=%p size=%u ready=%d magic=%02x%02x",
data,
(unsigned)data_size,
display_ready(state),
data_size > 0 ? data[0] : 0,
data_size > 1 ? data[1] : 0
);
return false;
}
size_t offset = 2;
int width = 0;
int height = 0;
if (!pbm_next_number(data, data_size, &offset, &width) ||
!pbm_next_number(data, data_size, &offset, &height) ||
width <= 0 || height <= 0) {
ESP_LOGE(TAG, "PBM dimension parse failed offset=%u w=%d h=%d", (unsigned)offset, width, height);
return false;
}
if (offset >= data_size || !ascii_space(data[offset])) {
ESP_LOGE(
TAG,
"PBM missing header separator offset=%u size=%u byte=%02x",
(unsigned)offset,
(unsigned)data_size,
offset < data_size ? data[offset] : 0
);
return false;
}
if (data[offset] == '\r' && offset + 1 < data_size && data[offset + 1] == '\n') {
offset += 2;
} else {
offset++;
}
size_t row_bytes = ((size_t)width + 7) / 8;
if (offset > data_size ||
(size_t)height > (data_size - offset) / row_bytes) {
ESP_LOGE(
TAG,
"PBM payload too small offset=%u size=%u row=%u h=%d",
(unsigned)offset,
(unsigned)data_size,
(unsigned)row_bytes,
height
);
return false;
}
bool success = false;
if (tt_lvgl_lock(pdMS_TO_TICKS(2000))) {
if (display_ready(state)) {
for (int source_y = 0; source_y < height; ++source_y) {
const uint8_t* row = data + offset + (size_t)source_y * row_bytes;
for (int source_x = 0; source_x < width; ++source_x) {
bool black = (row[source_x >> 3] & (0x80 >> (source_x & 7))) != 0;
put_pixel(state, x + source_x, y + source_y, black ? 0x0000 : 0xFFFF);
}
}
lv_obj_invalidate(state->draw_area);
state->override_active = true;
success = true;
}
tt_lvgl_unlock();
} else {
ESP_LOGE(TAG, "PBM LVGL lock timed out");
}
if (!success) ESP_LOGE(TAG, "PBM draw failed after lock ready=%d", display_ready(state));
return success;
}
static char* base64_encode(const uint8_t* input, size_t input_size) {
static const char alphabet[] =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
size_t output_size = ((input_size + 2) / 3) * 4;
char* output = malloc(output_size + 1);
if (output == NULL) {
return NULL;
}
size_t in = 0;
size_t out = 0;
while (in < input_size) {
uint32_t value = (uint32_t)input[in++] << 16;
bool have_second = in < input_size;
if (have_second) value |= (uint32_t)input[in++] << 8;
bool have_third = in < input_size;
if (have_third) value |= input[in++];
output[out++] = alphabet[(value >> 18) & 0x3F];
output[out++] = alphabet[(value >> 12) & 0x3F];
output[out++] = have_second ? alphabet[(value >> 6) & 0x3F] : '=';
output[out++] = have_third ? alphabet[value & 0x3F] : '=';
}
output[out] = '\0';
return output;
}
char* mcp_ui_get_screenshot_pbm_base64(McpScreenState* state) {
if (!display_ready(state)) {
return NULL;
}
size_t row_bytes = ((size_t)state->draw_width + 7) / 8;
char header[40];
int header_size = snprintf(
header,
sizeof(header),
"P4\n%u %u\n",
state->draw_width,
state->draw_height
);
size_t pbm_size = (size_t)header_size + row_bytes * state->draw_height;
uint8_t* pbm = calloc(1, pbm_size);
if (pbm == NULL) {
return NULL;
}
memcpy(pbm, header, header_size);
bool success = false;
if (tt_lvgl_lock(pdMS_TO_TICKS(2000))) {
if (display_ready(state)) {
for (int y = 0; y < state->draw_height; ++y) {
uint8_t* row = pbm + header_size + (size_t)y * row_bytes;
for (int x = 0; x < state->draw_width; ++x) {
uint16_t color = state->framebuffer[(size_t)y * state->draw_width + x];
int red = (color >> 11) & 0x1F;
int green = (color >> 5) & 0x3F;
int blue = color & 0x1F;
int luminance = red * 2 + green * 3 + blue;
if (luminance < 128) {
row[x >> 3] |= (0x80 >> (x & 7));
}
}
}
success = true;
}
tt_lvgl_unlock();
}
char* encoded = success ? base64_encode(pbm, pbm_size) : NULL;
free(pbm);
return encoded;
}
+62 -16
View File
@@ -5,6 +5,7 @@
#include <string.h>
#include <esp_log.h>
#include <esp_heap_caps.h>
#include <tt_lvgl.h>
#include <tt_lvgl_toolbar.h>
#include <tt_wifi.h>
@@ -44,7 +45,7 @@ void mcp_ui_set_last_tool(McpScreenState* state, const char* tool) {
}
bool mcp_ui_clear(McpScreenState* state, int color) {
if (state == NULL || !state->visible) {
if (state == NULL || !state->visible || state->framebuffer == NULL) {
return false;
}
@@ -52,12 +53,12 @@ bool mcp_ui_clear(McpScreenState* state, int color) {
if (tt_lvgl_lock(pdMS_TO_TICKS(1500))) {
if (state->visible && state->draw_area != NULL && lv_obj_is_valid(state->draw_area)) {
lv_obj_clean(state->draw_area);
lv_obj_set_style_bg_color(
state->draw_area,
color == 1 ? lv_color_black() : lv_color_white(),
LV_PART_MAIN
);
lv_obj_set_style_bg_opa(state->draw_area, LV_OPA_COVER, LV_PART_MAIN);
uint16_t fill = color == 1 ? 0x0000 : 0xFFFF;
size_t pixel_count = (size_t)state->draw_width * state->draw_height;
for (size_t i = 0; i < pixel_count; ++i) {
state->framebuffer[i] = fill;
}
lv_obj_invalidate(state->draw_area);
state->draw_color = color;
state->override_active = true;
success = true;
@@ -127,15 +128,16 @@ static void* create_data(void) {
}
static void destroy_data(void* data) {
free(data);
McpScreenState* state = data;
if (state != NULL && state->framebuffer != NULL) {
heap_caps_free(state->framebuffer);
}
free(state);
}
static void on_create(AppHandle app, void* data) {
McpScreenState* state = data;
state->app = app;
if (!mcp_services_start(state)) {
ESP_LOGE(TAG, "One or more MCP services failed to start");
}
}
static void on_destroy(AppHandle app, void* data) {
@@ -145,6 +147,7 @@ static void on_destroy(AppHandle app, void* data) {
static void on_show(AppHandle app, void* data, lv_obj_t* parent) {
McpScreenState* state = data;
ESP_LOGI(TAG, "onShow: starting foreground UI and MCP service");
state->visible = true;
lv_obj_remove_flag(parent, LV_OBJ_FLAG_SCROLLABLE);
@@ -156,17 +159,27 @@ static void on_show(AppHandle app, void* data, lv_obj_t* parent) {
lv_obj_align(toolbar, LV_ALIGN_TOP_MID, 0, 0);
state->server_label = lv_label_create(toolbar);
lv_label_set_text(state->server_label, state->http_ready ? "MCP :80 ready" : "MCP starting");
lv_label_set_text(state->server_label, "MCP starting");
lv_obj_set_style_text_color(
state->server_label,
state->http_ready ? lv_palette_main(LV_PALETTE_GREEN) : lv_palette_main(LV_PALETTE_ORANGE),
lv_palette_main(LV_PALETTE_ORANGE),
LV_PART_MAIN
);
state->tool_label = lv_label_create(toolbar);
lv_label_set_text(state->tool_label, "Last: none");
state->draw_area = lv_obj_create(parent);
if (!mcp_services_start(state)) {
ESP_LOGE(TAG, "Failed to start foreground MCP service");
lv_label_set_text(state->server_label, "MCP start failed");
lv_obj_set_style_text_color(
state->server_label,
lv_palette_main(LV_PALETTE_RED),
LV_PART_MAIN
);
}
state->draw_area = lv_canvas_create(parent);
lv_obj_set_width(state->draw_area, LV_PCT(100));
lv_obj_set_flex_grow(state->draw_area, 1);
lv_obj_set_style_radius(state->draw_area, 0, LV_PART_MAIN);
@@ -182,9 +195,39 @@ static void on_show(AppHandle app, void* data, lv_obj_t* parent) {
state->draw_width = lv_obj_get_content_width(state->draw_area);
state->draw_height = lv_obj_get_content_height(state->draw_area);
size_t required_size = (size_t)state->draw_width * state->draw_height * sizeof(uint16_t);
if (state->framebuffer == NULL || state->framebuffer_size != required_size) {
if (state->framebuffer != NULL) {
heap_caps_free(state->framebuffer);
state->framebuffer = NULL;
}
state->framebuffer = heap_caps_malloc(required_size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
if (state->framebuffer == NULL) {
state->framebuffer = heap_caps_malloc(required_size, MALLOC_CAP_8BIT);
}
state->framebuffer_size = state->framebuffer == NULL ? 0 : required_size;
}
if (state->framebuffer == NULL) {
ESP_LOGE(TAG, "Failed to allocate %u-byte framebuffer", (unsigned)required_size);
lv_obj_t* error = lv_label_create(state->draw_area);
lv_label_set_text(error, "Framebuffer allocation failed");
lv_obj_center(error);
return;
}
lv_canvas_set_buffer(
state->draw_area,
state->framebuffer,
state->draw_width,
state->draw_height,
LV_COLOR_FORMAT_RGB565
);
if (!state->override_active) {
lv_obj_set_style_bg_color(state->draw_area, lv_color_hex(0x101820), LV_PART_MAIN);
lv_obj_set_style_bg_opa(state->draw_area, LV_OPA_COVER, LV_PART_MAIN);
for (size_t i = 0; i < (size_t)state->draw_width * state->draw_height; ++i) {
state->framebuffer[i] = 0x10C3;
}
state->draw_color = 1;
lv_obj_t* title = lv_label_create(state->draw_area);
@@ -215,12 +258,15 @@ static void on_show(AppHandle app, void* data, lv_obj_t* parent) {
lv_obj_set_style_text_color(wifi, lv_color_white(), LV_PART_MAIN);
lv_obj_align(wifi, LV_ALIGN_CENTER, 0, 48);
}
}
static void on_hide(AppHandle app, void* data) {
(void)app;
McpScreenState* state = data;
ESP_LOGI(TAG, "onHide: stopping foreground MCP service");
state->visible = false;
mcp_services_hide(state);
state->draw_area = NULL;
state->server_label = NULL;
state->tool_label = NULL;
+27
View File
@@ -27,6 +27,8 @@ typedef struct {
lv_obj_t* server_label;
lv_obj_t* tool_label;
uint16_t* framebuffer;
size_t framebuffer_size;
uint16_t display_width;
uint16_t display_height;
uint16_t draw_width;
@@ -35,6 +37,7 @@ typedef struct {
} McpScreenState;
bool mcp_services_start(McpScreenState* state);
void mcp_services_hide(McpScreenState* state);
void mcp_services_stop(McpScreenState* state);
void mcp_ui_set_server_status(McpScreenState* state, const char* status);
@@ -47,3 +50,27 @@ bool mcp_ui_draw_text(
int y,
int size
);
bool mcp_ui_draw_rgb565_be(
McpScreenState* state,
const uint8_t* data,
size_t data_size,
int x,
int y,
int width,
int height
);
bool mcp_ui_draw_bmp(
McpScreenState* state,
const uint8_t* data,
size_t data_size,
int x,
int y
);
bool mcp_ui_draw_pbm(
McpScreenState* state,
const uint8_t* data,
size_t data_size,
int x,
int y
);
char* mcp_ui_get_screenshot_pbm_base64(McpScreenState* state);
+354 -50
View File
@@ -15,8 +15,9 @@
#define MCP_HTTP_PORT 80
#define MCP_HEADER_LIMIT 2048
#define MCP_BODY_LIMIT 8192
#define MCP_RESPONSE_LIMIT 8192
#define MCP_JSON_BODY_LIMIT (256 * 1024)
#define MCP_RAW_BODY_LIMIT (512 * 1024)
#define MCP_RESPONSE_LIMIT (64 * 1024)
static const char* TAG = "McpServer";
@@ -49,9 +50,110 @@ static const char* TOOLS_JSON =
"},"
"\"required\":[\"text\",\"x\",\"y\"]"
"}"
"},"
"{"
"\"name\":\"draw_image\","
"\"description\":\"Draw a bridge-preprocessed binary PBM image.\","
"\"inputSchema\":{"
"\"type\":\"object\","
"\"properties\":{"
"\"pbm_base64\":{\"type\":\"string\"},"
"\"x\":{\"type\":\"integer\",\"default\":0},"
"\"y\":{\"type\":\"integer\",\"default\":0},"
"\"dither\":{\"type\":\"boolean\",\"default\":true}"
"},"
"\"required\":[\"pbm_base64\"]"
"}"
"},"
"{"
"\"name\":\"draw_color_bmp\","
"\"description\":\"Draw an uncompressed 24-bit or 32-bit BMP image.\","
"\"inputSchema\":{"
"\"type\":\"object\","
"\"properties\":{"
"\"bmp_base64\":{\"type\":\"string\"},"
"\"x\":{\"type\":\"integer\",\"default\":0},"
"\"y\":{\"type\":\"integer\",\"default\":0}"
"},"
"\"required\":[\"bmp_base64\"]"
"}"
"},"
"{"
"\"name\":\"draw_raw_rgb565\","
"\"description\":\"Draw big-endian raw RGB565 pixels.\","
"\"inputSchema\":{"
"\"type\":\"object\","
"\"properties\":{"
"\"rgb565_base64\":{\"type\":\"string\"},"
"\"x\":{\"type\":\"integer\"},"
"\"y\":{\"type\":\"integer\"},"
"\"w\":{\"type\":\"integer\"},"
"\"h\":{\"type\":\"integer\"}"
"},"
"\"required\":[\"rgb565_base64\",\"x\",\"y\",\"w\",\"h\"]"
"}"
"},"
"{"
"\"name\":\"get_screenshot\","
"\"description\":\"Capture the MCP framebuffer as a PBM image.\","
"\"inputSchema\":{\"type\":\"object\",\"properties\":{}}"
"}"
"]";
static int base64_value(char character) {
if (character >= 'A' && character <= 'Z') return character - 'A';
if (character >= 'a' && character <= 'z') return character - 'a' + 26;
if (character >= '0' && character <= '9') return character - '0' + 52;
if (character == '+') return 62;
if (character == '/') return 63;
return -1;
}
static uint8_t* base64_decode(const char* input, size_t* output_size) {
if (input == NULL || output_size == NULL) {
return NULL;
}
const char* payload = input;
const char* marker = strstr(input, ";base64,");
if (marker != NULL) {
payload = marker + 8;
}
size_t input_size = strlen(payload);
uint8_t* output = malloc((input_size / 4) * 3 + 3);
if (output == NULL) {
return NULL;
}
uint32_t accumulator = 0;
int bits = 0;
size_t written = 0;
for (size_t index = 0; index < input_size; ++index) {
char character = payload[index];
if (character == '=') {
break;
}
int value = base64_value(character);
if (value < 0) {
if (character == '\r' || character == '\n' ||
character == ' ' || character == '\t') {
continue;
}
free(output);
return NULL;
}
accumulator = (accumulator << 6) | (uint32_t)value;
bits += 6;
if (bits >= 8) {
bits -= 8;
output[written++] = (uint8_t)((accumulator >> bits) & 0xFF);
}
}
*output_size = written;
return output;
}
static void close_socket(int* socket_fd) {
if (socket_fd != NULL && *socket_fd >= 0) {
close(*socket_fd);
@@ -125,15 +227,17 @@ static cJSON* handle_tool_call(McpScreenState* state, cJSON* id, cJSON* params)
if (strcmp(name, "get_capabilities") == 0) {
char capabilities[320];
uint16_t width = state->draw_width > 0 ? state->draw_width : state->display_width;
uint16_t height = state->draw_height > 0 ? state->draw_height : state->display_height;
snprintf(
capabilities,
sizeof(capabilities),
"{\"color\":true,\"width\":%u,\"height\":%u,"
"\"formats\":[\"rgb565_base64\",\"bmp_base64\"],"
"\"formats\":[\"rgb565_base64\",\"bmp_base64\",\"pbm_base64\"],"
"\"platform\":\"tactility\",\"appVersion\":\"0.1.0\","
"\"uiVisible\":%s}",
state->display_width,
state->display_height,
width,
height,
state->visible ? "true" : "false"
);
return make_tool_result(id, capabilities);
@@ -191,6 +295,110 @@ static cJSON* handle_tool_call(McpScreenState* state, cJSON* id, cJSON* params)
return make_tool_result(id, message);
}
if (strcmp(name, "draw_raw_rgb565") == 0) {
cJSON* encoded_item = cJSON_GetObjectItemCaseSensitive(arguments, "rgb565_base64");
cJSON* x_item = cJSON_GetObjectItemCaseSensitive(arguments, "x");
cJSON* y_item = cJSON_GetObjectItemCaseSensitive(arguments, "y");
cJSON* width_item = cJSON_GetObjectItemCaseSensitive(arguments, "w");
cJSON* height_item = cJSON_GetObjectItemCaseSensitive(arguments, "h");
if (!cJSON_IsString(encoded_item) || !cJSON_IsNumber(x_item) ||
!cJSON_IsNumber(y_item) || !cJSON_IsNumber(width_item) ||
!cJSON_IsNumber(height_item)) {
return make_error(id, -32602, "draw_raw_rgb565 requires rgb565_base64, x, y, w, and h");
}
size_t decoded_size = 0;
uint8_t* decoded = base64_decode(encoded_item->valuestring, &decoded_size);
if (decoded == NULL) {
return make_error(id, -32602, "Invalid RGB565 base64 data");
}
bool drawn = mcp_ui_draw_rgb565_be(
state,
decoded,
decoded_size,
x_item->valueint,
y_item->valueint,
width_item->valueint,
height_item->valueint
);
free(decoded);
return drawn
? make_tool_result(id, "Raw RGB565 data displayed successfully.")
: make_error(id, -32010, "Failed to draw RGB565 data");
}
if (strcmp(name, "draw_color_bmp") == 0) {
cJSON* encoded_item = cJSON_GetObjectItemCaseSensitive(arguments, "bmp_base64");
cJSON* x_item = cJSON_GetObjectItemCaseSensitive(arguments, "x");
cJSON* y_item = cJSON_GetObjectItemCaseSensitive(arguments, "y");
if (!cJSON_IsString(encoded_item)) {
return make_error(id, -32602, "Missing bmp_base64");
}
size_t decoded_size = 0;
uint8_t* decoded = base64_decode(encoded_item->valuestring, &decoded_size);
if (decoded == NULL) {
return make_error(id, -32602, "Invalid BMP base64 data");
}
bool drawn = mcp_ui_draw_bmp(
state,
decoded,
decoded_size,
cJSON_IsNumber(x_item) ? x_item->valueint : 0,
cJSON_IsNumber(y_item) ? y_item->valueint : 0
);
free(decoded);
return drawn
? make_tool_result(id, "Color BMP image displayed successfully.")
: make_error(id, -32602, "Unsupported or invalid BMP image");
}
if (strcmp(name, "draw_image") == 0) {
cJSON* encoded_item = cJSON_GetObjectItemCaseSensitive(arguments, "pbm_base64");
cJSON* x_item = cJSON_GetObjectItemCaseSensitive(arguments, "x");
cJSON* y_item = cJSON_GetObjectItemCaseSensitive(arguments, "y");
if (!cJSON_IsString(encoded_item)) {
return make_error(
id,
-32602,
"draw_image requires bridge-preprocessed pbm_base64"
);
}
size_t decoded_size = 0;
uint8_t* decoded = base64_decode(encoded_item->valuestring, &decoded_size);
if (decoded == NULL) {
return make_error(id, -32602, "Invalid PBM base64 data");
}
bool drawn = mcp_ui_draw_pbm(
state,
decoded,
decoded_size,
cJSON_IsNumber(x_item) ? x_item->valueint : 0,
cJSON_IsNumber(y_item) ? y_item->valueint : 0
);
free(decoded);
return drawn
? make_tool_result(id, "Image displayed successfully.")
: make_error(id, -32602, "Unsupported or invalid PBM image");
}
if (strcmp(name, "get_screenshot") == 0) {
char* encoded = mcp_ui_get_screenshot_pbm_base64(state);
if (encoded == NULL) {
return make_error(id, -32010, "Screenshot capture failed");
}
size_t result_size = strlen(encoded) + 17;
char* result = malloc(result_size);
if (result == NULL) {
free(encoded);
return make_error(id, -32603, "Out of memory");
}
snprintf(result, result_size, "__PBM_BASE64__:%s", encoded);
cJSON* response = make_tool_result(id, result);
free(result);
free(encoded);
return response;
}
return make_error(id, -32602, "Unknown tool");
}
@@ -289,94 +497,164 @@ static void send_http_response(int client, int status, const char* content_type,
}
}
static int query_integer(const char* path, const char* name, int default_value) {
const char* query = strchr(path, '?');
if (query == NULL) {
return default_value;
}
query++;
size_t name_size = strlen(name);
while (*query != '\0') {
if (strncmp(query, name, name_size) == 0 && query[name_size] == '=') {
return atoi(query + name_size + 1);
}
query = strchr(query, '&');
if (query == NULL) {
break;
}
query++;
}
return default_value;
}
static void handle_http_client(McpScreenState* state, int client) {
struct timeval timeout = { .tv_sec = 0, .tv_usec = 500000 };
struct timeval timeout = { .tv_sec = 0, .tv_usec = 250000 };
setsockopt(client, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout));
setsockopt(client, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout));
char* request = calloc(1, MCP_HEADER_LIMIT + MCP_BODY_LIMIT + 1);
if (request == NULL) {
char* headers = calloc(1, MCP_HEADER_LIMIT + 1);
if (headers == NULL) {
send_http_response(client, 400, "text/plain", "Out of memory");
return;
}
int received = 0;
int header_end = -1;
while (received < MCP_HEADER_LIMIT) {
int count = recv(client, request + received, MCP_HEADER_LIMIT - received, 0);
while (state->running && received < MCP_HEADER_LIMIT) {
int count = recv(client, headers + received, MCP_HEADER_LIMIT - received, 0);
if (count <= 0) {
free(request);
free(headers);
return;
}
received += count;
header_end = find_header_end(request, received);
header_end = find_header_end(headers, received);
if (header_end >= 0) {
break;
}
}
if (header_end < 0) {
send_http_response(client, 400, "text/plain", "Invalid HTTP headers");
free(request);
if (!state->running) {
free(headers);
return;
}
request[header_end - 4] = '\0';
if (header_end < 0) {
send_http_response(client, 400, "text/plain", "Invalid HTTP headers");
free(headers);
return;
}
headers[header_end - 4] = '\0';
char method[12] = {0};
char path[128] = {0};
if (sscanf(request, "%11s %127s", method, path) != 2) {
if (sscanf(headers, "%11s %127s", method, path) != 2) {
send_http_response(client, 400, "text/plain", "Invalid request line");
free(request);
free(headers);
return;
}
if (strcmp(method, "POST") != 0) {
send_http_response(client, 405, "text/plain", "POST required");
free(request);
return;
}
if (strcmp(path, "/api/mcp") != 0) {
send_http_response(client, 404, "text/plain", "Not found");
free(request);
free(headers);
return;
}
int content_length = parse_content_length(request);
if (content_length <= 0 || content_length > MCP_BODY_LIMIT) {
bool is_mcp = strcmp(path, "/api/mcp") == 0;
bool is_raw = strncmp(path, "/api/screen/raw", 15) == 0;
if (!is_mcp && !is_raw) {
send_http_response(client, 404, "text/plain", "Not found");
free(headers);
return;
}
int body_limit = is_raw ? MCP_RAW_BODY_LIMIT : MCP_JSON_BODY_LIMIT;
int content_length = parse_content_length(headers);
if (content_length <= 0 || content_length > body_limit) {
send_http_response(
client,
content_length > MCP_BODY_LIMIT ? 413 : 400,
content_length > body_limit ? 413 : 400,
"text/plain",
"Invalid Content-Length"
);
free(request);
free(headers);
return;
}
uint8_t* body = malloc((size_t)content_length + (is_mcp ? 1 : 0));
if (body == NULL) {
send_http_response(client, 400, "text/plain", "Out of memory");
free(headers);
return;
}
int body_received = received - header_end;
memmove(request, request + header_end, body_received);
while (body_received < content_length) {
int count = recv(client, request + body_received, content_length - body_received, 0);
if (body_received > content_length) {
body_received = content_length;
}
memcpy(body, headers + header_end, body_received);
free(headers);
while (state->running && body_received < content_length) {
int count = recv(client, body + body_received, content_length - body_received, 0);
if (count <= 0) {
send_http_response(client, 400, "text/plain", "Incomplete request body");
free(request);
free(body);
return;
}
body_received += count;
}
request[content_length] = '\0';
cJSON* response = handle_rpc(state, request);
char* response_text = cJSON_PrintUnformatted(response);
if (response_text != NULL && strlen(response_text) <= MCP_RESPONSE_LIMIT) {
send_http_response(client, 200, "application/json", response_text);
} else {
send_http_response(client, 400, "text/plain", "Response serialization failed");
if (!state->running || body_received < content_length) {
free(body);
return;
}
cJSON_free(response_text);
cJSON_Delete(response);
free(request);
if (is_raw) {
int x = query_integer(path, "x", 0);
int y = query_integer(path, "y", 0);
int width = query_integer(path, "w", state->draw_width);
int height = query_integer(path, "h", state->draw_height);
bool valid_size = width > 0 && height > 0 &&
(size_t)width * height * 2 == (size_t)content_length;
bool drawn = valid_size && mcp_ui_draw_rgb565_be(
state,
body,
content_length,
x,
y,
width,
height
);
send_http_response(
client,
drawn ? 200 : 400,
"text/plain",
drawn ? "OK" : "Invalid RGB565 payload"
);
} else {
body[content_length] = '\0';
cJSON* response = handle_rpc(state, (const char*)body);
char* response_text = cJSON_PrintUnformatted(response);
if (response_text != NULL && strlen(response_text) <= MCP_RESPONSE_LIMIT) {
send_http_response(client, 200, "application/json", response_text);
} else {
send_http_response(client, 400, "text/plain", "Response serialization failed");
}
cJSON_free(response_text);
cJSON_Delete(response);
}
free(body);
}
static void http_task(void* argument) {
@@ -455,11 +733,28 @@ suspend_for_owner:
bool mcp_services_start(McpScreenState* state) {
if (state == NULL || state->running) {
ESP_LOGE(TAG, "Service start rejected state=%p running=%d", state, state != NULL && state->running);
return false;
}
/*
* onHide only requests shutdown because it runs in Tactility's GUI
* lifecycle. Reap the worker after it has reached its safe suspended
* state before starting a new foreground server.
*/
if (state->http_task != NULL) {
if (!state->http_exited) {
ESP_LOGE(TAG, "Previous HTTP worker is still stopping");
return false;
}
vTaskDelete(state->http_task);
state->http_task = NULL;
}
state->running = true;
state->http_exited = false;
state->http_socket = -1;
state->client_socket = -1;
BaseType_t http_result = xTaskCreate(
http_task,
"mcp_http",
@@ -485,7 +780,7 @@ bool mcp_services_start(McpScreenState* state) {
return true;
}
void mcp_services_stop(McpScreenState* state) {
void mcp_services_hide(McpScreenState* state) {
if (state == NULL) {
return;
}
@@ -493,12 +788,17 @@ void mcp_services_stop(McpScreenState* state) {
state->running = false;
state->http_ready = false;
state->discovery_ready = false;
close_socket(&state->http_socket);
close_socket(&state->client_socket);
close_socket(&state->discovery_socket);
}
void mcp_services_stop(McpScreenState* state) {
if (state == NULL) {
return;
}
mcp_services_hide(state);
for (int attempt = 0; attempt < 80; ++attempt) {
if (state->http_exited) {
if (state->http_task == NULL || state->http_exited) {
break;
}
vTaskDelay(pdMS_TO_TICKS(10));
@@ -506,11 +806,15 @@ void mcp_services_stop(McpScreenState* state) {
if (state->http_task != NULL) {
/*
* Normal path: worker is suspended after closing its sockets.
* Fallback path: force-delete after 800 ms so loader_dispatch can
* never be held until the task watchdog fires.
* Normal path: the non-blocking worker has closed its sockets and
* suspended itself. The bounded fallback guarantees that no
* external-ELF instruction survives onDestroy.
*/
vTaskDelete(state->http_task);
state->http_task = NULL;
}
close_socket(&state->http_socket);
close_socket(&state->client_socket);
close_socket(&state->discovery_socket);
state->http_exited = true;
}