adolforeyna/tactility
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add simulator MCP screen drawing tools

Browse Source
This commit is contained in:
Adolfo Reyna
2026-09-22 16:51:50 -04:00
parent 9630707b41
commit 982bba70b2
2 changed files with 81 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Tactility/Source/app/mcpsettings/McpSettingsSimulatorApp.cpp
+1 -1
View File
@@ -20,7 +20,7 @@ void createMcpInfo(lv_obj_t* parent, void* userData) {
"MCP is available while the simulator web server is running.\n\n"
"Endpoint: POST /api/mcp\n"
"JSON-RPC methods: tools/list, tools/call\n\n"
"Simulator tools: get_capabilities, list_apps, run_app\n\n"
"Simulator tools: get_capabilities, clear_screen, draw_text, list_apps, run_app\n\n"
"Open the simulator dashboard/API at http://127.0.0.1/api/sysinfo.");
lv_label_set_long_mode(info, LV_LABEL_LONG_WRAP);
lv_obj_set_width(info, LV_PCT(90));
Tactility/Source/service/webserver/McpSimulatorHandler.cpp
+80 -1
View File
@@ -1,6 +1,8 @@
#ifndef ESP_PLATFORM
#include <Tactility/service/webserver/WebServerService.h>
#include <Tactility/lvgl/Lvgl.h>
#include <lvgl/lvgl.h>
#include <app/manager.h>
#include <app/start.h>
#include <cJSON.h>
@@ -15,6 +17,44 @@
namespace tt::service::webserver {
namespace {
constexpr size_t JSON_BODY_LIMIT = 64 * 1024;
lv_obj_t* gCanvas = nullptr;
bool gCanvasBlack = false;
lv_obj_t* getCanvas() {
if (gCanvas != nullptr) return gCanvas;
lv_display_t* display = lv_display_get_default();
if (display == nullptr) return nullptr;
lv_obj_t* layer = lv_display_get_layer_top(display);
gCanvas = lv_obj_create(layer);
lv_obj_set_pos(gCanvas, 0, 0);
lv_obj_set_size(gCanvas, lv_display_get_horizontal_resolution(display), lv_display_get_vertical_resolution(display));
lv_obj_set_style_pad_all(gCanvas, 0, LV_PART_MAIN);
lv_obj_set_style_border_width(gCanvas, 0, LV_PART_MAIN);
lv_obj_set_style_bg_opa(gCanvas, LV_OPA_TRANSP, LV_PART_MAIN);
lv_obj_clear_flag(gCanvas, LV_OBJ_FLAG_SCROLLABLE);
lv_obj_clear_flag(gCanvas, LV_OBJ_FLAG_CLICKABLE);
return gCanvas;
}
bool clearCanvas(int color) {
lv_obj_t* canvas = getCanvas();
if (canvas == nullptr) return false;
lv_obj_clean(canvas);
gCanvasBlack = color == 1;
lv_obj_set_style_bg_color(canvas, gCanvasBlack ? lv_color_black() : lv_color_white(), LV_PART_MAIN);
lv_obj_set_style_bg_opa(canvas, LV_OPA_COVER, LV_PART_MAIN);
return true;
}
bool drawCanvasText(const char* text, int x, int y) {
lv_obj_t* canvas = getCanvas();
if (canvas == nullptr) return false;
lv_obj_t* label = lv_label_create(canvas);
lv_label_set_text(label, text);
lv_obj_set_pos(label, x, y);
lv_obj_set_style_text_color(label, gCanvasBlack ? lv_color_white() : lv_color_black(), LV_PART_MAIN);
return true;
}
cJSON* makeError(const cJSON* id, int code, const char* message) {
cJSON* response = cJSON_CreateObject();
@@ -66,6 +106,18 @@ cJSON* makeToolList(const cJSON* id) {
cJSON_AddItemToObject(runApp, "inputSchema", cJSON_Parse("{\"type\":\"object\",\"properties\":{\"app_id\":{\"type\":\"string\"}},\"required\":[\"app_id\"]}"));
cJSON_AddItemToArray(tools, runApp);
cJSON* clearScreen = cJSON_CreateObject();
cJSON_AddStringToObject(clearScreen, "name", "clear_screen");
cJSON_AddStringToObject(clearScreen, "description", "Clear the MCP drawing canvas to white (0) or black (1).");
cJSON_AddItemToObject(clearScreen, "inputSchema", cJSON_Parse("{\"type\":\"object\",\"properties\":{\"color\":{\"type\":\"integer\",\"enum\":[0,1]}},\"required\":[\"color\"]}"));
cJSON_AddItemToArray(tools, clearScreen);
cJSON* drawText = cJSON_CreateObject();
cJSON_AddStringToObject(drawText, "name", "draw_text");
cJSON_AddStringToObject(drawText, "description", "Draw text on the simulator MCP canvas at x,y coordinates.");
cJSON_AddItemToObject(drawText, "inputSchema", cJSON_Parse("{\"type\":\"object\",\"properties\":{\"text\":{\"type\":\"string\"},\"x\":{\"type\":\"integer\"},\"y\":{\"type\":\"integer\"}},\"required\":[\"text\",\"x\",\"y\"]}"));
cJSON_AddItemToArray(tools, drawText);
if (id != nullptr) cJSON_AddItemToObject(response, "id", cJSON_Duplicate(id, true));
else cJSON_AddNullToObject(response, "id");
return response;
@@ -98,8 +150,35 @@ cJSON* handleCall(const cJSON* id, const cJSON* params) {
return makeError(id, -32602, "Invalid params");
}
if (strcmp(name->valuestring, "clear_screen") == 0) {
const cJSON* color = cJSON_GetObjectItemCaseSensitive(args, "color");
if (!cJSON_IsNumber(color) || (color->valueint != 0 && color->valueint != 1)) {
return makeError(id, -32602, "color must be 0 (white) or 1 (black)");
}
lvgl_lock();
const bool cleared = clearCanvas(color->valueint);
lvgl_unlock();
return cleared ? makeTextResult(id, "ok") : makeTextResult(id, "no active display", true);
}
if (strcmp(name->valuestring, "draw_text") == 0) {
const cJSON* text = cJSON_GetObjectItemCaseSensitive(args, "text");
const cJSON* x = cJSON_GetObjectItemCaseSensitive(args, "x");
const cJSON* y = cJSON_GetObjectItemCaseSensitive(args, "y");
if (!cJSON_IsString(text) || text->valuestring == nullptr || strlen(text->valuestring) > 4096 ||
!cJSON_IsNumber(x) || !cJSON_IsNumber(y) || x->valueint < 0 || y->valueint < 0) {
return makeError(id, -32602, "text, non-negative x and y are required (text max 4096 chars)");
}
lvgl_lock();
lv_display_t* display = lv_display_get_default();
const bool inBounds = display != nullptr && x->valueint < lv_display_get_horizontal_resolution(display) &&
y->valueint < lv_display_get_vertical_resolution(display);
const bool drawn = inBounds && drawCanvasText(text->valuestring, x->valueint, y->valueint);
lvgl_unlock();
if (!inBounds) return makeError(id, -32602, "coordinates are outside the simulator display");
return drawn ? makeTextResult(id, "ok") : makeTextResult(id, "could not draw on display", true);
}
if (strcmp(name->valuestring, "get_capabilities") == 0) {
return makeTextResult(id, "{\"platform\":\"tactility-simulator\",\"mcp\":true}");
return makeTextResult(id, "{\"platform\":\"tactility-simulator\",\"mcp\":true,\"drawing\":true}");
}
if (strcmp(name->valuestring, "list_apps") == 0) {
const std::string apps = listAppsJson();