adolforeyna/tactility
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: integrate MCP server and screen override into system firmware

Browse Source 
- Add McpSettings (load/save mcpEnabled, mcpOverrideEnabled to /data/service/mcp/settings.properties)
- Add McpSystem: global state, canvas drawing (RGB565, BMP, PBM, text), I2S audio (WAV, MP3 via minimp3, tone, record)
- Add McpHandler: unauthenticated POST /api/mcp (JSON-RPC 2.0, 13 tools) and POST /api/screen/raw endpoints
- Route MCP endpoints before Basic Auth check in WebServerService::handleApiPost
- Start HTTP server at boot if either webServerEnabled OR mcpEnabled is set
- Fix setEnabled: start unconditionally when called with true; only stop if both WS and MCP are disabled
- Add McpSettingsApp (Settings category): toggle MCP service + screen override, show live endpoint URL
- Add McpOverrideApp (appId one.tactility.mcpscreen): full-screen LVGL canvas for LLM-driven display override
- Register both new apps in Tactility.cpp
This commit is contained in:
Adolfo Reyna
2026-06-26 23:02:09 -04:00
parent 28ff01561a
commit 41004a3c6f
11 changed files with 4215 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Tactility/Source/Tactility.cpp
+4
View File
@@ -118,6 +118,8 @@ namespace app {
namespace apwebserver { extern const AppManifest manifest; }
namespace crashdiagnostics { extern const AppManifest manifest; }
namespace webserversettings { extern const AppManifest manifest; }
namespace mcpsettings { extern const AppManifest manifest; }
namespace mcpoverride { extern const AppManifest manifest; }
#if CONFIG_TT_TDECK_WORKAROUND == 1
namespace keyboardsettings { extern const AppManifest manifest; } // T-Deck only for now
namespace trackballsettings { extern const AppManifest manifest; } // T-Deck only for now
@@ -167,6 +169,8 @@ static void registerInternalApps() {
#ifdef ESP_PLATFORM
addAppManifest(app::apwebserver::manifest);
addAppManifest(app::webserversettings::manifest);
addAppManifest(app::mcpsettings::manifest);
addAppManifest(app::mcpoverride::manifest);
addAppManifest(app::crashdiagnostics::manifest);
addAppManifest(app::development::manifest);
#if defined(CONFIG_TT_TDECK_WORKAROUND)
Tactility/Source/app/mcpoverride/McpOverrideApp.cpp
+140
View File
@@ -0,0 +1,140 @@
#ifdef ESP_PLATFORM
#include <Tactility/Tactility.h>
#include <Tactility/mcp/McpSystem.h>
#include <Tactility/lvgl/Toolbar.h>
#include <Tactility/lvgl/LvglSync.h>
#include <Tactility/Logger.h>
#include <lvgl.h>
#include <tactility/lvgl_icon_shared.h>
#include <esp_heap_caps.h>
namespace tt::app::mcpoverride {
static const auto LOGGER = Logger("McpOverrideApp");
class McpOverrideApp final : public App {
public:
void onCreate(AppContext& app) override {
// Prepare global state
auto& state = mcp::getState();
state.overrideActive = false;
}
void onShow(AppContext& app, lv_obj_t* parent) override {
LOGGER.info("onShow: Starting MCP Override display canvas");
auto& state = mcp::getState();
lv_obj_remove_flag(parent, LV_OBJ_FLAG_SCROLLABLE);
lv_obj_set_flex_flow(parent, LV_FLEX_FLOW_COLUMN);
lv_obj_set_style_pad_all(parent, 0, LV_PART_MAIN);
lv_obj_set_style_pad_row(parent, 0, LV_PART_MAIN);
// Standard toolbar so the user can navigate back
lv_obj_t* toolbar = lvgl::toolbar_create(parent, app);
lv_obj_align(toolbar, LV_ALIGN_TOP_MID, 0, 0);
lv_obj_t* title_label = lv_label_create(toolbar);
lv_label_set_text(title_label, "MCP Override Screen");
// Create drawing canvas
state.drawArea = lv_canvas_create(parent);
lv_obj_set_width(state.drawArea, LV_PCT(100));
lv_obj_set_flex_grow(state.drawArea, 1);
lv_obj_set_style_radius(state.drawArea, 0, LV_PART_MAIN);
lv_obj_set_style_border_width(state.drawArea, 0, LV_PART_MAIN);
lv_obj_set_style_pad_all(state.drawArea, 0, LV_PART_MAIN);
lv_obj_remove_flag(state.drawArea, LV_OBJ_FLAG_SCROLLABLE);
// Get display metrics
lv_display_t* display = lv_obj_get_display(parent);
state.displayWidth = lv_display_get_horizontal_resolution(display);
state.displayHeight = lv_display_get_vertical_resolution(display);
lv_obj_update_layout(parent);
state.drawWidth = lv_obj_get_content_width(state.drawArea);
state.drawHeight = lv_obj_get_content_height(state.drawArea);
// Allocate framebuffer
size_t required_size = (size_t)state.drawWidth * state.drawHeight * sizeof(uint16_t);
if (state.framebuffer == nullptr || state.framebufferSize != required_size) {
if (state.framebuffer != nullptr) {
heap_caps_free(state.framebuffer);
state.framebuffer = nullptr;
}
state.framebuffer = (uint16_t*)heap_caps_malloc(required_size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
if (state.framebuffer == nullptr) {
state.framebuffer = (uint16_t*)heap_caps_malloc(required_size, MALLOC_CAP_8BIT);
}
state.framebufferSize = state.framebuffer == nullptr ? 0 : required_size;
}
if (state.framebuffer == nullptr) {
LOGGER.error("Failed to allocate {} bytes framebuffer", (unsigned)required_size);
lv_obj_t* error = lv_label_create(state.drawArea);
lv_label_set_text(error, "Framebuffer allocation failed");
lv_obj_center(error);
return;
}
lv_canvas_set_buffer(
state.drawArea,
state.framebuffer,
state.drawWidth,
state.drawHeight,
LV_COLOR_FORMAT_RGB565
);
// Initialize welcome/waiting screen if LLM hasn't written anything yet
if (!state.overrideActive) {
// Fill with a nice dark blue/slate color
for (size_t i = 0; i < (size_t)state.drawWidth * state.drawHeight; ++i) {
state.framebuffer[i] = 0x18E3;
}
state.drawColor = 1;
lv_obj_t* welcome_label = lv_label_create(state.drawArea);
lv_label_set_text(welcome_label, "Waiting for LLM...");
lv_obj_set_style_text_color(welcome_label, lv_color_white(), LV_PART_MAIN);
lv_obj_align(welcome_label, LV_ALIGN_CENTER, 0, -20);
lv_obj_t* desc_label = lv_label_create(state.drawArea);
lv_label_set_text_fmt(desc_label, "Display Resolution: %ux%u", state.drawWidth, state.drawHeight);
lv_obj_set_style_text_color(desc_label, lv_palette_lighten(LV_PALETTE_BLUE, 3), LV_PART_MAIN);
lv_obj_align(desc_label, LV_ALIGN_CENTER, 0, 10);
}
}
void onHide(AppContext& app) override {
LOGGER.info("onHide: Tearing down MCP Override canvas");
auto& state = mcp::getState();
state.drawArea = nullptr;
if (state.framebuffer != nullptr) {
heap_caps_free(state.framebuffer);
state.framebuffer = nullptr;
state.framebufferSize = 0;
}
state.overrideActive = false;
// Stop any running tone or recording to prevent stuck state
state.audioRunning = false;
}
void onDestroy(AppContext& app) override {
onHide(app);
}
};
extern const AppManifest manifest = {
.appId = "one.tactility.mcpscreen", // Keep the original appId for compatibility
.appName = "MCP Override Screen",
.appIcon = LVGL_ICON_SHARED_TOOLBAR,
.appCategory = Category::System,
.createApp = create<McpOverrideApp>
};
} // namespace
#endif // ESP_PLATFORM
Tactility/Source/app/mcpsettings/McpSettingsApp.cpp
+206
View File
@@ -0,0 +1,206 @@
#ifdef ESP_PLATFORM
#include <Tactility/Tactility.h>
#include <Tactility/settings/McpSettings.h>
#include <Tactility/settings/WebServerSettings.h>
#include <Tactility/service/webserver/WebServerService.h>
#include <Tactility/lvgl/Toolbar.h>
#include <Tactility/lvgl/LvglSync.h>
#include <Tactility/Logger.h>
#include <lvgl.h>
#include <tactility/lvgl_icon_shared.h>
#include <esp_netif.h>
#include <esp_wifi.h>
namespace tt::app::mcpsettings {
static const auto LOGGER = tt::Logger("McpSettingsApp");
class McpSettingsApp final : public App {
settings::mcp::McpSettings mcpSettings;
settings::mcp::McpSettings originalSettings;
settings::webserver::WebServerSettings wsSettings;
bool updated = false;
lv_obj_t* switchMcpEnabled = nullptr;
lv_obj_t* switchMcpOverrideEnabled = nullptr;
lv_obj_t* labelUrlValue = nullptr;
static void onMcpEnabledSwitch(lv_event_t* e) {
auto* app = static_cast<McpSettingsApp*>(lv_event_get_user_data(e));
bool enabled = lv_obj_has_state(app->switchMcpEnabled, LV_STATE_CHECKED);
getMainDispatcher().dispatch([app, enabled] {
app->mcpSettings.mcpEnabled = enabled;
app->updated = true;
if (lvgl::lock(100)) {
app->updateUrlDisplay();
lvgl::unlock();
}
});
}
static void onMcpOverrideSwitch(lv_event_t* e) {
auto* app = static_cast<McpSettingsApp*>(lv_event_get_user_data(e));
bool enabled = lv_obj_has_state(app->switchMcpOverrideEnabled, LV_STATE_CHECKED);
getMainDispatcher().dispatch([app, enabled] {
app->mcpSettings.mcpOverrideEnabled = enabled;
app->updated = true;
});
}
void updateUrlDisplay() {
if (!labelUrlValue) return;
if (!mcpSettings.mcpEnabled) {
lv_label_set_text(labelUrlValue, "Disabled");
return;
}
std::string url = "http://";
if (wsSettings.wifiMode == settings::webserver::WiFiMode::AccessPoint) {
url += "192.168.4.1";
} else {
esp_netif_t* netif = esp_netif_get_handle_from_ifkey("WIFI_STA_DEF");
if (netif != nullptr) {
esp_netif_ip_info_t ip_info;
if (esp_netif_get_ip_info(netif, &ip_info) == ESP_OK && ip_info.ip.addr != 0) {
char ip_str[16];
snprintf(ip_str, sizeof(ip_str), IPSTR, IP2STR(&ip_info.ip));
url += ip_str;
} else {
url = "Connecting...";
}
} else {
url = "Not connected";
}
}
if (url.starts_with("http://")) {
if (wsSettings.webServerPort != 80) {
url += ":" + std::to_string(wsSettings.webServerPort);
}
url += "/api/mcp";
}
lv_label_set_text(labelUrlValue, url.c_str());
}
public:
void onCreate(AppContext& app) override {
mcpSettings = settings::mcp::loadOrGetDefault();
originalSettings = mcpSettings;
wsSettings = settings::webserver::loadOrGetDefault();
}
void onShow(AppContext& app, lv_obj_t* parent) override {
lv_obj_set_flex_flow(parent, LV_FLEX_FLOW_COLUMN);
lv_obj_set_style_pad_row(parent, 0, LV_STATE_DEFAULT);
lv_obj_t* toolbar = lvgl::toolbar_create(parent, app);
// MCP Enable toggle on toolbar
switchMcpEnabled = lvgl::toolbar_add_switch_action(toolbar);
if (mcpSettings.mcpEnabled) {
lv_obj_add_state(switchMcpEnabled, LV_STATE_CHECKED);
}
lv_obj_add_event_cb(switchMcpEnabled, onMcpEnabledSwitch, LV_EVENT_VALUE_CHANGED, this);
auto* main_wrapper = lv_obj_create(parent);
lv_obj_set_flex_flow(main_wrapper, LV_FLEX_FLOW_COLUMN);
lv_obj_set_width(main_wrapper, LV_PCT(100));
lv_obj_set_flex_grow(main_wrapper, 1);
// Screen Override Switch
auto* override_wrapper = lv_obj_create(main_wrapper);
lv_obj_set_size(override_wrapper, LV_PCT(100), LV_SIZE_CONTENT);
lv_obj_set_style_pad_all(override_wrapper, 0, LV_STATE_DEFAULT);
lv_obj_set_style_border_width(override_wrapper, 0, LV_STATE_DEFAULT);
auto* override_label = lv_label_create(override_wrapper);
lv_label_set_text(override_label, "Screen Override");
lv_obj_align(override_label, LV_ALIGN_LEFT_MID, 0, 0);
switchMcpOverrideEnabled = lv_switch_create(override_wrapper);
if (mcpSettings.mcpOverrideEnabled) {
lv_obj_add_state(switchMcpOverrideEnabled, LV_STATE_CHECKED);
}
lv_obj_align(switchMcpOverrideEnabled, LV_ALIGN_RIGHT_MID, 0, 0);
lv_obj_add_event_cb(switchMcpOverrideEnabled, onMcpOverrideSwitch, LV_EVENT_VALUE_CHANGED, this);
// URL Display
auto* url_wrapper = lv_obj_create(main_wrapper);
lv_obj_set_size(url_wrapper, LV_PCT(100), LV_SIZE_CONTENT);
lv_obj_set_style_pad_all(url_wrapper, 10, LV_STATE_DEFAULT);
lv_obj_set_style_border_width(url_wrapper, 1, LV_STATE_DEFAULT);
lv_obj_set_flex_flow(url_wrapper, LV_FLEX_FLOW_COLUMN);
lv_obj_set_style_flex_cross_place(url_wrapper, LV_FLEX_ALIGN_START, 0);
auto* url_title = lv_label_create(url_wrapper);
lv_label_set_text(url_title, "MCP Endpoint URL:");
labelUrlValue = lv_label_create(url_wrapper);
if (lv_display_get_color_format(lv_obj_get_display(parent)) == LV_COLOR_FORMAT_L8) {
lv_obj_set_style_text_color(labelUrlValue, lv_theme_get_color_secondary(labelUrlValue), LV_PART_MAIN);
} else {
lv_obj_set_style_text_color(labelUrlValue, lv_palette_main(LV_PALETTE_BLUE), 0);
}
updateUrlDisplay();
// Info / Documentation text
auto* info_label = lv_label_create(main_wrapper);
lv_label_set_long_mode(info_label, LV_LABEL_LONG_WRAP);
lv_obj_set_width(info_label, LV_PCT(95));
if (lv_display_get_color_format(lv_obj_get_display(parent)) != LV_COLOR_FORMAT_L8) {
lv_obj_set_style_text_color(info_label, lv_palette_main(LV_PALETTE_GREY), 0);
}
lv_label_set_text(info_label,
"MCP (Model Context Protocol) Screen service allows LLMs to interact with the device "
"screen, audio, and tools directly.\n\n"
"Endpoints:\n"
"- POST /api/mcp (JSON-RPC tools)\n"
"- POST /api/screen/raw (big-endian RGB565 writes)\n\n"
"If Screen Override is enabled, the device automatically shows the canvas upon receiving drawing calls.");
}
void onHide(AppContext& app) override {
if (updated) {
const auto copy = mcpSettings;
const bool mcpStateChanged = (copy.mcpEnabled != originalSettings.mcpEnabled);
getMainDispatcher().dispatch([copy, mcpStateChanged]{
// Save to properties file
if (!settings::mcp::save(copy)) {
LOGGER.warn("Failed to persist MCP settings");
}
// Publish WebServerSettingsChanged event so the HTTP server restarts/refreshes if needed
service::webserver::getPubsub()->publish(service::webserver::WebServerEvent::WebServerSettingsChanged);
if (mcpStateChanged) {
LOGGER.info("MCP server state changed to {}", copy.mcpEnabled ? "enabled" : "disabled");
// Control the WebServer service immediately through setWebServerEnabled
// WebServerService will check mcpSettings internally when setEnabled is executed
service::webserver::setWebServerEnabled(copy.mcpEnabled);
}
});
}
}
};
extern const AppManifest manifest = {
.appId = "McpSettings",
.appName = "MCP Screen",
.appIcon = LVGL_ICON_SHARED_SETTINGS,
.appCategory = Category::Settings,
.createApp = create<McpSettingsApp>
};
} // namespace
#endif // ESP_PLATFORM
Tactility/Source/mcp/McpSystem.cpp
+1090
View File
File diff suppressed because it is too large Load Diff
Tactility/Source/service/webserver/McpHandler.cpp
+754
View File
@@ -0,0 +1,754 @@
#ifdef ESP_PLATFORM
#include <Tactility/service/webserver/WebServerService.h>
#include <Tactility/mcp/McpSystem.h>
#include <Tactility/settings/McpSettings.h>
#include <Tactility/lvgl/LvglSync.h>
#include <Tactility/Logger.h>
#include <cJSON.h>
#include <esp_log.h>
#include <esp_http_server.h>
#include <mbedtls/base64.h>
#include <string>
#include <vector>
#include <memory>
#include <sstream>
namespace tt::service::webserver {
static const auto LOGGER = Logger("McpHandler");
#define MCP_JSON_BODY_LIMIT (256 * 1024)
#define MCP_RAW_BODY_LIMIT (512 * 1024)
static const char* TOOLS_JSON =
"["
"{"
"\"name\":\"get_capabilities\","
"\"description\":\"Get the Tactility display capabilities.\","
"\"inputSchema\":{\"type\":\"object\",\"properties\":{}}"
"},"
"{"
"\"name\":\"clear_screen\","
"\"description\":\"Clear the MCP drawing area to white (0) or black (1).\","
"\"inputSchema\":{"
"\"type\":\"object\","
"\"properties\":{\"color\":{\"type\":\"integer\",\"enum\":[0,1]}},"
"\"required\":[\"color\"]"
"}"
"},"
"{"
"\"name\":\"draw_text\","
"\"description\":\"Draw text in the MCP drawing area at x,y coordinates.\","
"\"inputSchema\":{"
"\"type\":\"object\","
"\"properties\":{"
"\"text\":{\"type\":\"string\"},"
"\"x\":{\"type\":\"integer\"},"
"\"y\":{\"type\":\"integer\"},"
"\"size\":{\"type\":\"integer\",\"enum\":[1,2],\"default\":1}"
"},"
"\"required\":[\"text\",\"x\",\"y\"]"
"}"
"},"
"{"
"\"name\":\"draw_image\","
"\"description\":\"Draw a bridge-preprocessed binary PBM image.\","
"\"inputSchema\":{"
"\"type\":\"object\","
"\"properties\":{"
"\"pbm_base64\":{\"type\":\"string\"},"
"\"x\":{\"type\":\"integer\",\"default\":0},"
"\"y\":{\"type\":\"integer\",\"default\":0},"
"\"dither\":{\"type\":\"boolean\",\"default\":true}"
"},"
"\"required\":[\"pbm_base64\"]"
"}"
"},"
"{"
"\"name\":\"draw_color_bmp\","
"\"description\":\"Draw an uncompressed 24-bit or 32-bit BMP image.\","
"\"inputSchema\":{"
"\"type\":\"object\","
"\"properties\":{"
"\"bmp_base64\":{\"type\":\"string\"},"
"\"x\":{\"type\":\"integer\",\"default\":0},"
"\"y\":{\"type\":\"integer\",\"default\":0}"
"},"
"\"required\":[\"bmp_base64\"]"
"}"
"},"
"{"
"\"name\":\"draw_raw_rgb565\","
"\"description\":\"Draw big-endian raw RGB565 pixels.\","
"\"inputSchema\":{"
"\"type\":\"object\","
"\"properties\":{"
"\"rgb565_base64\":{\"type\":\"string\"},"
"\"x\":{\"type\":\"integer\"},"
"\"y\":{\"type\":\"integer\"},"
"\"w\":{\"type\":\"integer\"},"
"\"h\":{\"type\":\"integer\"}"
"},"
"\"required\":[\"rgb565_base64\",\"x\",\"y\",\"w\",\"h\"]"
"}"
"},"
"{"
"\"name\":\"get_screenshot\","
"\"description\":\"Capture the MCP framebuffer as a PBM image.\","
"\"inputSchema\":{\"type\":\"object\",\"properties\":{}}"
"},"
"{"
"\"name\":\"play_tone\","
"\"description\":\"Play a pure sine wave tone on the board speaker.\","
"\"inputSchema\":{\"type\":\"object\",\"properties\":{"
"\"frequency\":{\"type\":\"integer\",\"default\":440},"
"\"duration_ms\":{\"type\":\"integer\",\"default\":1000},"
"\"volume\":{\"type\":\"integer\",\"default\":50}"
"}}"
"},"
"{"
"\"name\":\"play_audio\","
"\"description\":\"Play a 16 kHz 16-bit PCM WAV file from app user data.\","
"\"inputSchema\":{\"type\":\"object\",\"properties\":{"
"\"filename\":{\"type\":\"string\"},"
"\"volume\":{\"type\":\"integer\",\"default\":50}"
"},\"required\":[\"filename\"]}"
"},"
"{"
"\"name\":\"record_voice\","
"\"description\":\"Record 16 kHz 16-bit mono PCM to app user data.\","
"\"inputSchema\":{\"type\":\"object\",\"properties\":{"
"\"duration_sec\":{\"type\":\"integer\",\"default\":4},"
"\"filename\":{\"type\":\"string\",\"default\":\"recording.pcm\"}"
"}}"
"},"
"{"
"\"name\":\"play_audio_base64\","
"\"description\":\"Decode and play a base64-encoded 16 kHz 16-bit PCM WAV.\","
"\"inputSchema\":{\"type\":\"object\",\"properties\":{"
"\"wav_base64\":{\"type\":\"string\"},"
"\"volume\":{\"type\":\"integer\",\"default\":50}"
"},\"required\":[\"wav_base64\"]}"
"},"
"{"
"\"name\":\"play_mp3\","
"\"description\":\"Stream an MP3 file from app user data to the speaker.\","
"\"inputSchema\":{\"type\":\"object\",\"properties\":{"
"\"filename\":{\"type\":\"string\"},"
"\"volume\":{\"type\":\"integer\",\"default\":50}"
"},\"required\":[\"filename\"]}"
"},"
"{"
"\"name\":\"play_mp3_base64\","
"\"description\":\"Decode and play a base64-encoded MP3.\","
"\"inputSchema\":{\"type\":\"object\",\"properties\":{"
"\"mp3_base64\":{\"type\":\"string\"},"
"\"volume\":{\"type\":\"integer\",\"default\":50}"
"},\"required\":[\"mp3_base64\"]}"
"}"
"]";
static uint8_t* base64_decode(const char* input, size_t* output_size) {
if (input == NULL || output_size == NULL) {
return NULL;
}
const char* payload = input;
const char* marker = strstr(input, ";base64,");
if (marker != NULL) {
payload = marker + 8;
}
size_t input_size = strlen(payload);
size_t approx_output_len = (input_size * 3) / 4;
uint8_t* output = (uint8_t*)malloc(approx_output_len + 4);
if (output == NULL) {
return NULL;
}
size_t actual_len = 0;
int ret = mbedtls_base64_decode(output, approx_output_len + 4, &actual_len,
(const unsigned char*)payload, input_size);
if (ret != 0) {
free(output);
return NULL;
}
*output_size = actual_len;
return output;
}
static cJSON* make_error(cJSON* id, int code, const char* message) {
cJSON* root = cJSON_CreateObject();
cJSON_AddStringToObject(root, "jsonrpc", "2.0");
cJSON* error = cJSON_AddObjectToObject(root, "error");
cJSON_AddNumberToObject(error, "code", code);
cJSON_AddStringToObject(error, "message", message);
if (id != NULL) {
cJSON_AddItemToObject(root, "id", cJSON_Duplicate(id, true));
} else {
cJSON_AddNullToObject(root, "id");
}
return root;
}
static cJSON* make_tool_result(cJSON* id, const char* text) {
cJSON* root = cJSON_CreateObject();
cJSON_AddStringToObject(root, "jsonrpc", "2.0");
cJSON* result = cJSON_AddObjectToObject(root, "result");
cJSON_AddArrayToObject(result, "content");
cJSON* item = cJSON_CreateObject();
cJSON_AddStringToObject(item, "type", "text");
cJSON_AddStringToObject(item, "text", text);
cJSON_AddItemToArray(cJSON_GetObjectItem(result, "content"), item);
if (id != NULL) {
cJSON_AddItemToObject(root, "id", cJSON_Duplicate(id, true));
} else {
cJSON_AddNullToObject(root, "id");
}
return root;
}
static cJSON* handle_tool_call(cJSON* id, cJSON* params) {
if (!cJSON_IsObject(params)) {
return make_error(id, -32602, "Invalid params");
}
cJSON* name_item = cJSON_GetObjectItemCaseSensitive(params, "name");
cJSON* arguments = cJSON_GetObjectItemCaseSensitive(params, "arguments");
if (!cJSON_IsString(name_item) || name_item->valuestring == NULL) {
return make_error(id, -32602, "Missing tool name");
}
if (arguments != NULL && !cJSON_IsObject(arguments)) {
return make_error(id, -32602, "Tool arguments must be an object");
}
const char* name = name_item->valuestring;
auto& state = mcp::getState();
if (strcmp(name, "get_capabilities") == 0) {
char capabilities[384];
uint16_t width = state.drawWidth > 0 ? state.drawWidth : state.displayWidth;
uint16_t height = state.drawHeight > 0 ? state.drawHeight : state.displayHeight;
snprintf(
capabilities,
sizeof(capabilities),
"{\"color\":true,\"width\":%u,\"height\":%u,"
"\"formats\":[\"rgb565_base64\",\"bmp_base64\",\"pbm_base64\"],"
"\"audio\":{\"sampleRate\":16000,\"bitsPerSample\":16,\"channels\":1},"
"\"platform\":\"tactility\",\"appVersion\":\"0.1.0\","
"\"uiVisible\":%s}",
width,
height,
(state.drawArea != nullptr) ? "true" : "false"
);
return make_tool_result(id, capabilities);
}
if (strcmp(name, "clear_screen") == 0) {
cJSON* color_item = cJSON_GetObjectItemCaseSensitive(arguments, "color");
if (!cJSON_IsNumber(color_item) ||
(color_item->valueint != 0 && color_item->valueint != 1)) {
return make_error(id, -32602, "color must be 0 or 1");
}
if (!mcp::clearScreen(color_item->valueint)) {
return make_error(id, -32010, "Failed to clear screen");
}
return make_tool_result(id, "Screen cleared.");
}
if (strcmp(name, "draw_text") == 0) {
cJSON* text_item = cJSON_GetObjectItemCaseSensitive(arguments, "text");
cJSON* x_item = cJSON_GetObjectItemCaseSensitive(arguments, "x");
cJSON* y_item = cJSON_GetObjectItemCaseSensitive(arguments, "y");
cJSON* size_item = cJSON_GetObjectItemCaseSensitive(arguments, "size");
if (!cJSON_IsString(text_item) || text_item->valuestring == NULL ||
!cJSON_IsNumber(x_item) || !cJSON_IsNumber(y_item)) {
return make_error(id, -32602, "draw_text requires text, x, and y");
}
if (strlen(text_item->valuestring) > 512) {
return make_error(id, -32602, "text exceeds 512 characters");
}
int size = cJSON_IsNumber(size_item) ? size_item->valueint : 1;
if (size != 1 && size != 2) {
return make_error(id, -32602, "size must be 1 or 2");
}
if (!mcp::drawText(text_item->valuestring, x_item->valueint, y_item->valueint, size)) {
return make_error(id, -32010, "Failed to draw text");
}
char message[160];
snprintf(
message,
sizeof(message),
"Successfully drew text at (%d, %d) with size %d.",
x_item->valueint,
y_item->valueint,
size
);
return make_tool_result(id, message);
}
if (strcmp(name, "draw_raw_rgb565") == 0) {
cJSON* encoded_item = cJSON_GetObjectItemCaseSensitive(arguments, "rgb565_base64");
cJSON* x_item = cJSON_GetObjectItemCaseSensitive(arguments, "x");
cJSON* y_item = cJSON_GetObjectItemCaseSensitive(arguments, "y");
cJSON* width_item = cJSON_GetObjectItemCaseSensitive(arguments, "w");
cJSON* height_item = cJSON_GetObjectItemCaseSensitive(arguments, "h");
if (!cJSON_IsString(encoded_item) || !cJSON_IsNumber(x_item) ||
!cJSON_IsNumber(y_item) || !cJSON_IsNumber(width_item) ||
!cJSON_IsNumber(height_item)) {
return make_error(id, -32602, "draw_raw_rgb565 requires rgb565_base64, x, y, w, and h");
}
size_t decoded_size = 0;
uint8_t* decoded = base64_decode(encoded_item->valuestring, &decoded_size);
if (decoded == NULL) {
return make_error(id, -32602, "Invalid RGB565 base64 data");
}
bool drawn = mcp::drawRgb565(
decoded,
decoded_size,
x_item->valueint,
y_item->valueint,
width_item->valueint,
height_item->valueint
);
free(decoded);
return drawn
? make_tool_result(id, "Raw RGB565 data displayed successfully.")
: make_error(id, -32010, "Failed to draw RGB565 data");
}
if (strcmp(name, "draw_color_bmp") == 0) {
cJSON* encoded_item = cJSON_GetObjectItemCaseSensitive(arguments, "bmp_base64");
cJSON* x_item = cJSON_GetObjectItemCaseSensitive(arguments, "x");
cJSON* y_item = cJSON_GetObjectItemCaseSensitive(arguments, "y");
if (!cJSON_IsString(encoded_item)) {
return make_error(id, -32602, "Missing bmp_base64");
}
size_t decoded_size = 0;
uint8_t* decoded = base64_decode(encoded_item->valuestring, &decoded_size);
if (decoded == NULL) {
return make_error(id, -32602, "Invalid BMP base64 data");
}
bool drawn = mcp::drawBmp(
decoded,
decoded_size,
cJSON_IsNumber(x_item) ? x_item->valueint : 0,
cJSON_IsNumber(y_item) ? y_item->valueint : 0
);
free(decoded);
return drawn
? make_tool_result(id, "Color BMP image displayed successfully.")
: make_error(id, -32602, "Unsupported or invalid BMP image");
}
if (strcmp(name, "draw_image") == 0) {
cJSON* encoded_item = cJSON_GetObjectItemCaseSensitive(arguments, "pbm_base64");
cJSON* x_item = cJSON_GetObjectItemCaseSensitive(arguments, "x");
cJSON* y_item = cJSON_GetObjectItemCaseSensitive(arguments, "y");
if (!cJSON_IsString(encoded_item)) {
return make_error(
id,
-32602,
"draw_image requires bridge-preprocessed pbm_base64"
);
}
size_t decoded_size = 0;
uint8_t* decoded = base64_decode(encoded_item->valuestring, &decoded_size);
if (decoded == NULL) {
return make_error(id, -32602, "Invalid PBM base64 data");
}
bool drawn = mcp::drawPbm(
decoded,
decoded_size,
cJSON_IsNumber(x_item) ? x_item->valueint : 0,
cJSON_IsNumber(y_item) ? y_item->valueint : 0
);
free(decoded);
return drawn
? make_tool_result(id, "Image displayed successfully.")
: make_error(id, -32602, "Unsupported or invalid PBM image");
}
if (strcmp(name, "get_screenshot") == 0) {
std::string encoded = mcp::getScreenshotPbmBase64();
if (encoded.empty()) {
return make_error(id, -32010, "Screenshot capture failed");
}
std::string result = "__PBM_BASE64__:" + encoded;
return make_tool_result(id, result.c_str());
}
if (strcmp(name, "play_tone") == 0) {
cJSON* frequency_item = cJSON_GetObjectItemCaseSensitive(arguments, "frequency");
cJSON* duration_item = cJSON_GetObjectItemCaseSensitive(arguments, "duration_ms");
cJSON* volume_item = cJSON_GetObjectItemCaseSensitive(arguments, "volume");
int frequency = cJSON_IsNumber(frequency_item) ? frequency_item->valueint : 440;
int duration = cJSON_IsNumber(duration_item) ? duration_item->valueint : 1000;
int volume = cJSON_IsNumber(volume_item) ? volume_item->valueint : 50;
if (frequency < 50 || frequency > 10000 ||
duration < 50 || duration > 5000 ||
volume < 0 || volume > 100) {
return make_error(id, -32602, "frequency, duration_ms, or volume is out of range");
}
std::string error;
if (!mcp::playTone(frequency, duration, volume, error)) {
return make_error(id, -32020, error.c_str());
}
char message[128];
snprintf(
message,
sizeof(message),
"Played tone of %dHz for %dms at volume %d.",
frequency,
duration,
volume
);
return make_tool_result(id, message);
}
if (strcmp(name, "record_voice") == 0) {
cJSON* duration_item = cJSON_GetObjectItemCaseSensitive(arguments, "duration_sec");
cJSON* filename_item = cJSON_GetObjectItemCaseSensitive(arguments, "filename");
int duration = cJSON_IsNumber(duration_item) ? duration_item->valueint : 4;
const char* filename = cJSON_IsString(filename_item)
? filename_item->valuestring
: "recording.pcm";
if (duration < 1 || duration > 15) {
return make_error(id, -32602, "duration_sec must be between 1 and 15");
}
std::string error;
size_t recorded_bytes = 0;
if (!mcp::recordVoice(duration, filename, recorded_bytes, error)) {
return make_error(id, -32020, error.c_str());
}
char message[180];
snprintf(
message,
sizeof(message),
"Successfully recorded %d seconds of audio to '%s' (%u bytes).",
duration,
filename,
(unsigned)recorded_bytes
);
return make_tool_result(id, message);
}
if (strcmp(name, "play_audio") == 0) {
cJSON* filename_item = cJSON_GetObjectItemCaseSensitive(arguments, "filename");
cJSON* volume_item = cJSON_GetObjectItemCaseSensitive(arguments, "volume");
int volume = cJSON_IsNumber(volume_item) ? volume_item->valueint : 50;
if (!cJSON_IsString(filename_item) || filename_item->valuestring == NULL) {
return make_error(id, -32602, "play_audio requires filename");
}
if (volume < 0 || volume > 100) {
return make_error(id, -32602, "volume must be between 0 and 100");
}
std::string error;
if (!mcp::playAudioFile(filename_item->valuestring, volume, error)) {
return make_error(id, -32020, error.c_str());
}
char message[160];
snprintf(
message,
sizeof(message),
"Successfully played audio file '%s'.",
filename_item->valuestring
);
return make_tool_result(id, message);
}
if (strcmp(name, "play_audio_base64") == 0) {
cJSON* encoded_item = cJSON_GetObjectItemCaseSensitive(arguments, "wav_base64");
cJSON* volume_item = cJSON_GetObjectItemCaseSensitive(arguments, "volume");
int volume = cJSON_IsNumber(volume_item) ? volume_item->valueint : 50;
if (!cJSON_IsString(encoded_item) || encoded_item->valuestring == NULL) {
return make_error(id, -32602, "play_audio_base64 requires wav_base64");
}
if (volume < 0 || volume > 100) {
return make_error(id, -32602, "volume must be between 0 and 100");
}
size_t decoded_size = 0;
uint8_t* decoded = base64_decode(encoded_item->valuestring, &decoded_size);
if (decoded == NULL) {
return make_error(id, -32602, "Invalid WAV base64 data");
}
std::string error;
bool played = mcp::playWavMemory(
decoded,
decoded_size,
volume,
error
);
free(decoded);
return played
? make_tool_result(id, "Successfully played base64 audio.")
: make_error(id, -32020, error.c_str());
}
if (strcmp(name, "play_mp3") == 0) {
cJSON* filename_item = cJSON_GetObjectItemCaseSensitive(arguments, "filename");
cJSON* volume_item = cJSON_GetObjectItemCaseSensitive(arguments, "volume");
int volume = cJSON_IsNumber(volume_item) ? volume_item->valueint : 50;
if (!cJSON_IsString(filename_item) || filename_item->valuestring == NULL) {
return make_error(id, -32602, "play_mp3 requires filename");
}
if (volume < 0 || volume > 100) {
return make_error(id, -32602, "volume must be between 0 and 100");
}
std::string error;
if (!mcp::playMp3File(filename_item->valuestring, volume, error)) {
return make_error(id, -32020, error.c_str());
}
char message[160];
snprintf(
message,
sizeof(message),
"Successfully played MP3 file '%s'.",
filename_item->valuestring
);
return make_tool_result(id, message);
}
if (strcmp(name, "play_mp3_base64") == 0) {
cJSON* encoded_item = cJSON_GetObjectItemCaseSensitive(arguments, "mp3_base64");
cJSON* volume_item = cJSON_GetObjectItemCaseSensitive(arguments, "volume");
int volume = cJSON_IsNumber(volume_item) ? volume_item->valueint : 50;
if (!cJSON_IsString(encoded_item) || encoded_item->valuestring == NULL) {
return make_error(id, -32602, "play_mp3_base64 requires mp3_base64");
}
if (volume < 0 || volume > 100) {
return make_error(id, -32602, "volume must be between 0 and 100");
}
size_t decoded_size = 0;
uint8_t* decoded = base64_decode(encoded_item->valuestring, &decoded_size);
if (decoded == NULL) {
return make_error(id, -32602, "Invalid MP3 base64 data");
}
std::string error;
bool played = mcp::playMp3Memory(
decoded,
decoded_size,
volume,
error
);
free(decoded);
return played
? make_tool_result(id, "Successfully played base64 MP3 audio.")
: make_error(id, -32020, error.c_str());
}
return make_error(id, -32602, "Unknown tool");
}
static cJSON* handle_rpc(const char* body) {
cJSON* request = cJSON_Parse(body);
if (request == NULL) {
return make_error(NULL, -32700, "Parse error");
}
cJSON* id = cJSON_GetObjectItemCaseSensitive(request, "id");
cJSON* jsonrpc = cJSON_GetObjectItemCaseSensitive(request, "jsonrpc");
cJSON* method = cJSON_GetObjectItemCaseSensitive(request, "method");
cJSON* params = cJSON_GetObjectItemCaseSensitive(request, "params");
if (!cJSON_IsObject(request) ||
!cJSON_IsString(jsonrpc) ||
strcmp(jsonrpc->valuestring, "2.0") != 0 ||
!cJSON_IsString(method)) {
cJSON* response = make_error(id, -32600, "Invalid Request");
cJSON_Delete(request);
return response;
}
cJSON* response = NULL;
if (strcmp(method->valuestring, "tools/list") == 0) {
cJSON* tools = cJSON_Parse(TOOLS_JSON);
if (tools == NULL) {
response = make_error(id, -32603, "Failed to construct tool list");
} else {
response = cJSON_CreateObject();
cJSON_AddStringToObject(response, "jsonrpc", "2.0");
cJSON* result = cJSON_AddObjectToObject(response, "result");
cJSON_AddItemToObject(result, "tools", tools);
if (id != NULL) {
cJSON_AddItemToObject(response, "id", cJSON_Duplicate(id, true));
} else {
cJSON_AddNullToObject(response, "id");
}
}
} else if (strcmp(method->valuestring, "tools/call") == 0) {
response = handle_tool_call(id, params);
} else {
response = make_error(id, -32601, "Method not found");
}
cJSON_Delete(request);
return response;
}
// POST /api/mcp
esp_err_t WebServerService::handleApiMcp(httpd_req_t* request) {
LOGGER.info("POST /api/mcp");
// Load MCP Settings and check if enabled
auto mcpSettings = settings::mcp::loadOrGetDefault();
if (!mcpSettings.mcpEnabled) {
httpd_resp_send_err(request, HTTPD_404_NOT_FOUND, "MCP server is disabled");
return ESP_FAIL;
}
int content_len = request->content_len;
if (content_len <= 0 || content_len > MCP_JSON_BODY_LIMIT) {
httpd_resp_send_err(request, HTTPD_400_BAD_REQUEST, "Invalid Content-Length");
return ESP_FAIL;
}
char* body = (char*)malloc(content_len + 1);
if (body == NULL) {
httpd_resp_send_err(request, HTTPD_500_INTERNAL_SERVER_ERROR, "Out of memory");
return ESP_FAIL;
}
int received = 0;
while (received < content_len) {
int ret = httpd_req_recv(request, body + received, content_len - received);
if (ret <= 0) {
if (ret == HTTPD_SOCK_ERR_TIMEOUT) {
continue;
}
free(body);
httpd_resp_send_err(request, HTTPD_400_BAD_REQUEST, "Incomplete request body");
return ESP_FAIL;
}
received += ret;
}
body[content_len] = '\0';
cJSON* response = handle_rpc(body);
free(body);
char* response_text = cJSON_PrintUnformatted(response);
cJSON_Delete(response);
if (response_text == NULL) {
httpd_resp_send_err(request, HTTPD_500_INTERNAL_SERVER_ERROR, "Serialization failed");
return ESP_FAIL;
}
httpd_resp_set_type(request, "application/json");
httpd_resp_set_hdr(request, "Access-Control-Allow-Origin", "*");
httpd_resp_sendstr(request, response_text);
cJSON_free(response_text);
return ESP_OK;
}
static int query_integer(const char* path, const char* name, int default_value) {
const char* query = strchr(path, '?');
if (query == NULL) {
return default_value;
}
query++;
size_t name_size = strlen(name);
while (*query != '\0') {
if (strncmp(query, name, name_size) == 0 && query[name_size] == '=') {
return atoi(query + name_size + 1);
}
query = strchr(query, '&');
if (query == NULL) {
break;
}
query++;
}
return default_value;
}
// POST /api/screen/raw
esp_err_t WebServerService::handleApiScreenRaw(httpd_req_t* request) {
LOGGER.info("POST /api/screen/raw");
// Load MCP Settings and check if enabled
auto mcpSettings = settings::mcp::loadOrGetDefault();
if (!mcpSettings.mcpEnabled) {
httpd_resp_send_err(request, HTTPD_404_NOT_FOUND, "MCP server is disabled");
return ESP_FAIL;
}
int content_len = request->content_len;
if (content_len <= 0 || content_len > MCP_RAW_BODY_LIMIT) {
httpd_resp_send_err(request, HTTPD_400_BAD_REQUEST, "Invalid Content-Length");
return ESP_FAIL;
}
uint8_t* body = (uint8_t*)malloc(content_len);
if (body == NULL) {
httpd_resp_send_err(request, HTTPD_500_INTERNAL_SERVER_ERROR, "Out of memory");
return ESP_FAIL;
}
int received = 0;
while (received < content_len) {
int ret = httpd_req_recv(request, (char*)body + received, content_len - received);
if (ret <= 0) {
if (ret == HTTPD_SOCK_ERR_TIMEOUT) {
continue;
}
free(body);
httpd_resp_send_err(request, HTTPD_400_BAD_REQUEST, "Incomplete request body");
return ESP_FAIL;
}
received += ret;
}
auto& state = mcp::getState();
const char* path = request->uri;
int x = query_integer(path, "x", 0);
int y = query_integer(path, "y", 0);
int width = query_integer(path, "w", state.drawWidth);
int height = query_integer(path, "h", state.drawHeight);
bool valid_size = width > 0 && height > 0 &&
(size_t)width * height * 2 == (size_t)content_len;
bool drawn = valid_size && mcp::drawRgb565(
body,
content_len,
x,
y,
width,
height
);
free(body);
if (drawn) {
httpd_resp_set_type(request, "text/plain");
httpd_resp_set_hdr(request, "Access-Control-Allow-Origin", "*");
httpd_resp_sendstr(request, "OK");
return ESP_OK;
} else {
httpd_resp_send_err(request, HTTPD_400_BAD_REQUEST, "Invalid RGB565 payload or size");
return ESP_FAIL;
}
}
} // namespace tt::service::webserver
#endif
Tactility/Source/service/webserver/WebServerService.cpp
+22 -5
View File
@@ -5,6 +5,7 @@
#include <Tactility/service/webserver/AssetVersion.h>
#include <Tactility/service/ServiceManifest.h>
#include <Tactility/settings/WebServerSettings.h>
#include <Tactility/settings/McpSettings.h>
#include <Tactility/MountPoints.h>
#include <Tactility/file/File.h>
#include <Tactility/Logger.h>
@@ -222,7 +223,9 @@ bool WebServerService::onStart(ServiceContext& service) {
lock.lock();
g_cachedSettings = settings::webserver::loadOrGetDefault();
g_settingsCached = true;
serverEnabled = g_cachedSettings.webServerEnabled;
auto mcpSettings = settings::mcp::loadOrGetDefault();
serverEnabled = g_cachedSettings.webServerEnabled || mcpSettings.mcpEnabled;
}
// Subscribe to settings change events to refresh cache
settingsEventSubscription = pubsub->subscribe([](WebServerEvent event) {
@@ -236,10 +239,10 @@ bool WebServerService::onStart(ServiceContext& service) {
// Start HTTP server only if enabled in settings (default: OFF to save memory)
if (serverEnabled) {
LOGGER.info("WebServer enabled in settings, starting HTTP server...");
LOGGER.info("WebServer or MCP enabled in settings, starting HTTP server...");
setEnabled(true);
} else {
LOGGER.info("WebServer disabled in settings, NOT starting HTTP server (saves ~10KB RAM)");
LOGGER.info("WebServer and MCP disabled in settings, NOT starting HTTP server (saves RAM)");
setEnabled(false);
}
@@ -268,11 +271,16 @@ void WebServerService::setEnabled(bool enabled) {
lock.lock();
if (enabled) {
// Start unconditionally if explicitly requested
if (!httpServer || !httpServer->isStarted()) {
startServer();
}
} else {
if (httpServer && httpServer->isStarted()) {
// Stop only if both web server and MCP are disabled
auto wsSettings = settings::webserver::loadOrGetDefault();
auto mcpSettings = settings::mcp::loadOrGetDefault();
bool anyEnabled = wsSettings.webServerEnabled || mcpSettings.mcpEnabled;
if (!anyEnabled && httpServer && httpServer->isStarted()) {
stopServer();
}
}
@@ -1035,13 +1043,22 @@ esp_err_t WebServerService::handleApiGet(httpd_req_t* request) {
// API POST dispatcher - all POST endpoints require authentication
esp_err_t WebServerService::handleApiPost(httpd_req_t* request) {
const char* uri = request->uri;
// Route MCP endpoints unauthenticated
if (strncmp(uri, "/api/mcp", 8) == 0) {
return handleApiMcp(request);
}
if (strncmp(uri, "/api/screen/raw", 15) == 0) {
return handleApiScreenRaw(request);
}
bool authPassed = false;
esp_err_t authResult = validateRequestAuth(request, authPassed);
if (!authPassed) {
return authResult;
}
const char* uri = request->uri;
if (strncmp(uri, "/api/apps/run", 13) == 0) {
return handleApiAppsRun(request);
}
Tactility/Source/settings/McpSettings.cpp
+67
View File
@@ -0,0 +1,67 @@
#include <Tactility/settings/McpSettings.h>
#include <Tactility/file/PropertiesFile.h>
#include <Tactility/file/File.h>
#include <Tactility/Logger.h>
#include <map>
#include <string>
namespace tt::settings::mcp {
static const auto LOGGER = Logger("McpSettings");
constexpr auto* SETTINGS_FILE = "/data/service/mcp/settings.properties";
constexpr auto* KEY_MCP_ENABLED = "mcpEnabled";
constexpr auto* KEY_MCP_OVERRIDE_ENABLED = "mcpOverrideEnabled";
bool load(McpSettings& settings) {
std::map<std::string, std::string> map;
if (!file::loadPropertiesFile(SETTINGS_FILE, map)) {
return false;
}
auto mcp_enabled = map.find(KEY_MCP_ENABLED);
auto mcp_override = map.find(KEY_MCP_OVERRIDE_ENABLED);
settings.mcpEnabled = (mcp_enabled != map.end())
? (mcp_enabled->second == "1" || mcp_enabled->second == "true")
: false;
settings.mcpOverrideEnabled = (mcp_override != map.end())
? (mcp_override->second == "1" || mcp_override->second == "true")
: true; // Default true
return true;
}
McpSettings getDefault() {
return McpSettings{
.mcpEnabled = false,
.mcpOverrideEnabled = true
};
}
McpSettings loadOrGetDefault() {
McpSettings settings;
if (!load(settings)) {
settings = getDefault();
file::findOrCreateDirectory("/data/service/mcp", 0755);
save(settings);
}
return settings;
}
bool save(const McpSettings& settings) {
file::findOrCreateDirectory("/data/service/mcp", 0755);
std::map<std::string, std::string> map;
map[KEY_MCP_ENABLED] = settings.mcpEnabled ? "true" : "false";
map[KEY_MCP_OVERRIDE_ENABLED] = settings.mcpOverrideEnabled ? "true" : "false";
if (!file::savePropertiesFile(SETTINGS_FILE, map)) {
LOGGER.error("Failed to save MCP settings to {}", SETTINGS_FILE);
return false;
}
return true;
}
} // namespace