adolforeyna/tactility
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement MCP drawing as screensaver instead of standalone app

Browse Source 
- Add ScreensaverType::McpScreen to DisplaySettings enum, toString/fromString
- Add 'MCP Screen' option to the Display settings screensaver dropdown
- New McpScreensaver (Screensaver subclass): creates full-screen lv_canvas on
  the overlay, allocates framebuffer in SPIRAM, registers canvas pointer in
  McpSystemState, shows 'Waiting for LLM...' until first draw command arrives
- Wire McpScreensaver into DisplayIdle::activateScreensaver() switch
- McpSystem::ensureOverrideScreen() now calls displayidle::findService()->
  startScreensaver() instead of launching McpOverrideApp — canvas appears
  automatically on any LLM draw command
- Remove McpOverrideApp (replaced by screensaver), deregister from Tactility.cpp
- Remove mcpOverrideEnabled from McpSettings — no longer needed
- Simplify McpSettingsApp: remove Screen Override toggle, update info text to
  direct users to Settings -> Display -> Screensaver -> MCP Screen
This commit is contained in:
Adolfo Reyna
2026-06-27 09:17:50 -04:00
parent 41004a3c6f
commit cd921e1aa9
11 changed files with 160 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Tactility/Include/Tactility/settings/DisplaySettings.h
+1
View File
@@ -18,6 +18,7 @@ enum class ScreensaverType {
Mystify,
MatrixRain,
StackChan,
McpScreen, // MCP LLM-driven canvas
Count // Sentinel for bounds checking - must be last
};
Tactility/Include/Tactility/settings/McpSettings.h
-1
View File
@@ -4,7 +4,6 @@ namespace tt::settings::mcp {
struct McpSettings {
bool mcpEnabled = false; // Enable MCP server endpoints on system web server
bool mcpOverrideEnabled = true; // Automatically switch to override screen on drawing commands
};
bool load(McpSettings& settings);
Tactility/Source/Tactility.cpp
-2
View File
@@ -119,7 +119,6 @@ namespace app {
namespace crashdiagnostics { extern const AppManifest manifest; }
namespace webserversettings { extern const AppManifest manifest; }
namespace mcpsettings { extern const AppManifest manifest; }
namespace mcpoverride { extern const AppManifest manifest; }
#if CONFIG_TT_TDECK_WORKAROUND == 1
namespace keyboardsettings { extern const AppManifest manifest; } // T-Deck only for now
namespace trackballsettings { extern const AppManifest manifest; } // T-Deck only for now
@@ -170,7 +169,6 @@ static void registerInternalApps() {
addAppManifest(app::apwebserver::manifest);
addAppManifest(app::webserversettings::manifest);
addAppManifest(app::mcpsettings::manifest);
addAppManifest(app::mcpoverride::manifest);
addAppManifest(app::crashdiagnostics::manifest);
addAppManifest(app::development::manifest);
#if defined(CONFIG_TT_TDECK_WORKAROUND)
Tactility/Source/app/display/Display.cpp
+2 -1
View File
@@ -286,7 +286,8 @@ public:
screensaverDropdown = lv_dropdown_create(screensaver_wrapper);
// Note: order correlates with settings::display::ScreensaverType enum order
lv_dropdown_set_options(screensaverDropdown, "None\nBouncing Balls\nMystify\nMatrix Rain\nStackChan");
lv_dropdown_set_options(screensaverDropdown, "None\nBouncing Balls\nMystify\nMatrix Rain\nStackChan\nMCP Screen");
lv_obj_align(screensaverDropdown, LV_ALIGN_RIGHT_MID, 0, 0);
lv_obj_add_event_cb(screensaverDropdown, onScreensaverChanged, LV_EVENT_VALUE_CHANGED, this);
lv_dropdown_set_selected(screensaverDropdown, static_cast<uint16_t>(displaySettings.screensaverType));
Tactility/Source/app/mcpsettings/McpSettingsApp.cpp
+5 -34
View File
@@ -26,7 +26,6 @@ class McpSettingsApp final : public App {
bool updated = false;
lv_obj_t* switchMcpEnabled = nullptr;
lv_obj_t* switchMcpOverrideEnabled = nullptr;
lv_obj_t* labelUrlValue = nullptr;
static void onMcpEnabledSwitch(lv_event_t* e) {
@@ -42,15 +41,6 @@ class McpSettingsApp final : public App {
});
}
static void onMcpOverrideSwitch(lv_event_t* e) {
auto* app = static_cast<McpSettingsApp*>(lv_event_get_user_data(e));
bool enabled = lv_obj_has_state(app->switchMcpOverrideEnabled, LV_STATE_CHECKED);
getMainDispatcher().dispatch([app, enabled] {
app->mcpSettings.mcpOverrideEnabled = enabled;
app->updated = true;
});
}
void updateUrlDisplay() {
if (!labelUrlValue) return;
@@ -60,7 +50,7 @@ class McpSettingsApp final : public App {
}
std::string url = "http://";
if (wsSettings.wifiMode == settings::webserver::WiFiMode::AccessPoint) {
url += "192.168.4.1";
} else {
@@ -114,23 +104,6 @@ public:
lv_obj_set_width(main_wrapper, LV_PCT(100));
lv_obj_set_flex_grow(main_wrapper, 1);
// Screen Override Switch
auto* override_wrapper = lv_obj_create(main_wrapper);
lv_obj_set_size(override_wrapper, LV_PCT(100), LV_SIZE_CONTENT);
lv_obj_set_style_pad_all(override_wrapper, 0, LV_STATE_DEFAULT);
lv_obj_set_style_border_width(override_wrapper, 0, LV_STATE_DEFAULT);
auto* override_label = lv_label_create(override_wrapper);
lv_label_set_text(override_label, "Screen Override");
lv_obj_align(override_label, LV_ALIGN_LEFT_MID, 0, 0);
switchMcpOverrideEnabled = lv_switch_create(override_wrapper);
if (mcpSettings.mcpOverrideEnabled) {
lv_obj_add_state(switchMcpOverrideEnabled, LV_STATE_CHECKED);
}
lv_obj_align(switchMcpOverrideEnabled, LV_ALIGN_RIGHT_MID, 0, 0);
lv_obj_add_event_cb(switchMcpOverrideEnabled, onMcpOverrideSwitch, LV_EVENT_VALUE_CHANGED, this);
// URL Display
auto* url_wrapper = lv_obj_create(main_wrapper);
lv_obj_set_size(url_wrapper, LV_PCT(100), LV_SIZE_CONTENT);
@@ -138,10 +111,10 @@ public:
lv_obj_set_style_border_width(url_wrapper, 1, LV_STATE_DEFAULT);
lv_obj_set_flex_flow(url_wrapper, LV_FLEX_FLOW_COLUMN);
lv_obj_set_style_flex_cross_place(url_wrapper, LV_FLEX_ALIGN_START, 0);
auto* url_title = lv_label_create(url_wrapper);
lv_label_set_text(url_title, "MCP Endpoint URL:");
labelUrlValue = lv_label_create(url_wrapper);
if (lv_display_get_color_format(lv_obj_get_display(parent)) == LV_COLOR_FORMAT_L8) {
lv_obj_set_style_text_color(labelUrlValue, lv_theme_get_color_secondary(labelUrlValue), LV_PART_MAIN);
@@ -164,7 +137,8 @@ public:
"Endpoints:\n"
"- POST /api/mcp (JSON-RPC tools)\n"
"- POST /api/screen/raw (big-endian RGB565 writes)\n\n"
"If Screen Override is enabled, the device automatically shows the canvas upon receiving drawing calls.");
"To show the LLM canvas, select 'MCP Screen' in Settings -> Display -> Screensaver. "
"The canvas also pops up automatically when an LLM sends a draw command.");
}
void onHide(AppContext& app) override {
@@ -183,9 +157,6 @@ public:
if (mcpStateChanged) {
LOGGER.info("MCP server state changed to {}", copy.mcpEnabled ? "enabled" : "disabled");
// Control the WebServer service immediately through setWebServerEnabled
// WebServerService will check mcpSettings internally when setEnabled is executed
service::webserver::setWebServerEnabled(copy.mcpEnabled);
}
});
Tactility/Source/mcp/McpSystem.cpp
+14 -11
View File
@@ -6,6 +6,7 @@
#include <Tactility/file/File.h>
#include <Tactility/app/App.h>
#include <Tactility/Logger.h>
#include <Tactility/service/displayidle/DisplayIdleService.h>
#include <freertos/FreeRTOS.h>
#include <freertos/task.h>
@@ -76,19 +77,21 @@ static bool display_ready(McpSystemState& state) {
static bool ensureOverrideScreen() {
auto& state = getState();
auto mcpSettings = settings::mcp::loadOrGetDefault();
if (!mcpSettings.mcpOverrideEnabled) {
return false;
}
if (state.drawArea == nullptr) {
LOGGER.info("MCP Screen Override triggered: starting McpOverrideApp");
tt::app::start("one.tactility.mcpscreen");
// Wait up to 500ms for UI to initialize
for (int i = 0; i < 10; ++i) {
vTaskDelay(pdMS_TO_TICKS(50));
if (state.drawArea != nullptr) {
break;
// Activate the MCP Screen screensaver via DisplayIdle
auto idleService = service::displayidle::findService();
if (idleService) {
LOGGER.info("MCP draw triggered: starting MCP screensaver");
idleService->startScreensaver();
// Wait up to 500ms for the canvas to be registered
for (int i = 0; i < 10; ++i) {
vTaskDelay(pdMS_TO_TICKS(50));
if (state.drawArea != nullptr) {
break;
}
}
} else {
LOGGER.warn("DisplayIdle service not found; cannot activate MCP screensaver");
}
}
return state.drawArea != nullptr;
Tactility/Source/service/displayidle/DisplayIdle.cpp
+4
View File
@@ -7,6 +7,7 @@
#include "MatrixRainScreensaver.h"
#include "MystifyScreensaver.h"
#include "StackChanScreensaver.h"
#include "McpScreensaver.h"
#include <Tactility/Logger.h>
#include <Tactility/CoreDefines.h>
@@ -103,6 +104,9 @@ void DisplayIdleService::activateScreensaver() {
case settings::display::ScreensaverType::StackChan:
screensaver = std::make_unique<StackChanScreensaver>();
break;
case settings::display::ScreensaverType::McpScreen:
screensaver = std::make_unique<McpScreensaver>();
break;
case settings::display::ScreensaverType::None:
default:
// Just black screen, no animated screensaver
Tactility/Source/service/displayidle/McpScreensaver.cpp
+98
View File
@@ -0,0 +1,98 @@
#ifdef ESP_PLATFORM
#include "McpScreensaver.h"
#include <Tactility/mcp/McpSystem.h>
#include <Tactility/Logger.h>
#include <esp_heap_caps.h>
namespace tt::service::displayidle {
static const auto LOGGER = Logger("McpScreensaver");
void McpScreensaver::start(lv_obj_t* overlay, lv_coord_t screenW, lv_coord_t screenH) {
auto& state = mcp::getState();
// Full-screen canvas on the overlay
lv_obj_t* canvas = lv_canvas_create(overlay);
lv_obj_set_size(canvas, screenW, screenH);
lv_obj_set_pos(canvas, 0, 0);
lv_obj_set_style_radius(canvas, 0, LV_PART_MAIN);
lv_obj_set_style_border_width(canvas, 0, LV_PART_MAIN);
lv_obj_set_style_pad_all(canvas, 0, LV_PART_MAIN);
lv_obj_remove_flag(canvas, LV_OBJ_FLAG_SCROLLABLE);
// Allocate framebuffer (prefer SPIRAM)
size_t requiredSize = (size_t)screenW * screenH * sizeof(uint16_t);
framebuffer = (uint16_t*)heap_caps_malloc(requiredSize, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
if (framebuffer == nullptr) {
framebuffer = (uint16_t*)heap_caps_malloc(requiredSize, MALLOC_CAP_8BIT);
}
framebufferSize = (framebuffer != nullptr) ? requiredSize : 0;
if (framebuffer == nullptr) {
LOGGER.error("Failed to allocate {}B framebuffer", (unsigned)requiredSize);
lv_obj_t* err = lv_label_create(canvas);
lv_label_set_text(err, "Framebuffer alloc failed");
lv_obj_center(err);
return;
}
// Fill with a dark slate background
size_t pixelCount = (size_t)screenW * screenH;
for (size_t i = 0; i < pixelCount; ++i) {
framebuffer[i] = 0x18E3; // dark blue-grey
}
lv_canvas_set_buffer(canvas, framebuffer, screenW, screenH, LV_COLOR_FORMAT_RGB565);
// Waiting label (removed on first MCP draw via lv_obj_clean)
lv_obj_t* waitLabel = lv_label_create(canvas);
lv_label_set_text(waitLabel, "Waiting for LLM...");
lv_obj_set_style_text_color(waitLabel, lv_color_white(), LV_PART_MAIN);
lv_obj_align(waitLabel, LV_ALIGN_CENTER, 0, -20);
lv_obj_t* resLabel = lv_label_create(canvas);
lv_label_set_text_fmt(resLabel, "Display: %dx%d", (int)screenW, (int)screenH);
lv_obj_set_style_text_color(resLabel, lv_palette_lighten(LV_PALETTE_BLUE, 3), LV_PART_MAIN);
lv_obj_align(resLabel, LV_ALIGN_CENTER, 0, 10);
// Register with McpSystemState
std::lock_guard<std::mutex> lock(state.mutex);
state.drawArea = canvas;
state.framebuffer = framebuffer;
state.framebufferSize = framebufferSize;
state.displayWidth = (uint16_t)screenW;
state.displayHeight = (uint16_t)screenH;
state.drawWidth = (uint16_t)screenW;
state.drawHeight = (uint16_t)screenH;
// Don't reset overrideActive — if the LLM already drew, we keep the content
LOGGER.info("McpScreensaver started ({}x{})", (int)screenW, (int)screenH);
}
void McpScreensaver::stop() {
auto& state = mcp::getState();
{
std::lock_guard<std::mutex> lock(state.mutex);
state.drawArea = nullptr;
state.framebuffer = nullptr;
state.framebufferSize = 0;
state.overrideActive = false;
}
if (framebuffer != nullptr) {
heap_caps_free(framebuffer);
framebuffer = nullptr;
framebufferSize = 0;
}
LOGGER.info("McpScreensaver stopped");
}
void McpScreensaver::update(lv_coord_t /*screenW*/, lv_coord_t /*screenH*/) {
// MCP draws on demand via HTTP — no per-frame animation needed
}
} // namespace tt::service::displayidle
#endif // ESP_PLATFORM
Tactility/Source/service/displayidle/McpScreensaver.h
+30
View File
@@ -0,0 +1,30 @@
#pragma once
#ifdef ESP_PLATFORM
#include "Screensaver.h"
#include <cstdint>
namespace tt::service::displayidle {
/**
* MCP Screen screensaver.
* Creates a full-screen LVGL canvas on the overlay and registers it in
* McpSystemState so that MCP HTTP draw commands can paint to it.
* Dismissed by a touch event (handled by the parent DisplayIdle overlay).
*/
class McpScreensaver final : public Screensaver {
uint16_t* framebuffer = nullptr;
size_t framebufferSize = 0;
public:
McpScreensaver() = default;
~McpScreensaver() override = default;
void start(lv_obj_t* overlay, lv_coord_t screenW, lv_coord_t screenH) override;
void stop() override;
void update(lv_coord_t screenW, lv_coord_t screenH) override;
};
} // namespace tt::service::displayidle
#endif // ESP_PLATFORM
Tactility/Source/settings/DisplaySettings.cpp
+5
View File
@@ -78,6 +78,8 @@ static std::string toString(ScreensaverType type) {
return "MatrixRain";
case StackChan:
return "StackChan";
case McpScreen:
return "McpScreen";
default:
std::unreachable();
}
@@ -99,6 +101,9 @@ static bool fromString(const std::string& str, ScreensaverType& type) {
} else if (str == "StackChan") {
type = ScreensaverType::StackChan;
return true;
} else if (str == "McpScreen") {
type = ScreensaverType::McpScreen;
return true;
} else {
return false;
}
Tactility/Source/settings/McpSettings.cpp
+1 -10
View File
@@ -11,7 +11,6 @@ static const auto LOGGER = Logger("McpSettings");
constexpr auto* SETTINGS_FILE = "/data/service/mcp/settings.properties";
constexpr auto* KEY_MCP_ENABLED = "mcpEnabled";
constexpr auto* KEY_MCP_OVERRIDE_ENABLED = "mcpOverrideEnabled";
bool load(McpSettings& settings) {
std::map<std::string, std::string> map;
@@ -20,23 +19,16 @@ bool load(McpSettings& settings) {
}
auto mcp_enabled = map.find(KEY_MCP_ENABLED);
auto mcp_override = map.find(KEY_MCP_OVERRIDE_ENABLED);
settings.mcpEnabled = (mcp_enabled != map.end())
? (mcp_enabled->second == "1" || mcp_enabled->second == "true")
: false;
settings.mcpOverrideEnabled = (mcp_override != map.end())
? (mcp_override->second == "1" || mcp_override->second == "true")
: true; // Default true
return true;
}
McpSettings getDefault() {
return McpSettings{
.mcpEnabled = false,
.mcpOverrideEnabled = true
.mcpEnabled = false
};
}
@@ -55,7 +47,6 @@ bool save(const McpSettings& settings) {
std::map<std::string, std::string> map;
map[KEY_MCP_ENABLED] = settings.mcpEnabled ? "true" : "false";
map[KEY_MCP_OVERRIDE_ENABLED] = settings.mcpOverrideEnabled ? "true" : "false";
if (!file::savePropertiesFile(SETTINGS_FILE, map)) {
LOGGER.error("Failed to save MCP settings to {}", SETTINGS_FILE);