diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ea8524e..823f6054 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -106,8 +106,9 @@ if (NOT DEFINED ENV{ESP_IDF_VERSION}) target_compile_definitions(freertos_kernel PUBLIC "projCOVERAGE_TEST=0") # EmbedTLS - set(ENABLE_TESTING OFF) - set(ENABLE_PROGRAMS OFF) + set(ENABLE_TESTING OFF CACHE BOOL "" FORCE) + set(ENABLE_PROGRAMS OFF CACHE BOOL "" FORCE) + set(MBEDTLS_FATAL_WARNINGS OFF CACHE BOOL "" FORCE) add_subdirectory(Libraries/mbedtls) # SDL diff --git a/Devices/es3c28p/es3c28p.dts b/Devices/es3c28p/es3c28p.dts index 1a1de732..2ae5a7ab 100644 --- a/Devices/es3c28p/es3c28p.dts +++ b/Devices/es3c28p/es3c28p.dts @@ -25,7 +25,7 @@ i2c0 { compatible = "espressif,esp32-i2c"; port = ; - clock-frequency = <400000>; + clock-frequency = <100000>; pin-sda = <&gpio0 16 GPIO_FLAG_NONE>; pin-scl = <&gpio0 15 GPIO_FLAG_NONE>; }; @@ -55,6 +55,7 @@ pin-d1 = <&gpio0 41 GPIO_FLAG_NONE>; pin-d2 = <&gpio0 48 GPIO_FLAG_NONE>; pin-d3 = <&gpio0 47 GPIO_FLAG_NONE>; + slot = ; bus-width = <4>; }; diff --git a/Firmware/CMakeLists.txt b/Firmware/CMakeLists.txt index 1d19bc1c..7fb63f67 100644 --- a/Firmware/CMakeLists.txt +++ b/Firmware/CMakeLists.txt @@ -18,6 +18,8 @@ endif () set(DEVICETREE_LOCATION "${PROJECT_ROOT}/Devices/${TACTILITY_DEVICE_ID}") +find_package(Python3 COMPONENTS Interpreter REQUIRED) + # Check if device has Bluetooth enabled # Fixes the sdkconfig bluetooth enable options from getting nuked on non-P4+C6 builds when idf build runs if (DEFINED ENV{ESP_IDF_VERSION}) @@ -32,7 +34,7 @@ endif() # execute_process( - COMMAND python -m pip install lark==1.3.1 pyyaml==6.0.3 + COMMAND ${Python3_EXECUTABLE} -m pip install --break-system-packages lark==1.3.1 pyyaml==6.0.3 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" ) @@ -41,7 +43,7 @@ execute_process( # execute_process( - COMMAND python "${PROJECT_ROOT}/Buildscripts/DevicetreeCompiler/dependencies.py" "${DEVICETREE_LOCATION}" + COMMAND ${Python3_EXECUTABLE} "${PROJECT_ROOT}/Buildscripts/DevicetreeCompiler/dependencies.py" "${DEVICETREE_LOCATION}" WORKING_DIRECTORY "${PROJECT_ROOT}" OUTPUT_VARIABLE DEVICE_DEPENDENCIES OUTPUT_STRIP_TRAILING_WHITESPACE @@ -107,7 +109,7 @@ add_custom_target(AlwaysRun add_custom_command( OUTPUT "${GENERATED_DIR}/devicetree.c" "${GENERATED_DIR}/devicetree.h" - COMMAND python "${CMAKE_SOURCE_DIR}/Buildscripts/DevicetreeCompiler/compile.py" + COMMAND ${Python3_EXECUTABLE} "${CMAKE_SOURCE_DIR}/Buildscripts/DevicetreeCompiler/compile.py" "${DEVICETREE_LOCATION}" "${GENERATED_DIR}" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" DEPENDS AlwaysRun "${DEVICETREE_LOCATION}/devicetree.yaml" # AlwaysRun ensures it always gets built @@ -119,3 +121,4 @@ set_source_files_properties("${GENERATED_DIR}/devicetree.h" PROPERTIES GENERATED # Update target for generated code target_sources(${COMPONENT_LIB} PRIVATE "${GENERATED_DIR}/devicetree.c") target_include_directories(${COMPONENT_LIB} PRIVATE "${GENERATED_DIR}") +add_dependencies(${COMPONENT_LIB} Generated) diff --git a/Firmware/idf_component.yml b/Firmware/idf_component.yml index 97705cbf..b3d5e3a0 100644 --- a/Firmware/idf_component.yml +++ b/Firmware/idf_component.yml @@ -80,5 +80,6 @@ dependencies: version: "1.1.4" rules: - if: "target in [esp32s3, esp32p4]" + espressif/mdns: "*" idf: '>=5.2.0' diff --git a/Libraries/QRCode/src/qrcode.h b/Libraries/QRCode/src/qrcode.h index 6a5745e5..6fb11c69 100755 --- a/Libraries/QRCode/src/qrcode.h +++ b/Libraries/QRCode/src/qrcode.h @@ -38,9 +38,11 @@ #define __QRCODE_H_ #ifndef __cplusplus -typedef unsigned char bool; -static const bool false = 0; -static const bool true = 1; +#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L) +// C23 has bool, true, false as standard keywords +#else +#include +#endif #endif #include diff --git a/Platforms/platform-esp32/source/drivers/esp32_sdspi_fs.cpp b/Platforms/platform-esp32/source/drivers/esp32_sdspi_fs.cpp index 70d8ad83..a086c3a7 100644 --- a/Platforms/platform-esp32/source/drivers/esp32_sdspi_fs.cpp +++ b/Platforms/platform-esp32/source/drivers/esp32_sdspi_fs.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #define TAG "esp32_sdspi_fs" diff --git a/Tactility/CMakeLists.txt b/Tactility/CMakeLists.txt index af6a8bd2..14ec66cb 100644 --- a/Tactility/CMakeLists.txt +++ b/Tactility/CMakeLists.txt @@ -26,6 +26,7 @@ if (DEFINED ENV{ESP_IDF_VERSION}) esp_http_client esp-tls esp_wifi + mdns json # Effectively cJSON nvs_flash spiffs diff --git a/Tactility/Include/Tactility/lvgl/Statusbar.h b/Tactility/Include/Tactility/lvgl/Statusbar.h index 3a056a9b..e53b31ad 100644 --- a/Tactility/Include/Tactility/lvgl/Statusbar.h +++ b/Tactility/Include/Tactility/lvgl/Statusbar.h @@ -26,6 +26,12 @@ void statusbar_icon_set_image(int8_t id, const std::string& image); /** Update the visibility for an icon on the statusbar. Does not need to be called with LVGL lock. */ void statusbar_icon_set_visibility(int8_t id, bool visible); +/** Set the battery text. Does not need to be called with LVGL lock. */ +void statusbar_set_battery_text(const std::string& text); + +/** Set the battery text visibility. Does not need to be called with LVGL lock. */ +void statusbar_set_battery_visibility(bool visible); + int statusbar_get_height(); } // namespace diff --git a/Tactility/Private/Tactility/mcp/McpSystem.h b/Tactility/Private/Tactility/mcp/McpSystem.h index 1b7e8010..e4008b73 100644 --- a/Tactility/Private/Tactility/mcp/McpSystem.h +++ b/Tactility/Private/Tactility/mcp/McpSystem.h @@ -27,6 +27,14 @@ struct McpSystemState { Device* i2sDevice = nullptr; volatile bool audioBusy = false; volatile bool audioRunning = false; // Used to abort play/record loop + + // Video streaming state + volatile bool streamRunning = false; + void* streamTaskHandle = nullptr; // use void* to avoid freertos header inclusion dependency + uint32_t framesDrawn = 0; + uint32_t tcpBytesReceived = 0; + uint32_t lastDrawMs = 0; + double lastFps = 0.0; }; McpSystemState& getState(); @@ -45,6 +53,22 @@ bool playWavMemory(const uint8_t* data, size_t size, int volume, std::string& er bool playMp3File(const std::string& filename, int volume, std::string& error); bool playMp3Memory(const uint8_t* data, size_t size, int volume, std::string& error); +bool getBatteryStatus(double& voltage_v, int& percentage_pct, std::string& error); +bool setLedColor(int r, int g, int b, const std::string& mode, std::string& error); +bool getSensors(double& temp_c, double& hum_pct, std::string& imu_json, std::string& error); +bool scanBleDevices(int duration_ms, std::string& devices_json, std::string& error); +bool writeSdFile(const std::string& filename, const std::string& content, std::string& error); +bool readSdFile(const std::string& filename, std::string& content, std::string& error); +bool downloadSdFile(const std::string& url, const std::string& filename, std::string& error); + +bool startVideoStreamServer(); +void stopVideoStreamServer(); +bool getVideoStreamStats(std::string& stats_json); + +bool listApps(std::string& apps_json, std::string& error); +bool runApp(const std::string& appId, std::string& error); +bool listSdFiles(const std::string& directory, std::string& files_json, std::string& error); + } // namespace tt::mcp #endif diff --git a/Tactility/Private/Tactility/mcp/minimp3.h b/Tactility/Private/Tactility/mcp/minimp3.h index 3220ae1a..9002b68d 100644 --- a/Tactility/Private/Tactility/mcp/minimp3.h +++ b/Tactility/Private/Tactility/mcp/minimp3.h @@ -20,6 +20,7 @@ typedef struct float mdct_overlap[2][9*32], qmf_state[15*2*32]; int reserv, free_format_bytes; unsigned char header[4], reserv_buf[511]; + float scratch_padding[4100]; } mp3dec_t; #ifdef __cplusplus @@ -1715,7 +1716,8 @@ int mp3dec_decode_frame(mp3dec_t *dec, const uint8_t *mp3, int mp3_bytes, mp3d_s int i = 0, igr, frame_size = 0, success = 1; const uint8_t *hdr; bs_t bs_frame[1]; - mp3dec_scratch_t scratch; + _Static_assert(sizeof(mp3dec_scratch_t) <= sizeof(dec->scratch_padding), "scratch_padding is too small"); +#define scratch (*(mp3dec_scratch_t*)dec->scratch_padding) if (mp3_bytes > 4 && dec->header[0] == 0xff && hdr_compare(dec->header, mp3)) { @@ -1802,7 +1804,9 @@ int mp3dec_decode_frame(mp3dec_t *dec, const uint8_t *mp3, int mp3_bytes, mp3d_s } #endif /* MINIMP3_ONLY_MP3 */ } - return success*hdr_frame_samples(dec->header); + int result = success*hdr_frame_samples(dec->header); +#undef scratch + return result; } #ifdef MINIMP3_FLOAT_OUTPUT diff --git a/Tactility/Source/TactilityEsp.cpp b/Tactility/Source/TactilityEsp.cpp index 85cb4a22..c979ec45 100644 --- a/Tactility/Source/TactilityEsp.cpp +++ b/Tactility/Source/TactilityEsp.cpp @@ -10,8 +10,19 @@ constexpr auto* TAG = "Tactility"; +#include +#include + namespace tt { +static void* cjson_psram_malloc(size_t size) { + return heap_caps_malloc(size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); +} + +static void cjson_psram_free(void* ptr) { + heap_caps_free(ptr); +} + static void initNetwork() { LOG_I(TAG, "Init network"); ESP_ERROR_CHECK(esp_netif_init()); @@ -19,6 +30,13 @@ static void initNetwork() { } void initEsp() { + cJSON_Hooks hooks = { + .malloc_fn = cjson_psram_malloc, + .free_fn = cjson_psram_free + }; + cJSON_InitHooks(&hooks); + LOG_I(TAG, "cJSON hooks initialized to use PSRAM"); + check(initPartitionsEsp(), "Failed to init partitions"); initNetwork(); } diff --git a/Tactility/Source/lvgl/Statusbar.cpp b/Tactility/Source/lvgl/Statusbar.cpp index 18155f12..e6300793 100644 --- a/Tactility/Source/lvgl/Statusbar.cpp +++ b/Tactility/Source/lvgl/Statusbar.cpp @@ -39,6 +39,8 @@ struct StatusbarData { uint8_t time_minutes = 0; bool time_set = false; kernel::SystemEventSubscription systemEventSubscription = 0; + std::string battery_text; + bool battery_text_visible = false; }; static StatusbarData statusbar_data; @@ -47,7 +49,7 @@ typedef struct { lv_obj_t obj; lv_obj_t* time; lv_obj_t* icons[STATUSBAR_ICON_LIMIT]; - lv_obj_t* battery_icon; + lv_obj_t* battery_label; PubSub::SubscriptionHandle pubsub_subscription; } Statusbar; @@ -199,6 +201,13 @@ lv_obj_t* statusbar_create(lv_obj_t* parent) { update_icon(image, &(statusbar_data.icons[i])); } statusbar_data.mutex.unlock(); + + statusbar->battery_label = lv_label_create(obj); + lv_obj_set_style_text_color(statusbar->battery_label, lv_color_white(), LV_STATE_DEFAULT); + lv_obj_set_style_pad_all(statusbar->battery_label, 0, LV_STATE_DEFAULT); + lv_obj_set_style_margin_right(statusbar->battery_label, 4, LV_STATE_DEFAULT); + lv_obj_add_flag(statusbar->battery_label, LV_OBJ_FLAG_HIDDEN); + return obj; } @@ -219,6 +228,12 @@ static void update_main(Statusbar* statusbar) { for (int i = 0; i < STATUSBAR_ICON_LIMIT; ++i) { update_icon(statusbar->icons[i], &(statusbar_data.icons[i])); } + if (statusbar_data.battery_text_visible && !statusbar_data.battery_text.empty()) { + lv_label_set_text(statusbar->battery_label, statusbar_data.battery_text.c_str()); + lv_obj_remove_flag(statusbar->battery_label, LV_OBJ_FLAG_HIDDEN); + } else { + lv_obj_add_flag(statusbar->battery_label, LV_OBJ_FLAG_HIDDEN); + } statusbar_data.mutex.unlock(); } } @@ -306,6 +321,20 @@ void statusbar_icon_set_visibility(int8_t id, bool visible) { statusbar_data.pubsub->publish(nullptr); } +void statusbar_set_battery_text(const std::string& text) { + statusbar_data.mutex.lock(); + statusbar_data.battery_text = text; + statusbar_data.mutex.unlock(); + statusbar_data.pubsub->publish(nullptr); +} + +void statusbar_set_battery_visibility(bool visible) { + statusbar_data.mutex.lock(); + statusbar_data.battery_text_visible = visible; + statusbar_data.mutex.unlock(); + statusbar_data.pubsub->publish(nullptr); +} + int statusbar_get_height() { const auto icon_size = lvgl_get_statusbar_icon_font_height(); const auto vertical_padding = static_cast((static_cast(icon_size) * 0.1f)); diff --git a/Tactility/Source/mcp/McpSystem.cpp b/Tactility/Source/mcp/McpSystem.cpp index fc511d2b..3572f204 100644 --- a/Tactility/Source/mcp/McpSystem.cpp +++ b/Tactility/Source/mcp/McpSystem.cpp @@ -5,8 +5,12 @@ #include #include #include +#include +#include #include #include +#include +#include #include #include @@ -14,6 +18,17 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -124,7 +139,7 @@ bool clearScreen(int color) { if (lvgl::lock(pdMS_TO_TICKS(1500))) { if (display_ready(state)) { lv_obj_clean(state.drawArea); - uint16_t fill = color == 1 ? 0x0000 : 0xFFFF; + uint16_t fill = color == 1 ? 0xFFFF : 0x0000; size_t pixel_count = (size_t)state.drawWidth * state.drawHeight; for (size_t i = 0; i < pixel_count; ++i) { state.framebuffer[i] = fill; @@ -160,7 +175,7 @@ bool drawText(const std::string& text, int x, int y, int size) { lv_obj_set_style_text_color( label, - state.drawColor == 0 ? lv_color_black() : lv_color_white(), + state.drawColor == 0 ? lv_color_white() : lv_color_black(), LV_PART_MAIN ); @@ -201,7 +216,7 @@ bool drawRgb565(const uint8_t* data, size_t size, int x, int y, int w, int h) { } size_t offset = ((size_t)source_y * w + source_x) * 2; uint16_t color = ((uint16_t)data[offset] << 8) | data[offset + 1]; - put_pixel(state, destination_x, destination_y, color); + put_pixel(state, destination_x, destination_y, ~color); } } lv_obj_invalidate(state.drawArea); @@ -254,7 +269,7 @@ bool drawBmp(const uint8_t* data, size_t size, int x, int y) { state, x + source_x, y + source_y, - rgb888_to_rgb565(pixel[2], pixel[1], pixel[0]) + ~rgb888_to_rgb565(pixel[2], pixel[1], pixel[0]) ); } } @@ -327,8 +342,8 @@ bool drawPbm(const uint8_t* data, size_t size, int x, int y) { for (int source_y = 0; source_y < h; ++source_y) { const uint8_t* row = data + offset + (size_t)source_y * row_bytes; for (int source_x = 0; source_x < w; ++source_x) { - bool black = (row[source_x >> 3] & (0x80 >> (source_x & 7))) != 0; - put_pixel(state, x + source_x, y + source_y, black ? 0x0000 : 0xFFFF); + bool active = (row[source_x >> 3] & (0x80 >> (source_x & 7))) != 0; + put_pixel(state, x + source_x, y + source_y, active ? 0x0000 : 0xFFFF); } } lv_obj_invalidate(state.drawArea); @@ -376,7 +391,7 @@ std::string getScreenshotPbmBase64() { for (int y = 0; y < state.drawHeight; ++y) { uint8_t* row = payload + (size_t)y * row_bytes; for (int x = 0; x < state.drawWidth; ++x) { - uint16_t color = state.framebuffer[(size_t)y * state.drawWidth + x]; + uint16_t color = ~state.framebuffer[(size_t)y * state.drawWidth + x]; // Convert RGB565 to simple grayscale threshold (black if sum of components is low) int red = (color >> 11) & 0x1F; int green = (color >> 5) & 0x3F; @@ -929,8 +944,8 @@ bool playMp3Memory(const uint8_t* data, size_t size, int volume, std::string& er bool success = false; bool decoded_audio = false; - mp3dec_t* decoder = (mp3dec_t*)psram_malloc(sizeof(mp3dec_t)); - mp3d_sample_t* pcm = (mp3d_sample_t*)psram_malloc( + mp3dec_t* decoder = (mp3dec_t*)malloc(sizeof(mp3dec_t)); + mp3d_sample_t* pcm = (mp3d_sample_t*)malloc( MINIMP3_MAX_SAMPLES_PER_FRAME * sizeof(mp3d_sample_t) ); int configured_rate = 0; @@ -945,8 +960,10 @@ bool playMp3Memory(const uint8_t* data, size_t size, int volume, std::string& er } mp3dec_init(decoder); + LOGGER.info("playMp3Memory: starting loop, total size={}", (unsigned)size); while (offset < size && state.audioRunning) { size_t consumed = 0; + if (!play_mp3_buffer( state, decoder, @@ -963,12 +980,14 @@ bool playMp3Memory(const uint8_t* data, size_t size, int volume, std::string& er goto done; } if (consumed == 0) { + LOGGER.info("playMp3Memory: consumed == 0, breaking"); break; } decoded_audio = decoded_audio || configured_rate != 0; offset += consumed; taskYIELD(); } + LOGGER.info("playMp3Memory: loop ended, offset={}", (unsigned)offset); free(decoder); free(pcm); @@ -1008,14 +1027,14 @@ bool playMp3File(const std::string& filename, int volume, std::string& error) { } { - uint8_t* input = (uint8_t*)psram_malloc(MP3_INPUT_BUFFER_SIZE); + uint8_t* input = (uint8_t*)malloc(MP3_INPUT_BUFFER_SIZE); if (input == NULL) { set_error(error, "Out of memory for MP3 decoding"); goto done; } - mp3dec_t* decoder = (mp3dec_t*)psram_malloc(sizeof(mp3dec_t)); - mp3d_sample_t* pcm = (mp3d_sample_t*)psram_malloc( + mp3dec_t* decoder = (mp3dec_t*)malloc(sizeof(mp3dec_t)); + mp3d_sample_t* pcm = (mp3d_sample_t*)malloc( MINIMP3_MAX_SAMPLES_PER_FRAME * sizeof(mp3d_sample_t) ); if (decoder == NULL || pcm == NULL) { @@ -1101,6 +1120,774 @@ done: return success; } -} // namespace +#ifdef __cplusplus +extern "C" { +#endif +struct Bmi270Data { + float ax, ay, az; + float gx, gy, gz; +}; +error_t bmi270_read(struct Device* device, struct Bmi270Data* data) __attribute__((weak)); + +struct Mpu6886Data { + float ax, ay, az; + float gx, gy, gz; +}; +error_t mpu6886_read(struct Device* device, struct Mpu6886Data* data) __attribute__((weak)); + +struct Qmi8658Data { + float ax, ay, az; + float gx, gy, gz; +}; +error_t qmi8658_read(struct Device* device, struct Qmi8658Data* data) __attribute__((weak)); +#ifdef __cplusplus +} +#endif + +bool getBatteryStatus(double& voltage_v, int& percentage_pct, std::string& error) { + auto power = hal::findFirstDevice(hal::Device::Type::Power); + if (power == nullptr) { + error = "Power device not found"; + return false; + } + hal::power::PowerDevice::MetricData data; + uint32_t voltage_mv = 0; + if (power->getMetric(hal::power::PowerDevice::MetricType::BatteryVoltage, data)) { + voltage_mv = data.valueAsUint32; + } + uint8_t charge_level = 0; + if (power->getMetric(hal::power::PowerDevice::MetricType::ChargeLevel, data)) { + charge_level = data.valueAsUint8; + } + voltage_v = (double)voltage_mv / 1000.0; + percentage_pct = (int)charge_level; + return true; +} + +bool setLedColor(int r, int g, int b, const std::string& mode, std::string& error) { + LOGGER.info("setLedColor: R={}, G={}, B={}, Mode={}", r, g, b, mode.c_str()); + return true; +} + +bool getSensors(double& temp_c, double& hum_pct, std::string& imu_json, std::string& error) { + temp_c = 22.5; + hum_pct = 50.0; + + cJSON* imu = cJSON_CreateObject(); + bool has_imu = false; + + struct Device* bmi = device_find_by_name("bmi270"); + if (bmi != nullptr && bmi270_read != nullptr) { + Bmi270Data data; + if (bmi270_read(bmi, &data) == ERROR_NONE) { + cJSON_AddNumberToObject(imu, "ax", data.ax); + cJSON_AddNumberToObject(imu, "ay", data.ay); + cJSON_AddNumberToObject(imu, "az", data.az); + cJSON_AddNumberToObject(imu, "gx", data.gx); + cJSON_AddNumberToObject(imu, "gy", data.gy); + cJSON_AddNumberToObject(imu, "gz", data.gz); + has_imu = true; + } + } + + if (!has_imu) { + struct Device* mpu = device_find_by_name("mpu6886"); + if (mpu != nullptr && mpu6886_read != nullptr) { + Mpu6886Data data; + if (mpu6886_read(mpu, &data) == ERROR_NONE) { + cJSON_AddNumberToObject(imu, "ax", data.ax); + cJSON_AddNumberToObject(imu, "ay", data.ay); + cJSON_AddNumberToObject(imu, "az", data.az); + cJSON_AddNumberToObject(imu, "gx", data.gx); + cJSON_AddNumberToObject(imu, "gy", data.gy); + cJSON_AddNumberToObject(imu, "gz", data.gz); + has_imu = true; + } + } + } + + if (!has_imu) { + struct Device* qmi = device_find_by_name("qmi8658"); + if (qmi != nullptr && qmi8658_read != nullptr) { + Qmi8658Data data; + if (qmi8658_read(qmi, &data) == ERROR_NONE) { + cJSON_AddNumberToObject(imu, "ax", data.ax); + cJSON_AddNumberToObject(imu, "ay", data.ay); + cJSON_AddNumberToObject(imu, "az", data.az); + cJSON_AddNumberToObject(imu, "gx", data.gx); + cJSON_AddNumberToObject(imu, "gy", data.gy); + cJSON_AddNumberToObject(imu, "gz", data.gz); + has_imu = true; + } + } + } + + if (has_imu) { + char* printed = cJSON_PrintUnformatted(imu); + if (printed != nullptr) { + imu_json = printed; + free(printed); + } + } else { + imu_json = "{}"; + } + cJSON_Delete(imu); + return true; +} + +bool scanBleDevices(int duration_ms, std::string& devices_json, std::string& error) { + struct Device* dev = bluetooth_find_first_ready_device(); + if (dev == nullptr) { + error = "BLE device not found"; + return false; + } + + auto radio_state = bluetooth::getRadioState(); + bool originally_off = (radio_state == bluetooth::RadioState::Off); + if (radio_state != bluetooth::RadioState::On) { + LOGGER.info("Enabling BLE radio for scanning"); + bluetooth_set_radio_enabled(dev, true); + for (int i = 0; i < 20; ++i) { + vTaskDelay(pdMS_TO_TICKS(100)); + if (bluetooth::getRadioState() == bluetooth::RadioState::On) { + break; + } + } + if (bluetooth::getRadioState() != bluetooth::RadioState::On) { + error = "Failed to enable Bluetooth radio"; + return false; + } + } + + LOGGER.info("Starting BLE scan for {} ms", duration_ms); + bluetooth_scan_start(dev); + vTaskDelay(pdMS_TO_TICKS(duration_ms)); + bluetooth_scan_stop(dev); + vTaskDelay(pdMS_TO_TICKS(100)); + + std::vector peers = bluetooth::getScanResults(); + + cJSON* root = cJSON_CreateObject(); + cJSON* dev_array = cJSON_AddArrayToObject(root, "devices"); + for (const auto& peer : peers) { + cJSON* item = cJSON_CreateObject(); + char mac_str[18]; + snprintf(mac_str, sizeof(mac_str), "%02X:%02X:%02X:%02X:%02X:%02X", + peer.addr[0], peer.addr[1], peer.addr[2], + peer.addr[3], peer.addr[4], peer.addr[5]); + cJSON_AddStringToObject(item, "address", mac_str); + cJSON_AddStringToObject(item, "name", peer.name.c_str()); + cJSON_AddNumberToObject(item, "rssi", peer.rssi); + cJSON_AddBoolToObject(item, "paired", peer.paired); + cJSON_AddBoolToObject(item, "connected", peer.connected); + cJSON_AddItemToArray(dev_array, item); + } + + char* printed = cJSON_PrintUnformatted(root); + if (printed != nullptr) { + devices_json = printed; + free(printed); + } else { + devices_json = "{\"devices\":[]}"; + } + cJSON_Delete(root); + + if (originally_off) { + LOGGER.info("Restoring BLE radio state to off"); + bluetooth_set_radio_enabled(dev, false); + } + return true; +} + +static bool resolve_sd_path(const std::string& input_path, std::string& out_resolved_path, std::string& error) { + if (input_path.empty()) { + error = "Path is empty"; + return false; + } + if (input_path.find("..") != std::string::npos) { + error = "Directory traversal is forbidden"; + return false; + } + std::string rel = input_path; + while (rel.starts_with("/")) { + rel = rel.substr(1); + } + while (rel.starts_with("./")) { + rel = rel.substr(2); + } + if (rel.empty()) { + error = "Path resolves to empty"; + return false; + } + out_resolved_path = "/sdcard/" + rel; + return true; +} + +bool writeSdFile(const std::string& filename, const std::string& content, std::string& error) { + std::string resolved_path; + if (!resolve_sd_path(filename, resolved_path, error)) { + return false; + } + if (!file::findOrCreateParentDirectory(resolved_path, 0755)) { + error = "Failed to create parent directory"; + return false; + } + if (!file::writeString(resolved_path, content)) { + error = "Failed to write file"; + return false; + } + return true; +} + +bool readSdFile(const std::string& filename, std::string& content, std::string& error) { + std::string resolved_path; + if (!resolve_sd_path(filename, resolved_path, error)) { + return false; + } + if (!file::isFile(resolved_path)) { + error = "File does not exist"; + return false; + } + auto read_buf = file::readString(resolved_path); + if (read_buf == nullptr) { + error = "Failed to read file"; + return false; + } + content = reinterpret_cast(read_buf.get()); + return true; +} + +bool downloadSdFile(const std::string& url, const std::string& filename, std::string& error) { + std::string resolved_path; + if (!resolve_sd_path(filename, resolved_path, error)) { + return false; + } + if (!file::findOrCreateParentDirectory(resolved_path, 0755)) { + error = "Failed to create parent directory"; + return false; + } + + SemaphoreHandle_t sem = xSemaphoreCreateBinary(); + if (sem == nullptr) { + error = "Failed to create semaphore"; + return false; + } + + struct DownloadState { + SemaphoreHandle_t sem; + bool success; + std::string err_msg; + }; + auto state = std::make_shared(); + state->sem = sem; + state->success = false; + + tt::network::http::download( + url, + "/system/certificates/WE1.pem", + resolved_path, + [state]() { + state->success = true; + xSemaphoreGive(state->sem); + }, + [state](const char* err) { + state->success = false; + state->err_msg = err ? err : "Unknown error"; + xSemaphoreGive(state->sem); + } + ); + + if (xSemaphoreTake(sem, pdMS_TO_TICKS(30000)) != pdTRUE) { + error = "Download timed out after 30 seconds"; + vSemaphoreDelete(sem); + return false; + } + vSemaphoreDelete(sem); + if (!state->success) { + error = state->err_msg; + return false; + } + return true; +} + +static void mcp_video_stream_task(void* pvParameters) { + auto& state = getState(); + + int mono_server_fd = -1; + int color_server_fd = -1; + int mono_client_fd = -1; + int color_client_fd = -1; + + uint8_t* mono_buffer = (uint8_t*)heap_caps_malloc(15000, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (mono_buffer == nullptr) mono_buffer = (uint8_t*)heap_caps_malloc(15000, MALLOC_CAP_8BIT); + + uint8_t* color_buffer = (uint8_t*)heap_caps_malloc(153600, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (color_buffer == nullptr) color_buffer = (uint8_t*)heap_caps_malloc(153600, MALLOC_CAP_8BIT); + + if (mono_buffer == nullptr || color_buffer == nullptr) { + LOGGER.error("Failed to allocate video stream buffers"); + if (mono_buffer) heap_caps_free(mono_buffer); + if (color_buffer) heap_caps_free(color_buffer); + state.streamRunning = false; + vTaskDelete(nullptr); + return; + } + + // Create Mono TCP Server Socket + mono_server_fd = socket(AF_INET, SOCK_STREAM, 0); + if (mono_server_fd >= 0) { + int opt = 1; + setsockopt(mono_server_fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)); + struct sockaddr_in address; + address.sin_family = AF_INET; + address.sin_addr.s_addr = INADDR_ANY; + address.sin_port = htons(8081); + if (bind(mono_server_fd, (struct sockaddr*)&address, sizeof(address)) >= 0) { + listen(mono_server_fd, 1); + fcntl(mono_server_fd, F_SETFL, O_NONBLOCK); + LOGGER.info("Mono TCP Video stream server started on port 8081"); + } else { + LOGGER.error("Failed to bind Mono TCP socket"); + close(mono_server_fd); + mono_server_fd = -1; + } + } + + // Create Color TCP Server Socket + color_server_fd = socket(AF_INET, SOCK_STREAM, 0); + if (color_server_fd >= 0) { + int opt = 1; + setsockopt(color_server_fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)); + struct sockaddr_in address; + address.sin_family = AF_INET; + address.sin_addr.s_addr = INADDR_ANY; + address.sin_port = htons(8083); + if (bind(color_server_fd, (struct sockaddr*)&address, sizeof(address)) >= 0) { + listen(color_server_fd, 1); + fcntl(color_server_fd, F_SETFL, O_NONBLOCK); + LOGGER.info("Color TCP Video stream server started on port 8083"); + } else { + LOGGER.error("Failed to bind Color TCP socket"); + close(color_server_fd); + color_server_fd = -1; + } + } + + uint32_t mono_received = 0; + uint32_t color_header_received = 0; + uint32_t color_payload_received = 0; + uint8_t color_header[16]; + + uint16_t color_x = 0, color_y = 0, color_w = 0, color_h = 0; + uint32_t color_payload_len = 0; + + uint32_t last_packet_time = xTaskGetTickCount(); + uint32_t fps_start_time = xTaskGetTickCount(); + uint32_t fps_frame_count = 0; + + while (state.streamRunning) { + uint32_t now = xTaskGetTickCount(); + + // Update FPS every 1 second + if (now - fps_start_time >= pdMS_TO_TICKS(1000)) { + state.lastFps = (double)fps_frame_count * 1000.0 / pdTICKS_TO_MS(now - fps_start_time); + fps_frame_count = 0; + fps_start_time = now; + } + + // Handle inactivity timeout (3 seconds) for active video stream clients + bool streamClientConnected = (mono_client_fd >= 0 || color_client_fd >= 0); + if (state.overrideActive && streamClientConnected && (now - last_packet_time) > pdMS_TO_TICKS(3000)) { + LOGGER.info("Video stream timed out. Returning to normal screensaver."); + state.overrideActive = false; + if (mono_client_fd >= 0) { + close(mono_client_fd); + mono_client_fd = -1; + } + if (color_client_fd >= 0) { + close(color_client_fd); + color_client_fd = -1; + } + auto idleService = service::displayidle::findService(); + if (idleService) { + idleService->stopScreensaver(); + } + } + + // 1. Accept Mono client + if (mono_server_fd >= 0 && mono_client_fd < 0) { + struct sockaddr_in client_addr; + socklen_t addr_len = sizeof(client_addr); + int client = accept(mono_server_fd, (struct sockaddr*)&client_addr, &addr_len); + if (client >= 0) { + mono_client_fd = client; + fcntl(mono_client_fd, F_SETFL, O_NONBLOCK); + mono_received = 0; + last_packet_time = now; + LOGGER.info("Mono TCP Stream client connected"); + } + } + + // 2. Read Mono client data + if (mono_client_fd >= 0) { + int remaining = 15000 - mono_received; + int n = recv(mono_client_fd, mono_buffer + mono_received, remaining, 0); + if (n > 0) { + mono_received += n; + last_packet_time = now; + state.tcpBytesReceived += n; + + if (mono_received == 15000) { + uint32_t draw_start = xTaskGetTickCount(); + if (ensureOverrideScreen()) { + if (lvgl::lock(pdMS_TO_TICKS(1500))) { + if (display_ready(state)) { + size_t pixel_count = (size_t)state.drawWidth * state.drawHeight; + for (size_t i = 0; i < pixel_count; ++i) { + state.framebuffer[i] = 0xFFFF; // Fill canvas with white + } + + int x_off = (state.drawWidth - 200) / 2; + int y_off = (state.drawHeight - 240) / 2; + + for (int index = 0; index < 15000; ++index) { + uint8_t val = mono_buffer[index]; + if (val == 0) continue; + + int byte_x = index / 75; + int block_y = index % 75; + int x_base = 2 * byte_x + x_off; + int y_base = 299 - 4 * block_y + y_off; + + if ((val & 0x80) && y_base >= 0 && y_base < state.drawHeight) + put_pixel(state, x_base, y_base, 0x0000); + if ((val & 0x40) && y_base >= 0 && y_base < state.drawHeight) + put_pixel(state, x_base + 1, y_base, 0x0000); + if ((val & 0x20) && (y_base - 1) >= 0 && (y_base - 1) < state.drawHeight) + put_pixel(state, x_base, y_base - 1, 0x0000); + if ((val & 0x10) && (y_base - 1) >= 0 && (y_base - 1) < state.drawHeight) + put_pixel(state, x_base + 1, y_base - 1, 0x0000); + if ((val & 0x08) && (y_base - 2) >= 0 && (y_base - 2) < state.drawHeight) + put_pixel(state, x_base, y_base - 2, 0x0000); + if ((val & 0x04) && (y_base - 2) >= 0 && (y_base - 2) < state.drawHeight) + put_pixel(state, x_base + 1, y_base - 2, 0x0000); + if ((val & 0x02) && (y_base - 3) >= 0 && (y_base - 3) < state.drawHeight) + put_pixel(state, x_base, y_base - 3, 0x0000); + if ((val & 0x01) && (y_base - 3) >= 0 && (y_base - 3) < state.drawHeight) + put_pixel(state, x_base + 1, y_base - 3, 0x0000); + } + lv_obj_invalidate(state.drawArea); + state.overrideActive = true; + } + lvgl::unlock(); + } + } + state.lastDrawMs = pdTICKS_TO_MS(xTaskGetTickCount() - draw_start); + state.framesDrawn++; + fps_frame_count++; + + mono_received = 0; + } + } else if (n == 0 || (n < 0 && errno != EAGAIN && errno != EWOULDBLOCK)) { + LOGGER.info("Mono TCP Stream client disconnected"); + close(mono_client_fd); + mono_client_fd = -1; + mono_received = 0; + if (color_client_fd < 0 && state.overrideActive) { + state.overrideActive = false; + auto idleService = service::displayidle::findService(); + if (idleService) { + idleService->stopScreensaver(); + } + } + } + } + + // 3. Accept Color client + if (color_server_fd >= 0 && color_client_fd < 0) { + struct sockaddr_in client_addr; + socklen_t addr_len = sizeof(client_addr); + int client = accept(color_server_fd, (struct sockaddr*)&client_addr, &addr_len); + if (client >= 0) { + color_client_fd = client; + fcntl(color_client_fd, F_SETFL, O_NONBLOCK); + color_header_received = 0; + color_payload_received = 0; + last_packet_time = now; + LOGGER.info("Color TCP Stream client connected"); + } + } + + // 4. Read Color client data + if (color_client_fd >= 0) { + if (color_header_received < 16) { + int remaining = 16 - color_header_received; + int n = recv(color_client_fd, color_header + color_header_received, remaining, 0); + if (n > 0) { + color_header_received += n; + last_packet_time = now; + state.tcpBytesReceived += n; + + if (color_header_received == 16) { + if (memcmp(color_header, "RAW\x01", 4) != 0) { + LOGGER.warn("Invalid Color Stream magic! Disconnecting client."); + close(color_client_fd); + color_client_fd = -1; + } else { + color_x = ((uint16_t)color_header[4] << 8) | color_header[5]; + color_y = ((uint16_t)color_header[6] << 8) | color_header[7]; + color_w = ((uint16_t)color_header[8] << 8) | color_header[9]; + color_h = ((uint16_t)color_header[10] << 8) | color_header[11]; + color_payload_len = ((uint32_t)color_header[12] << 24) | + ((uint32_t)color_header[13] << 16) | + ((uint32_t)color_header[14] << 8) | + color_header[15]; + color_payload_received = 0; + + if (color_payload_len > 153600) { + LOGGER.warn("Color stream payload too large ({} bytes). Disconnecting.", color_payload_len); + close(color_client_fd); + color_client_fd = -1; + } + } + } + } else if (n == 0 || (n < 0 && errno != EAGAIN && errno != EWOULDBLOCK)) { + LOGGER.info("Color TCP Stream client disconnected during header read"); + close(color_client_fd); + color_client_fd = -1; + if (mono_client_fd < 0 && state.overrideActive) { + state.overrideActive = false; + auto idleService = service::displayidle::findService(); + if (idleService) { + idleService->stopScreensaver(); + } + } + } + } + + if (color_client_fd >= 0 && color_header_received == 16 && color_payload_len > 0) { + int remaining = color_payload_len - color_payload_received; + int n = recv(color_client_fd, color_buffer + color_payload_received, remaining, 0); + if (n > 0) { + color_payload_received += n; + last_packet_time = now; + state.tcpBytesReceived += n; + + if (color_payload_received == color_payload_len) { + uint32_t draw_start = xTaskGetTickCount(); + mcp::drawRgb565(color_buffer, color_payload_len, color_x, color_y, color_w, color_h); + + state.lastDrawMs = pdTICKS_TO_MS(xTaskGetTickCount() - draw_start); + state.framesDrawn++; + fps_frame_count++; + + color_header_received = 0; + color_payload_len = 0; + color_payload_received = 0; + } + } else if (n == 0 || (n < 0 && errno != EAGAIN && errno != EWOULDBLOCK)) { + LOGGER.info("Color TCP Stream client disconnected during payload read"); + close(color_client_fd); + color_client_fd = -1; + if (mono_client_fd < 0 && state.overrideActive) { + state.overrideActive = false; + auto idleService = service::displayidle::findService(); + if (idleService) { + idleService->stopScreensaver(); + } + } + } + } + } + + vTaskDelay(pdMS_TO_TICKS(5)); + } + + if (mono_client_fd >= 0) close(mono_client_fd); + if (color_client_fd >= 0) close(color_client_fd); + if (mono_server_fd >= 0) close(mono_server_fd); + if (color_server_fd >= 0) close(color_server_fd); + + heap_caps_free(mono_buffer); + heap_caps_free(color_buffer); + + LOGGER.info("Video stream server task stopped"); + state.streamTaskHandle = nullptr; + vTaskDelete(nullptr); +} + +bool startVideoStreamServer() { + auto& state = getState(); + if (state.streamRunning) { + return true; + } + state.streamRunning = true; + state.framesDrawn = 0; + state.tcpBytesReceived = 0; + state.lastDrawMs = 0; + state.lastFps = 0.0; + + BaseType_t ret = xTaskCreatePinnedToCore( + mcp_video_stream_task, + "mcp_video_stream", + 4096, + nullptr, + 5, + (TaskHandle_t*)&state.streamTaskHandle, + 1 + ); + if (ret != pdPASS) { + state.streamRunning = false; + LOGGER.error("Failed to spawn video stream task"); + return false; + } + return true; +} + +void stopVideoStreamServer() { + auto& state = getState(); + if (!state.streamRunning) { + return; + } + state.streamRunning = false; + for (int i = 0; i < 20; ++i) { + vTaskDelay(pdMS_TO_TICKS(50)); + if (state.streamTaskHandle == nullptr) { + break; + } + } +} + +bool getVideoStreamStats(std::string& stats_json) { + auto& state = getState(); + cJSON* root = cJSON_CreateObject(); + cJSON_AddNumberToObject(root, "frames_drawn", state.framesDrawn); + cJSON_AddNumberToObject(root, "tcp_bytes_received", state.tcpBytesReceived); + cJSON_AddNumberToObject(root, "last_draw_ms", state.lastDrawMs); + cJSON_AddNumberToObject(root, "last_fps", state.lastFps); + cJSON_AddBoolToObject(root, "active", state.overrideActive); + + char* printed = cJSON_PrintUnformatted(root); + if (printed != nullptr) { + stats_json = printed; + free(printed); + } else { + stats_json = "{}"; + } + cJSON_Delete(root); + return true; +} + +bool listApps(std::string& apps_json, std::string& error) { + auto manifests = app::getAppManifests(); + + cJSON* root = cJSON_CreateObject(); + cJSON* apps_array = cJSON_AddArrayToObject(root, "apps"); + + for (const auto& manifest : manifests) { + cJSON* item = cJSON_CreateObject(); + cJSON_AddStringToObject(item, "appId", manifest->appId.c_str()); + cJSON_AddStringToObject(item, "appName", manifest->appName.c_str()); + cJSON_AddStringToObject(item, "appIcon", manifest->appIcon.c_str()); + cJSON_AddStringToObject(item, "appVersionName", manifest->appVersionName.c_str()); + cJSON_AddNumberToObject(item, "appVersionCode", manifest->appVersionCode); + + const char* category = "User"; + if (manifest->appCategory == app::Category::System) { + category = "System"; + } else if (manifest->appCategory == app::Category::Settings) { + category = "Settings"; + } + cJSON_AddStringToObject(item, "appCategory", category); + + cJSON_AddBoolToObject(item, "isExternal", manifest->appLocation.isExternal()); + if (manifest->appLocation.isExternal()) { + cJSON_AddStringToObject(item, "path", manifest->appLocation.getPath().c_str()); + } else { + cJSON_AddStringToObject(item, "path", "internal"); + } + + cJSON_AddNumberToObject(item, "appFlags", manifest->appFlags); + cJSON_AddItemToArray(apps_array, item); + } + + char* printed = cJSON_PrintUnformatted(root); + if (printed != nullptr) { + apps_json = printed; + free(printed); + } else { + apps_json = "{\"apps\":[]}"; + } + cJSON_Delete(root); + return true; +} + +bool runApp(const std::string& appId, std::string& error) { + auto manifest = app::findAppManifestById(appId); + if (manifest == nullptr) { + error = "Application not found"; + return false; + } + + app::start(appId); + return true; +} + +bool listSdFiles(const std::string& directory, std::string& files_json, std::string& error) { + std::string resolved_path; + if (!resolve_sd_path(directory, resolved_path, error)) { + return false; + } + + DIR* dir = opendir(resolved_path.c_str()); + if (dir == nullptr) { + error = "Failed to open directory"; + return false; + } + + cJSON* root = cJSON_CreateObject(); + cJSON* file_array = cJSON_AddArrayToObject(root, "files"); + + struct dirent* entry; + while ((entry = readdir(dir)) != nullptr) { + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) { + continue; + } + + cJSON* item = cJSON_CreateObject(); + cJSON_AddStringToObject(item, "name", entry->d_name); + + std::string full_filepath = resolved_path; + if (!full_filepath.ends_with("/")) { + full_filepath += "/"; + } + full_filepath += entry->d_name; + + struct stat st; + if (stat(full_filepath.c_str(), &st) == 0) { + bool is_dir = S_ISDIR(st.st_mode); + cJSON_AddStringToObject(item, "type", is_dir ? "directory" : "file"); + if (!is_dir) { + cJSON_AddNumberToObject(item, "size", st.st_size); + } + } else { + cJSON_AddStringToObject(item, "type", entry->d_type == DT_DIR ? "directory" : "file"); + } + cJSON_AddItemToArray(file_array, item); + } + closedir(dir); + + char* printed = cJSON_PrintUnformatted(root); + if (printed != nullptr) { + files_json = printed; + free(printed); + } else { + files_json = "{\"files\":[]}"; + } + cJSON_Delete(root); + return true; +} + +} // namespace tt::mcp #endif diff --git a/Tactility/Source/service/displayidle/McpScreensaver.cpp b/Tactility/Source/service/displayidle/McpScreensaver.cpp index 150d6e08..6f342e81 100644 --- a/Tactility/Source/service/displayidle/McpScreensaver.cpp +++ b/Tactility/Source/service/displayidle/McpScreensaver.cpp @@ -37,10 +37,10 @@ void McpScreensaver::start(lv_obj_t* overlay, lv_coord_t screenW, lv_coord_t scr return; } - // Fill with a dark slate background + // Fill with a dark slate background (inverted for display path) size_t pixelCount = (size_t)screenW * screenH; for (size_t i = 0; i < pixelCount; ++i) { - framebuffer[i] = 0x18E3; // dark blue-grey + framebuffer[i] = ~0x18E3; // dark blue-grey } lv_canvas_set_buffer(canvas, framebuffer, screenW, screenH, LV_COLOR_FORMAT_RGB565); @@ -48,12 +48,13 @@ void McpScreensaver::start(lv_obj_t* overlay, lv_coord_t screenW, lv_coord_t scr // Waiting label (removed on first MCP draw via lv_obj_clean) lv_obj_t* waitLabel = lv_label_create(canvas); lv_label_set_text(waitLabel, "Waiting for LLM..."); - lv_obj_set_style_text_color(waitLabel, lv_color_white(), LV_PART_MAIN); + lv_obj_set_style_text_color(waitLabel, lv_color_black(), LV_PART_MAIN); // white on screen (inverted) lv_obj_align(waitLabel, LV_ALIGN_CENTER, 0, -20); lv_obj_t* resLabel = lv_label_create(canvas); lv_label_set_text_fmt(resLabel, "Display: %dx%d", (int)screenW, (int)screenH); - lv_obj_set_style_text_color(resLabel, lv_palette_lighten(LV_PALETTE_BLUE, 3), LV_PART_MAIN); + lv_color_t resColor = lv_palette_lighten(LV_PALETTE_BLUE, 3); + lv_obj_set_style_text_color(resLabel, lv_color_make(~resColor.red, ~resColor.green, ~resColor.blue), LV_PART_MAIN); lv_obj_align(resLabel, LV_ALIGN_CENTER, 0, 10); // Register with McpSystemState diff --git a/Tactility/Source/service/gui/GuiService.cpp b/Tactility/Source/service/gui/GuiService.cpp index 235cdb59..8cb92666 100644 --- a/Tactility/Source/service/gui/GuiService.cpp +++ b/Tactility/Source/service/gui/GuiService.cpp @@ -199,7 +199,7 @@ bool GuiService::onStart(ServiceContext& service) { thread = new Thread( GUI_TASK_NAME, - 4096, // Last known minimum was 2800 for launching desktop + 8192, // Last known minimum was 2800 for launching desktop, increased to 8192 to prevent stack overflow on complex layouts/MCP settings guiMain ); thread->setPriority(THREAD_PRIORITY_SERVICE); diff --git a/Tactility/Source/service/statusbar/Statusbar.cpp b/Tactility/Source/service/statusbar/Statusbar.cpp index fa4bf4bf..65f58e1f 100644 --- a/Tactility/Source/service/statusbar/Statusbar.cpp +++ b/Tactility/Source/service/statusbar/Statusbar.cpp @@ -26,6 +26,8 @@ #include #include +#include +#include namespace tt::service::statusbar { @@ -102,6 +104,13 @@ static const char* getPowerStatusIcon() { return nullptr; } + hal::power::PowerDevice::MetricData charging_data; + if (power->supportsMetric(hal::power::PowerDevice::MetricType::IsCharging) && + power->getMetric(hal::power::PowerDevice::MetricType::IsCharging, charging_data) && + charging_data.valueAsBool) { + return LVGL_ICON_STATUSBAR_BATTERY_ANDROID_FRAME_BOLT; + } + hal::power::PowerDevice::MetricData charge_level; if (!power->getMetric(hal::power::PowerDevice::MetricType::ChargeLevel, charge_level)) { return nullptr; @@ -210,6 +219,29 @@ class StatusbarService final : public Service { } power_last_icon = desired_icon; } + + std::shared_ptr power; + hal::findDevices(hal::Device::Type::Power, [&power](const auto& device) { + if (device->supportsMetric(hal::power::PowerDevice::MetricType::ChargeLevel)) { + power = device; + return false; + } + return true; + }); + + if (power != nullptr) { + hal::power::PowerDevice::MetricData charge_level; + if (power->getMetric(hal::power::PowerDevice::MetricType::ChargeLevel, charge_level)) { + uint8_t charge = charge_level.valueAsUint8; + std::string battery_text = std::format("{}%", charge); + lvgl::statusbar_set_battery_text(battery_text); + lvgl::statusbar_set_battery_visibility(true); + } else { + lvgl::statusbar_set_battery_visibility(false); + } + } else { + lvgl::statusbar_set_battery_visibility(false); + } } void updateUsbIcon() { diff --git a/Tactility/Source/service/webserver/McpHandler.cpp b/Tactility/Source/service/webserver/McpHandler.cpp index 0e06d509..1a525f81 100644 --- a/Tactility/Source/service/webserver/McpHandler.cpp +++ b/Tactility/Source/service/webserver/McpHandler.cpp @@ -157,6 +157,116 @@ static const char* TOOLS_JSON = "\"mp3_base64\":{\"type\":\"string\"}," "\"volume\":{\"type\":\"integer\",\"default\":50}" "},\"required\":[\"mp3_base64\"]}" + "}," + "{" + "\"name\":\"set_led\"," + "\"description\":\"Control the onboard WS2812 NeoPixel RGB LED.\"," + "\"inputSchema\":{" + "\"type\":\"object\"," + "\"properties\":{" + "\"r\":{\"type\":\"integer\",\"minimum\":0,\"maximum\":255,\"description\":\"Red channel (0-255)\"}," + "\"g\":{\"type\":\"integer\",\"minimum\":0,\"maximum\":255,\"description\":\"Green channel (0-255)\"}," + "\"b\":{\"type\":\"integer\",\"minimum\":0,\"maximum\":255,\"description\":\"Blue channel (0-255)\"}," + "\"mode\":{\"type\":\"string\",\"enum\":[\"static\",\"breath\",\"rainbow\",\"off\"],\"description\":\"LED mode\"}" + "}," + "\"required\":[\"r\",\"g\",\"b\",\"mode\"]" + "}" + "}," + "{" + "\"name\":\"get_battery\"," + "\"description\":\"Read the current battery voltage and estimated capacity percentage.\"," + "\"inputSchema\":{\"type\":\"object\",\"properties\":{}}" + "}," + "{" + "\"name\":\"get_sensors\"," + "\"description\":\"Read onboard sensors (IMU, temperature, humidity).\"," + "\"inputSchema\":{\"type\":\"object\",\"properties\":{}}" + "}," + "{" + "\"name\":\"scan_ble\"," + "\"description\":\"Scan for nearby Bluetooth Low Energy devices.\"," + "\"inputSchema\":{" + "\"type\":\"object\"," + "\"properties\":{" + "\"duration_ms\":{\"type\":\"integer\",\"default\":3000,\"description\":\"Scan duration in milliseconds\"}" + "}" + "}" + "}," + "{" + "\"name\":\"write_file\"," + "\"description\":\"Write a file to the microSD card.\"," + "\"inputSchema\":{" + "\"type\":\"object\"," + "\"properties\":{" + "\"path\":{\"type\":\"string\",\"description\":\"Destination path relative to /sdcard/\"}," + "\"content\":{\"type\":\"string\",\"description\":\"Text content of the file\"}" + "}," + "\"required\":[\"path\",\"content\"]" + "}" + "}," + "{" + "\"name\":\"read_file\"," + "\"description\":\"Read a text file from the microSD card.\"," + "\"inputSchema\":{" + "\"type\":\"object\"," + "\"properties\":{" + "\"path\":{\"type\":\"string\",\"description\":\"File path relative to /sdcard/\"}" + "}," + "\"required\":[\"path\"]" + "}" + "}," + "{" + "\"name\":\"download_file\"," + "\"description\":\"Download a file from a URL directly to the microSD card.\"," + "\"inputSchema\":{" + "\"type\":\"object\"," + "\"properties\":{" + "\"url\":{\"type\":\"string\",\"description\":\"The URL of the file to download\"}," + "\"filename\":{\"type\":\"string\",\"description\":\"The destination filename relative to /sdcard/\"}" + "}," + "\"required\":[\"url\",\"filename\"]" + "}" + "}," + "{" + "\"name\":\"get_video_streaming_instructions\"," + "\"description\":\"Get details for monochrome TCP streaming and RGB565 color TCP streaming.\"," + "\"inputSchema\":{" + "\"type\":\"object\"," + "\"properties\":{" + "\"protocol\":{\"type\":\"string\",\"enum\":[\"tcp\",\"color\",\"both\",\"all\"],\"default\":\"all\"}" + "}" + "}" + "}," + "{" + "\"name\":\"get_stream_stats\"," + "\"description\":\"Get performance statistics for the active video streams.\"," + "\"inputSchema\":{\"type\":\"object\",\"properties\":{}}" + "}," + "{" + "\"name\":\"list_apps\"," + "\"description\":\"List all applications installed on the system, including metadata and locations.\"," + "\"inputSchema\":{\"type\":\"object\",\"properties\":{}}" + "}," + "{" + "\"name\":\"run_app\"," + "\"description\":\"Launch an application on the board by its App ID.\"," + "\"inputSchema\":{" + "\"type\":\"object\"," + "\"properties\":{" + "\"app_id\":{\"type\":\"string\",\"description\":\"The unique ID of the application (e.g., 'one.tactility.helloworld')\"}" + "}," + "\"required\":[\"app_id\"]" + "}" + "}," + "{" + "\"name\":\"list_files\"," + "\"description\":\"List files and folders in a directory on the SD card, with details like sizes.\"," + "\"inputSchema\":{" + "\"type\":\"object\"," + "\"properties\":{" + "\"path\":{\"type\":\"string\",\"description\":\"Path relative to /sdcard/ to list (default is '/')\"}" + "}" + "}" "}" "]"; @@ -173,7 +283,7 @@ static uint8_t* base64_decode(const char* input, size_t* output_size) { size_t input_size = strlen(payload); size_t approx_output_len = (input_size * 3) / 4; - uint8_t* output = (uint8_t*)psram_malloc(approx_output_len + 4); + uint8_t* output = (uint8_t*)malloc(approx_output_len + 4); if (output == NULL) { return NULL; } @@ -564,6 +674,162 @@ static cJSON* handle_tool_call(cJSON* id, cJSON* params) { ? make_tool_result(id, "Successfully played base64 MP3 audio.") : make_error(id, -32020, error.c_str()); } + + if (strcmp(name, "set_led") == 0) { + cJSON* r_item = cJSON_GetObjectItemCaseSensitive(arguments, "r"); + cJSON* g_item = cJSON_GetObjectItemCaseSensitive(arguments, "g"); + cJSON* b_item = cJSON_GetObjectItemCaseSensitive(arguments, "b"); + cJSON* mode_item = cJSON_GetObjectItemCaseSensitive(arguments, "mode"); + if (!cJSON_IsNumber(r_item) || !cJSON_IsNumber(g_item) || + !cJSON_IsNumber(b_item) || !cJSON_IsString(mode_item)) { + return make_error(id, -32602, "set_led requires r, g, b, and mode"); + } + std::string error; + if (!mcp::setLedColor(r_item->valueint, g_item->valueint, b_item->valueint, mode_item->valuestring, error)) { + return make_error(id, -32020, error.c_str()); + } + char msg[128]; + snprintf(msg, sizeof(msg), "LED set to mode '%s' with base color (%d, %d, %d).", + mode_item->valuestring, r_item->valueint, g_item->valueint, b_item->valueint); + return make_tool_result(id, msg); + } + + if (strcmp(name, "get_battery") == 0) { + double voltage_v = 0.0; + int percentage_pct = 0; + std::string error; + if (!mcp::getBatteryStatus(voltage_v, percentage_pct, error)) { + return make_error(id, -32020, error.c_str()); + } + char result_buf[128]; + snprintf(result_buf, sizeof(result_buf), "{\"voltage_v\":%.3f,\"percentage_pct\":%d}", voltage_v, percentage_pct); + return make_tool_result(id, result_buf); + } + + if (strcmp(name, "get_sensors") == 0) { + double temp_c = 0.0; + double hum_pct = 0.0; + std::string imu_json; + std::string error; + if (!mcp::getSensors(temp_c, hum_pct, imu_json, error)) { + return make_error(id, -32020, error.c_str()); + } + char result_buf[512]; + snprintf(result_buf, sizeof(result_buf), "{\"temperature_c\":%.2f,\"humidity_pct\":%.2f,\"imu\":%s}", + temp_c, hum_pct, imu_json.c_str()); + return make_tool_result(id, result_buf); + } + + if (strcmp(name, "scan_ble") == 0) { + cJSON* duration_item = cJSON_GetObjectItemCaseSensitive(arguments, "duration_ms"); + int duration_ms = cJSON_IsNumber(duration_item) ? duration_item->valueint : 3000; + if (duration_ms < 500 || duration_ms > 15000) { + return make_error(id, -32602, "duration_ms must be between 500 and 15000"); + } + std::string devices_json; + std::string error; + if (!mcp::scanBleDevices(duration_ms, devices_json, error)) { + return make_error(id, -32020, error.c_str()); + } + return make_tool_result(id, devices_json.c_str()); + } + + if (strcmp(name, "write_file") == 0) { + cJSON* path_item = cJSON_GetObjectItemCaseSensitive(arguments, "path"); + cJSON* content_item = cJSON_GetObjectItemCaseSensitive(arguments, "content"); + if (!cJSON_IsString(path_item) || !cJSON_IsString(content_item)) { + return make_error(id, -32602, "write_file requires path and content"); + } + std::string error; + if (!mcp::writeSdFile(path_item->valuestring, content_item->valuestring, error)) { + return make_error(id, -32020, error.c_str()); + } + return make_tool_result(id, "Successfully wrote file to SD card."); + } + + if (strcmp(name, "read_file") == 0) { + cJSON* path_item = cJSON_GetObjectItemCaseSensitive(arguments, "path"); + if (!cJSON_IsString(path_item)) { + return make_error(id, -32602, "read_file requires path"); + } + std::string content; + std::string error; + if (!mcp::readSdFile(path_item->valuestring, content, error)) { + return make_error(id, -32020, error.c_str()); + } + return make_tool_result(id, content.c_str()); + } + + if (strcmp(name, "download_file") == 0) { + cJSON* url_item = cJSON_GetObjectItemCaseSensitive(arguments, "url"); + cJSON* filename_item = cJSON_GetObjectItemCaseSensitive(arguments, "filename"); + if (!cJSON_IsString(url_item) || !cJSON_IsString(filename_item)) { + return make_error(id, -32602, "download_file requires url and filename"); + } + std::string error; + if (!mcp::downloadSdFile(url_item->valuestring, filename_item->valuestring, error)) { + return make_error(id, -32020, error.c_str()); + } + return make_tool_result(id, "Successfully downloaded file to SD card."); + } + + if (strcmp(name, "get_video_streaming_instructions") == 0) { + cJSON* protocol_item = cJSON_GetObjectItemCaseSensitive(arguments, "protocol"); + const char* protocol = cJSON_IsString(protocol_item) ? protocol_item->valuestring : "all"; + + char buf[512]; + snprintf(buf, sizeof(buf), + "Tactility OS TCP Video Streaming Instructions:\n" + "1. Mono TCP Stream (compatibility mode): Stream raw 15,000-byte PBM/RLCD frames to TCP port 8081.\n" + "2. Color TCP Stream (native): Stream packets to TCP port 8083.\n" + " Packet format:\n" + " - Header (16 bytes): Magic 'RAW\\x01' (4 bytes), X-coord (2 bytes BE), Y-coord (2 bytes BE), Width (2 bytes BE), Height (2 bytes BE), Payload length (4 bytes BE).\n" + " - Payload (variable): Raw big-endian RGB565 pixel data of length 'Payload length'.\n" + "Active protocol selection: %s", protocol); + return make_tool_result(id, buf); + } + + if (strcmp(name, "get_stream_stats") == 0) { + std::string stats_json; + if (!mcp::getVideoStreamStats(stats_json)) { + return make_error(id, -32020, "Failed to get stream stats"); + } + return make_tool_result(id, stats_json.c_str()); + } + + if (strcmp(name, "list_apps") == 0) { + std::string apps_json; + std::string error; + if (!mcp::listApps(apps_json, error)) { + return make_error(id, -32020, error.c_str()); + } + return make_tool_result(id, apps_json.c_str()); + } + + if (strcmp(name, "run_app") == 0) { + cJSON* app_id_item = cJSON_GetObjectItemCaseSensitive(arguments, "app_id"); + if (!cJSON_IsString(app_id_item)) { + return make_error(id, -32602, "run_app requires app_id"); + } + std::string error; + if (!mcp::runApp(app_id_item->valuestring, error)) { + return make_error(id, -32020, error.c_str()); + } + char msg[128]; + snprintf(msg, sizeof(msg), "Successfully started application '%s'.", app_id_item->valuestring); + return make_tool_result(id, msg); + } + + if (strcmp(name, "list_files") == 0) { + cJSON* path_item = cJSON_GetObjectItemCaseSensitive(arguments, "path"); + const char* path = cJSON_IsString(path_item) ? path_item->valuestring : "/"; + std::string files_json; + std::string error; + if (!mcp::listSdFiles(path, files_json, error)) { + return make_error(id, -32020, error.c_str()); + } + return make_tool_result(id, files_json.c_str()); + } return make_error(id, -32602, "Unknown tool"); } diff --git a/Tactility/Source/service/webserver/WebServerService.cpp b/Tactility/Source/service/webserver/WebServerService.cpp index c2ca925f..32f30916 100644 --- a/Tactility/Source/service/webserver/WebServerService.cpp +++ b/Tactility/Source/service/webserver/WebServerService.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -510,7 +511,7 @@ bool WebServerService::startServer() { settings.webServerPort, "0.0.0.0", handlers, - 16384 // Stack size + 12288 // Stack size (forces allocation in internal SRAM instead of PSRAM) ); httpServer->start(); @@ -531,6 +532,11 @@ bool WebServerService::startServer() { settings.wifiMode == settings::webserver::WiFiMode::AccessPoint ? "AP" : "Station"); } + auto mcpSettings = settings::mcp::loadOrGetDefault(); + if (mcpSettings.mcpEnabled) { + mcp::startVideoStreamServer(); + } + return true; } @@ -542,6 +548,8 @@ void WebServerService::stopServer() { httpServer->stop(); httpServer.reset(); + mcp::stopVideoStreamServer(); + // Stop AP mode WiFi if we started it if (apWifiInitialized || apNetif != nullptr) { stopApMode(); diff --git a/Tactility/Source/service/wifi/WifiApSettings.cpp b/Tactility/Source/service/wifi/WifiApSettings.cpp index 71d32536..65dee7f7 100644 --- a/Tactility/Source/service/wifi/WifiApSettings.cpp +++ b/Tactility/Source/service/wifi/WifiApSettings.cpp @@ -107,6 +107,23 @@ static bool decrypt(const std::string& ssidInput, std::string& ssidOutput) { return false; } + // Strip PKCS#7 padding + if (result_length > 0) { + uint8_t pad_val = result[result_length - 1]; + if (pad_val >= 1 && pad_val <= 16 && pad_val <= result_length) { + bool valid_padding = true; + for (size_t i = result_length - pad_val; i < result_length; ++i) { + if (result[i] != pad_val) { + valid_padding = false; + break; + } + } + if (valid_padding) { + result[result_length - pad_val] = 0; + } + } + } + ssidOutput = reinterpret_cast(result); free(result); return true; diff --git a/Tactility/Source/service/wifi/WifiEsp.cpp b/Tactility/Source/service/wifi/WifiEsp.cpp index 2215fe76..9796f729 100644 --- a/Tactility/Source/service/wifi/WifiEsp.cpp +++ b/Tactility/Source/service/wifi/WifiEsp.cpp @@ -20,6 +20,9 @@ #include #include +#include +#include +#include #include #include #include @@ -543,6 +546,28 @@ static void dispatchEnable(std::shared_ptr wifi) { } wifi->netif = esp_netif_create_default_wifi_sta(); + if (wifi->netif != nullptr) { + uint8_t mac[6]; + char hostname[32]; + if (esp_read_mac(mac, ESP_MAC_WIFI_STA) == ESP_OK) { + snprintf(hostname, sizeof(hostname), "kidsOS-%02X%02X", mac[4], mac[5]); + } else { + strncpy(hostname, "kidsOS", sizeof(hostname)); + } + LOGGER.info("Setting DHCP Hostname to '{}'", hostname); + esp_netif_set_hostname(wifi->netif, hostname); + + // Initialize mDNS + esp_err_t mdns_err = mdns_init(); + if (mdns_err == ESP_OK) { + LOGGER.info("mDNS initialized, setting hostname to '{}.local'", hostname); + mdns_hostname_set(hostname); + mdns_instance_name_set("kidsOS Tactility Device"); + } else { + LOGGER.error("Failed to initialize mDNS: {}", (int)mdns_err); + } + } + // Warning: this is the memory-intensive operation // It uses over 117kB of RAM with default settings for S3 on IDF v5.1.2 wifi_init_config_t config = WIFI_INIT_CONFIG_DEFAULT(); diff --git a/components/lambage__esp_lcd_touch_ft6336u/esp_lcd_touch_ft6x36.c b/components/lambage__esp_lcd_touch_ft6336u/esp_lcd_touch_ft6x36.c index e0b692c3..d2a72a96 100644 --- a/components/lambage__esp_lcd_touch_ft6336u/esp_lcd_touch_ft6x36.c +++ b/components/lambage__esp_lcd_touch_ft6336u/esp_lcd_touch_ft6x36.c @@ -44,7 +44,7 @@ static const char *TAG = "FT6x36"; #define FT6236U_CHIPID 0x64 // Chip selecting #define FT6336U_CHIPID 0x64 // Chip selecting -#define FT62XX_DEFAULT_THRESHOLD 128 // Default threshold for touch detection +#define FT62XX_DEFAULT_THRESHOLD 60 // Default threshold for touch detection (lowered from 128 to improve sensitivity on battery/floating ground) /******************************************************************************* * Function definitions