From a4be0d73aef417d88a65d426dde3045ccb8b86ca Mon Sep 17 00:00:00 2001 From: Adolfo Reyna Date: Wed, 1 Jul 2026 13:45:04 -0400 Subject: [PATCH] feat(reynabot): Improve UI and update voice endpoint to kids gateway --- Apps/ReynaBot/CMakeLists.txt | 16 + Apps/ReynaBot/main/CMakeLists.txt | 8 + Apps/ReynaBot/main/Source/main.c | 700 ++++++++++++++++++++++++++ Apps/ReynaBot/main/Source/websocket.c | 240 +++++++++ Apps/ReynaBot/main/Source/websocket.h | 59 +++ Apps/ReynaBot/manifest.properties | 10 + 6 files changed, 1033 insertions(+) create mode 100644 Apps/ReynaBot/CMakeLists.txt create mode 100644 Apps/ReynaBot/main/CMakeLists.txt create mode 100644 Apps/ReynaBot/main/Source/main.c create mode 100644 Apps/ReynaBot/main/Source/websocket.c create mode 100644 Apps/ReynaBot/main/Source/websocket.h create mode 100644 Apps/ReynaBot/manifest.properties diff --git a/Apps/ReynaBot/CMakeLists.txt b/Apps/ReynaBot/CMakeLists.txt new file mode 100644 index 0000000..77efb35 --- /dev/null +++ b/Apps/ReynaBot/CMakeLists.txt @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 3.20) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) + +if (DEFINED ENV{TACTILITY_SDK_PATH}) + set(TACTILITY_SDK_PATH $ENV{TACTILITY_SDK_PATH}) +else() + set(TACTILITY_SDK_PATH "../../release/TactilitySDK") + message(WARNING "TACTILITY_SDK_PATH is not set, defaulting to ${TACTILITY_SDK_PATH}") +endif() + +include("${TACTILITY_SDK_PATH}/TactilitySDK.cmake") +set(EXTRA_COMPONENT_DIRS ${TACTILITY_SDK_PATH}) + +project(ReynaBot) +tactility_project(ReynaBot) diff --git a/Apps/ReynaBot/main/CMakeLists.txt b/Apps/ReynaBot/main/CMakeLists.txt new file mode 100644 index 0000000..7d660cd --- /dev/null +++ b/Apps/ReynaBot/main/CMakeLists.txt @@ -0,0 +1,8 @@ +file(GLOB_RECURSE SOURCE_FILES Source/*.c) +set(CJSON_SOURCE "$ENV{IDF_PATH}/components/json/cJSON/cJSON.c") + +idf_component_register( + SRCS ${SOURCE_FILES} ${CJSON_SOURCE} + INCLUDE_DIRS Source "$ENV{IDF_PATH}/components/json/cJSON" + REQUIRES TactilitySDK lwip +) diff --git a/Apps/ReynaBot/main/Source/main.c b/Apps/ReynaBot/main/Source/main.c new file mode 100644 index 0000000..c1882b6 --- /dev/null +++ b/Apps/ReynaBot/main/Source/main.c @@ -0,0 +1,700 @@ +#include +#include +#include +#include + +#include +#include + +#include "websocket.h" +#include + +#include +#include +#include +#include + +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "esp_log.h" + +#define TAG "ReynaBot" + +typedef enum { + STATE_IDLE, + STATE_CONNECTING, + STATE_LISTENING, + STATE_THINKING, + STATE_SPEAKING, + STATE_ERROR +} ReynaBotState; + +typedef struct { + AppHandle app; + bool visible; + ReynaBotState state; + char server_url[128]; + char device_id[64]; + char api_key[128]; + + char last_transcript[256]; + char last_response[512]; + char error_message[128]; + + // UI components + lv_obj_t* lbl_status; + lv_obj_t* lbl_transcript; + lv_obj_t* lbl_response; + lv_obj_t* btn_ptt; + lv_obj_t* btn_ptt_label; + + // PTT control + bool is_pressed; + bool start_session; + bool stop_session; + bool cancel_session; + + // FreeRTOS Tasks + TaskHandle_t worker_task; + TaskHandle_t rx_task; + + // Hardware + struct Device* i2s_dev; + + // WebSocket + int ws_fd; + bool ws_connected; + bool ws_done; + + bool ui_update_pending; +} ReynaBotCtx; + +/* ─── Forward Declarations ─── */ +static void reynabot_task(void* arg); +static void reynabot_rx_task(void* arg); +static void update_ui(ReynaBotCtx* ctx); +static void load_config(ReynaBotCtx* ctx); + +/* ─── Helper for URL parsing ─── */ +static bool parse_ws_url(const char* url, char* host, int* port, char* path) { + if (strncmp(url, "ws://", 5) != 0) return false; + const char* p = url + 5; + const char* colon = strchr(p, ':'); + const char* slash = strchr(p, '/'); + + if (slash == NULL) { + strcpy(path, "/"); + } else { + strcpy(path, slash); + } + + if (colon != NULL && (slash == NULL || colon < slash)) { + int host_len = colon - p; + memcpy(host, p, host_len); + host[host_len] = '\0'; + *port = atoi(colon + 1); + } else { + int host_len = slash ? (slash - p) : strlen(p); + memcpy(host, p, host_len); + host[host_len] = '\0'; + *port = 80; + } + return true; +} + +/* ─── Config Loading/Saving ─── */ +static void load_config(ReynaBotCtx* ctx) { + // Default fallback config + snprintf(ctx->server_url, sizeof(ctx->server_url), "ws://192.168.68.126:8643/api/esp32/voice/ws"); + snprintf(ctx->device_id, sizeof(ctx->device_id), "reynabot_screen"); + snprintf(ctx->api_key, sizeof(ctx->api_key), "hmek_sXB7921bZ9FXTVKARExqUZ7ttBxtEoURHRU0JCB-gNY"); + + char path[256]; + size_t path_size = sizeof(path); + tt_app_get_user_data_child_path(ctx->app, "config.json", path, &path_size); + + FILE* file = fopen(path, "r"); + if (file == NULL) { + // Create user data directory if it doesn't exist + char dir_path[256]; + size_t dir_size = sizeof(dir_path); + tt_app_get_user_data_path(ctx->app, dir_path, &dir_size); + mkdir(dir_path, 0755); + + // Write default config file + file = fopen(path, "w"); + if (file != NULL) { + fprintf(file, "{\n \"server_url\": \"ws://192.168.68.126:8643/api/esp32/voice/ws\",\n \"device_id\": \"reynabot_screen\",\n \"api_key\": \"mcT1YA1vOr9wXSiHpCYalweEGGZKX-PIfZv2drp8BSg\"\n}\n"); + fclose(file); + } + ESP_LOGI(TAG, "Created default config.json at %s", path); + return; + } + + fseek(file, 0, SEEK_END); + long size = ftell(file); + fseek(file, 0, SEEK_SET); + + if (size > 0 && size < 4096) { + char* buf = malloc(size + 1); + if (buf != NULL) { + size_t read_bytes = fread(buf, 1, size, file); + buf[read_bytes] = '\0'; + cJSON* json = cJSON_Parse(buf); + if (json != NULL) { + cJSON* url_item = cJSON_GetObjectItem(json, "server_url"); + if (url_item != NULL && url_item->valuestring != NULL) { + strncpy(ctx->server_url, url_item->valuestring, sizeof(ctx->server_url) - 1); + } + cJSON* dev_item = cJSON_GetObjectItem(json, "device_id"); + if (dev_item != NULL && dev_item->valuestring != NULL) { + strncpy(ctx->device_id, dev_item->valuestring, sizeof(ctx->device_id) - 1); + } + cJSON* key_item = cJSON_GetObjectItem(json, "api_key"); + if (key_item != NULL && key_item->valuestring != NULL) { + strncpy(ctx->api_key, key_item->valuestring, sizeof(ctx->api_key) - 1); + } + cJSON_Delete(json); + } + free(buf); + } + } + fclose(file); + ESP_LOGI(TAG, "Loaded config: server=%s device=%s", ctx->server_url, ctx->device_id); +} + +/* ─── JSON Message Processing ─── */ +static void parse_json_message(ReynaBotCtx* ctx, const char* json_str) { + cJSON* json = cJSON_Parse(json_str); + if (json == NULL) return; + + cJSON* evt_item = cJSON_GetObjectItem(json, "event"); + if (evt_item != NULL && evt_item->valuestring != NULL) { + const char* evt = evt_item->valuestring; + ESP_LOGI(TAG, "WS Event: %s", evt); + + if (strcmp(evt, "ready") == 0) { + ESP_LOGI(TAG, "Server ready"); + } else if (strcmp(evt, "listening") == 0) { + ctx->state = STATE_LISTENING; + ctx->ui_update_pending = true; + } else if (strcmp(evt, "transcript") == 0) { + cJSON* txt_item = cJSON_GetObjectItem(json, "text"); + if (txt_item != NULL && txt_item->valuestring != NULL) { + strncpy(ctx->last_transcript, txt_item->valuestring, sizeof(ctx->last_transcript) - 1); + ctx->ui_update_pending = true; + } + } else if (strcmp(evt, "thinking") == 0) { + ctx->state = STATE_THINKING; + ctx->ui_update_pending = true; + } else if (strcmp(evt, "response_text") == 0) { + cJSON* txt_item = cJSON_GetObjectItem(json, "text"); + if (txt_item != NULL && txt_item->valuestring != NULL) { + strncpy(ctx->last_response, txt_item->valuestring, sizeof(ctx->last_response) - 1); + ctx->ui_update_pending = true; + } + } else if (strcmp(evt, "audio_start") == 0) { + ctx->state = STATE_SPEAKING; + ctx->ui_update_pending = true; + + // I2S is already configured globally at session startup + } else if (strcmp(evt, "audio_end") == 0) { + ESP_LOGI(TAG, "Audio response ended"); + } else if (strcmp(evt, "done") == 0) { + ctx->ws_done = true; + } else if (strcmp(evt, "error") == 0) { + cJSON* msg_item = cJSON_GetObjectItem(json, "message"); + if (msg_item != NULL && msg_item->valuestring != NULL) { + strncpy(ctx->error_message, msg_item->valuestring, sizeof(ctx->error_message) - 1); + } else { + snprintf(ctx->error_message, sizeof(ctx->error_message), "Unknown server error"); + } + ctx->state = STATE_ERROR; + ctx->ui_update_pending = true; + } + } + cJSON_Delete(json); +} + +/* ─── WebSocket RX (Receive) Task ─── */ +static void reynabot_rx_task(void* arg) { + ReynaBotCtx* ctx = (ReynaBotCtx*)arg; + uint8_t* rx_buf = malloc(8192); + if (rx_buf == NULL) { + ESP_LOGE(TAG, "Failed to allocate RX buffer"); + ctx->ws_done = true; + ctx->rx_task = NULL; + vTaskDelete(NULL); + return; + } + int opcode; + + ESP_LOGI(TAG, "WS Receive task started"); + + while (ctx->ws_fd >= 0) { + int r = ws_recv(ctx->ws_fd, &opcode, rx_buf, 8191); + if (r < 0) { + if (r == -1) { + ESP_LOGI(TAG, "WS connection closed or read error, errno=%d", errno); + } else { + ESP_LOGE(TAG, "WS rx buffer overflow"); + } + ctx->ws_done = true; + break; + } + + if (opcode == 0x01) { // Text frame (JSON) + rx_buf[r] = '\0'; + parse_json_message(ctx, (char*)rx_buf); + } else if (opcode == 0x02) { // Binary frame (Audio data) + if (ctx->state == STATE_SPEAKING && ctx->i2s_dev != NULL) { + const uint8_t* payload_ptr = rx_buf; + size_t payload_len = r; + + // Skip WAV header if present in the first chunk + if (payload_len > 44 && memcmp(payload_ptr, "RIFF", 4) == 0) { + payload_ptr += 44; + payload_len -= 44; + } + + size_t written = 0; + device_lock(ctx->i2s_dev); + i2s_controller_write(ctx->i2s_dev, payload_ptr, payload_len, &written, pdMS_TO_TICKS(100)); + device_unlock(ctx->i2s_dev); + } + } else if (opcode == 0x09) { // PING frame + ESP_LOGI(TAG, "WS PING received, sending PONG"); + ws_send_pong(ctx->ws_fd, rx_buf, r); + } + } + + free(rx_buf); + ctx->rx_task = NULL; + vTaskDelete(NULL); +} + +/* ─── UI Press-to-Talk Event Callback ─── */ +static void ptt_event_cb(lv_event_t* e) { + ReynaBotCtx* ctx = (ReynaBotCtx*)lv_event_get_user_data(e); + lv_event_code_t code = lv_event_get_code(e); + + if (code == LV_EVENT_PRESSED) { + ctx->is_pressed = true; + if (ctx->state == STATE_IDLE || ctx->state == STATE_ERROR) { + ctx->start_session = true; + } + } else if (code == LV_EVENT_RELEASED || code == LV_EVENT_PRESS_LOST) { + ctx->is_pressed = false; + if (ctx->state == STATE_LISTENING) { + ctx->stop_session = true; + } else if (ctx->state == STATE_CONNECTING) { + ctx->cancel_session = true; + } + } +} + +/* ─── Core UI Update Function ─── */ +static void update_ui(ReynaBotCtx* ctx) { + if (!ctx->visible) return; + + if (tt_lvgl_lock(pdMS_TO_TICKS(500))) { + if (!ctx->visible) { + tt_lvgl_unlock(); + return; + } + + switch (ctx->state) { + case STATE_IDLE: + lv_label_set_text(ctx->lbl_status, "Ready"); + lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0x9E9E9E), LV_PART_MAIN); + lv_label_set_text(ctx->btn_ptt_label, LV_SYMBOL_AUDIO " Hold to Talk"); + lv_obj_set_style_bg_color(ctx->btn_ptt, lv_color_hex(0x6200EE), LV_PART_MAIN); + lv_obj_remove_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN); + break; + case STATE_CONNECTING: + lv_label_set_text(ctx->lbl_status, "Connecting..."); + lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0xFFC107), LV_PART_MAIN); + lv_label_set_text(ctx->btn_ptt_label, "Connecting..."); + lv_obj_set_style_bg_color(ctx->btn_ptt, lv_color_hex(0x757575), LV_PART_MAIN); + lv_obj_remove_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN); + break; + case STATE_LISTENING: + lv_label_set_text(ctx->lbl_status, "Listening..."); + lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0x00E676), LV_PART_MAIN); + lv_label_set_text(ctx->btn_ptt_label, "Release to Stop"); + lv_obj_set_style_bg_color(ctx->btn_ptt, lv_color_hex(0xD50000), LV_PART_MAIN); + lv_obj_remove_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN); + break; + case STATE_THINKING: + lv_label_set_text(ctx->lbl_status, "Thinking..."); + lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0x00B0FF), LV_PART_MAIN); + lv_obj_add_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN); + break; + case STATE_SPEAKING: + lv_label_set_text(ctx->lbl_status, "Speaking..."); + lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0xAA00FF), LV_PART_MAIN); + lv_obj_add_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN); + break; + case STATE_ERROR: + lv_label_set_text(ctx->lbl_status, "Error"); + lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0xFF1744), LV_PART_MAIN); + lv_label_set_text(ctx->btn_ptt_label, LV_SYMBOL_REFRESH " Try Again"); + lv_obj_set_style_bg_color(ctx->btn_ptt, lv_color_hex(0xFF1744), LV_PART_MAIN); + lv_obj_remove_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN); + + // Show the full error message in the bot's response bubble + if (ctx->error_message[0] != '\0') { + snprintf(ctx->last_response, sizeof(ctx->last_response), "Error: %s", ctx->error_message); + } + break; + } + + lv_label_set_text(ctx->lbl_transcript, ctx->last_transcript[0] != '\0' ? ctx->last_transcript : "(Your question will appear here)"); + lv_label_set_text(ctx->lbl_response, ctx->last_response[0] != '\0' ? ctx->last_response : "(Answer will appear here)"); + + tt_lvgl_unlock(); + } +} + +/* ─── Streaming & Main Worker Task ─── */ +static void reynabot_task(void* arg) { + ReynaBotCtx* ctx = (ReynaBotCtx*)arg; + char host[64]; + char path[128]; + int port = 80; + + ctx->ws_fd = -1; + + + while (ctx->visible) { + if (ctx->start_session) { + ctx->start_session = false; + ctx->stop_session = false; + ctx->cancel_session = false; + + ctx->state = STATE_CONNECTING; + ctx->last_transcript[0] = '\0'; + ctx->last_response[0] = '\0'; + ctx->error_message[0] = '\0'; + ctx->ws_done = false; + update_ui(ctx); + + if (!parse_ws_url(ctx->server_url, host, &port, path)) { + ctx->state = STATE_ERROR; + snprintf(ctx->error_message, sizeof(ctx->error_message), "Invalid Server URL"); + update_ui(ctx); + continue; + } + + ESP_LOGI(TAG, "Connecting: host=%s port=%d path=%s", host, port, path); + int fd = ws_connect(host, port, path, ctx->device_id, ctx->api_key); + if (fd < 0) { + ctx->state = STATE_ERROR; + snprintf(ctx->error_message, sizeof(ctx->error_message), "Connect failed"); + update_ui(ctx); + continue; + } + + ctx->ws_fd = fd; + ctx->ws_connected = true; + + // Configure I2S on demand for this session (16 kHz, 16-bit, mono) + struct I2sConfig session_cfg = { + .communication_format = I2S_FORMAT_STAND_I2S, + .sample_rate = 16000, + .bits_per_sample = 16, + .channel_left = 0, + .channel_right = I2S_CHANNEL_NONE + }; + if (ctx->i2s_dev != NULL) { + device_lock(ctx->i2s_dev); + i2s_controller_set_config(ctx->i2s_dev, &session_cfg); + device_unlock(ctx->i2s_dev); + } + + // Spawn background RX task to read and parse events + xTaskCreate(reynabot_rx_task, "reynabot_rx", 4096, ctx, 6, &ctx->rx_task); + + // Send start event handshake + char start_json[256]; + snprintf(start_json, sizeof(start_json), + "{\"event\":\"start\",\"device_id\":\"%s\",\"sample_rate\":16000,\"channels\":1,\"sample_width\":2,\"format\":\"pcm_s16le\"}", + ctx->device_id); + if (ws_send(ctx->ws_fd, (const uint8_t*)start_json, strlen(start_json), false) < 0) { + ctx->state = STATE_ERROR; + snprintf(ctx->error_message, sizeof(ctx->error_message), "Handshake send failed"); + ws_close(ctx->ws_fd); + ctx->ws_fd = -1; + ctx->ws_connected = false; + update_ui(ctx); + continue; + } + + // Wait for server to switch us to LISTENING + int timeout_ms = 4000; + while (ctx->state != STATE_LISTENING && timeout_ms > 0 && !ctx->cancel_session && !ctx->ws_done) { + vTaskDelay(pdMS_TO_TICKS(50)); + timeout_ms -= 50; + } + + if (ctx->state != STATE_LISTENING) { + if (ctx->state != STATE_ERROR) { + ctx->state = STATE_ERROR; + snprintf(ctx->error_message, sizeof(ctx->error_message), "Handshake timeout"); + } + ws_close(ctx->ws_fd); + ctx->ws_fd = -1; + ctx->ws_connected = false; + update_ui(ctx); + continue; + } + + // I2S is already configured globally at session startup + + uint8_t* buffer = malloc(1024); + if (buffer == NULL) { + ESP_LOGE(TAG, "Failed to allocate record buffer"); + ctx->state = STATE_ERROR; + snprintf(ctx->error_message, sizeof(ctx->error_message), "Out of memory"); + ws_close(ctx->ws_fd); + ctx->ws_fd = -1; + ctx->ws_connected = false; + update_ui(ctx); + continue; + } + size_t total_sent_bytes = 0; + + // Stream audio loop while PTT is held + while (ctx->is_pressed && !ctx->stop_session && !ctx->cancel_session && !ctx->ws_done && total_sent_bytes < 320000) { + size_t bytes_read = 0; + error_t r = i2s_controller_read(ctx->i2s_dev, buffer, 1024, &bytes_read, pdMS_TO_TICKS(100)); + if (r == ERROR_NONE && bytes_read > 0) { + if (ws_send(ctx->ws_fd, buffer, bytes_read, true) < 0) { + ESP_LOGE(TAG, "Audio stream send failed"); + break; + } + total_sent_bytes += bytes_read; + } + } + + free(buffer); + + if (ctx->cancel_session || total_sent_bytes < 3200) { + ESP_LOGI(TAG, "Cancelling audio session"); + const char* cancel_json = "{\"event\":\"cancel\"}"; + ws_send(ctx->ws_fd, (const uint8_t*)cancel_json, strlen(cancel_json), false); + ctx->state = STATE_IDLE; + } else { + const char* stop_json = "{\"event\":\"stop\"}"; + ws_send(ctx->ws_fd, (const uint8_t*)stop_json, strlen(stop_json), false); + ctx->state = STATE_THINKING; + update_ui(ctx); + + // Wait for response to finish (increase to 90s to match socket timeout) + int wait_timeout_ms = 90000; + while (!ctx->ws_done && ctx->state != STATE_ERROR && wait_timeout_ms > 0) { + vTaskDelay(pdMS_TO_TICKS(100)); + wait_timeout_ms -= 100; + if (ctx->ui_update_pending) { + ctx->ui_update_pending = false; + update_ui(ctx); + } + } + + if (!ctx->ws_done && ctx->state != STATE_ERROR) { + ctx->state = STATE_ERROR; + snprintf(ctx->error_message, sizeof(ctx->error_message), "Response timeout"); + } + } + + // Clean up socket + int fd_to_close = ctx->ws_fd; + ctx->ws_fd = -1; + ctx->ws_connected = false; + ws_close(fd_to_close); + + // Reset I2S controller to release DMA and stop white noise + if (ctx->i2s_dev != NULL) { + device_lock(ctx->i2s_dev); + i2s_controller_reset(ctx->i2s_dev); + device_unlock(ctx->i2s_dev); + } + + // Wait for receive task to exit + int rx_timeout = 100; + while (ctx->rx_task != NULL && rx_timeout > 0) { + vTaskDelay(pdMS_TO_TICKS(10)); + rx_timeout--; + } + + if (ctx->state != STATE_ERROR) { + ctx->state = STATE_IDLE; + } + update_ui(ctx); + } + + vTaskDelay(pdMS_TO_TICKS(100)); + if (ctx->ui_update_pending) { + ctx->ui_update_pending = false; + update_ui(ctx); + } + } + + ctx->worker_task = NULL; + vTaskDelete(NULL); +} + +/* ─── App Lifecycle ─── */ + +static void* create_data(void) { + ReynaBotCtx* ctx = calloc(1, sizeof(ReynaBotCtx)); + if (ctx != NULL) { + ctx->state = STATE_IDLE; + ctx->ws_fd = -1; + } + return ctx; +} + +static void destroy_data(void* data) { + free(data); +} + +static void on_create(AppHandle app, void* data) { + ReynaBotCtx* ctx = (ReynaBotCtx*)data; + ctx->app = app; +} + +static void on_destroy(AppHandle app, void* data) { + // No-op +} + +static void on_show(AppHandle app, void* data, lv_obj_t* parent) { + ReynaBotCtx* ctx = (ReynaBotCtx*)data; + ctx->visible = true; + + load_config(ctx); + + // Find I2S controller + ctx->i2s_dev = device_find_by_name("i2s0"); + if (ctx->i2s_dev == NULL) { + ESP_LOGE(TAG, "I2S controller 'i2s0' not found!"); + } + + // Style the parent screen + lv_obj_set_flex_flow(parent, LV_FLEX_FLOW_COLUMN); + lv_obj_set_flex_align(parent, LV_FLEX_ALIGN_START, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER); + lv_obj_set_style_pad_all(parent, 10, LV_PART_MAIN); + lv_obj_set_style_pad_row(parent, 12, LV_PART_MAIN); + lv_obj_set_style_bg_color(parent, lv_color_hex(0x0C0B12), LV_PART_MAIN); // Rich dark background + + // Add App Toolbar + lv_obj_t* toolbar = tt_lvgl_toolbar_create_for_app(parent, app); + lv_obj_align(toolbar, LV_ALIGN_TOP_MID, 0, 0); + + // Status Label inside toolbar (replaces the static "ReynaBot Voice" label) + ctx->lbl_status = lv_label_create(toolbar); + lv_label_set_text(ctx->lbl_status, "Ready"); + lv_obj_add_flag(ctx->lbl_status, LV_OBJ_FLAG_FLOATING); + lv_obj_align(ctx->lbl_status, LV_ALIGN_RIGHT_MID, -10, 0); + lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0x9E9E9E), LV_PART_MAIN); + + // Conversation bubble container + lv_obj_t* conv_card = lv_obj_create(parent); + lv_obj_set_width(conv_card, lv_pct(95)); + lv_obj_set_flex_grow(conv_card, 1); + lv_obj_set_flex_flow(conv_card, LV_FLEX_FLOW_COLUMN); + lv_obj_set_flex_align(conv_card, LV_FLEX_ALIGN_START, LV_FLEX_ALIGN_START, LV_FLEX_ALIGN_START); + lv_obj_set_style_pad_all(conv_card, 12, LV_PART_MAIN); + lv_obj_set_style_pad_row(conv_card, 10, LV_PART_MAIN); + lv_obj_set_style_bg_color(conv_card, lv_color_hex(0x13121A), LV_PART_MAIN); + lv_obj_set_style_border_color(conv_card, lv_color_hex(0x23212C), LV_PART_MAIN); + lv_obj_set_style_border_width(conv_card, 1, LV_PART_MAIN); + lv_obj_set_style_radius(conv_card, 12, LV_PART_MAIN); + + // User bubble + lv_obj_t* user_label_title = lv_label_create(conv_card); + lv_label_set_text(user_label_title, "👤 You:"); + lv_obj_set_style_text_color(user_label_title, lv_color_hex(0x00E676), LV_PART_MAIN); + + ctx->lbl_transcript = lv_label_create(conv_card); + lv_label_set_text(ctx->lbl_transcript, "(Your question will appear here)"); + lv_label_set_long_mode(ctx->lbl_transcript, LV_LABEL_LONG_WRAP); + lv_obj_set_width(ctx->lbl_transcript, lv_pct(100)); + lv_obj_set_style_text_color(ctx->lbl_transcript, lv_color_hex(0xE0E0E0), LV_PART_MAIN); + + // Separator line + lv_obj_t* sep = lv_line_create(conv_card); + static lv_point_precise_t line_points[] = { {0, 0}, {300, 0} }; + lv_line_set_points(sep, line_points, 2); + lv_obj_set_style_line_color(sep, lv_color_hex(0x2D2B36), LV_PART_MAIN); + lv_obj_set_style_line_width(sep, 1, LV_PART_MAIN); + lv_obj_set_width(sep, lv_pct(100)); + + // Assistant bubble + lv_obj_t* bot_label_title = lv_label_create(conv_card); + lv_label_set_text(bot_label_title, "🤖 ReynaBot:"); + lv_obj_set_style_text_color(bot_label_title, lv_color_hex(0xAA00FF), LV_PART_MAIN); + + ctx->lbl_response = lv_label_create(conv_card); + lv_label_set_text(ctx->lbl_response, "(Answer will appear here)"); + lv_label_set_long_mode(ctx->lbl_response, LV_LABEL_LONG_WRAP); + lv_obj_set_width(ctx->lbl_response, lv_pct(100)); + lv_obj_set_style_text_color(ctx->lbl_response, lv_color_hex(0xE0E0E0), LV_PART_MAIN); + + // Push-to-Talk Button + ctx->btn_ptt = lv_btn_create(parent); + lv_obj_set_size(ctx->btn_ptt, 180, 50); + lv_obj_set_style_radius(ctx->btn_ptt, 25, LV_PART_MAIN); + lv_obj_set_style_bg_color(ctx->btn_ptt, lv_color_hex(0x6200EE), LV_PART_MAIN); + + ctx->btn_ptt_label = lv_label_create(ctx->btn_ptt); + lv_label_set_text(ctx->btn_ptt_label, LV_SYMBOL_AUDIO " Hold to Talk"); + lv_obj_center(ctx->btn_ptt_label); + + lv_obj_add_event_cb(ctx->btn_ptt, ptt_event_cb, LV_EVENT_ALL, ctx); + + // Launch main worker task + xTaskCreate(reynabot_task, "reynabot_worker", 6144, ctx, 5, &ctx->worker_task); +} + +static void on_hide(AppHandle app, void* data) { + ReynaBotCtx* ctx = (ReynaBotCtx*)data; + if (ctx == NULL) return; + + ESP_LOGI(TAG, "on_hide: cleaning up"); + ctx->visible = false; + + if (ctx->ws_fd >= 0) { + int fd_to_close = ctx->ws_fd; + ctx->ws_fd = -1; + ws_close(fd_to_close); + } + + // Reset I2S controller to stop DMA and looping noise + if (ctx->i2s_dev != NULL) { + device_lock(ctx->i2s_dev); + i2s_controller_reset(ctx->i2s_dev); + device_unlock(ctx->i2s_dev); + } + + // Wait briefly for tasks to exit + int timeout = 100; + while ((ctx->worker_task != NULL || ctx->rx_task != NULL) && timeout > 0) { + vTaskDelay(pdMS_TO_TICKS(10)); + timeout--; + } +} + +int main(int argc, char* argv[]) { + (void)argc; + (void)argv; + tt_app_register((AppRegistration) { + .createData = create_data, + .destroyData = destroy_data, + .onCreate = on_create, + .onDestroy = on_destroy, + .onShow = on_show, + .onHide = on_hide + }); + return 0; +} diff --git a/Apps/ReynaBot/main/Source/websocket.c b/Apps/ReynaBot/main/Source/websocket.c new file mode 100644 index 0000000..caffac5 --- /dev/null +++ b/Apps/ReynaBot/main/Source/websocket.c @@ -0,0 +1,240 @@ +#include "websocket.h" + +#include +#include +#include +#include + +#include +#include +#include + +static int recv_all(int fd, void* buf, size_t len) { + size_t total = 0; + char* p = (char*)buf; + while (total < len) { + int r = lwip_recv(fd, p + total, len - total, 0); + if (r <= 0) { + return -1; + } + total += r; + } + return 0; +} + +static uint16_t my_htons(uint16_t val) { + return (uint16_t)(((val & 0xff) << 8) | ((val & 0xff00) >> 8)); +} + +int ws_connect(const char* host, int port, const char* path, const char* device_id, const char* auth_key) { + int fd = lwip_socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) return -1; + + struct sockaddr_in addr; + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = my_htons(port); + addr.sin_addr.s_addr = ipaddr_addr(host); + + if (lwip_connect(fd, (struct sockaddr*)&addr, sizeof(addr)) < 0) { + close(fd); + return -1; + } + + // Set socket receive timeout (e.g. 90 seconds) to prevent blocking indefinitely + struct timeval tv; + tv.tv_sec = 90; + tv.tv_usec = 0; + lwip_setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + + // Send HTTP upgrade handshake request + char req[1024]; + snprintf(req, sizeof(req), + "GET %s HTTP/1.1\r\n" + "Host: %s:%d\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n" + "Sec-WebSocket-Version: 13\r\n" + "Authorization: Bearer %s\r\n" + "X-Device-ID: %s\r\n" + "\r\n", + path, host, port, auth_key, device_id); + + if (lwip_send(fd, req, strlen(req), 0) < 0) { + close(fd); + return -1; + } + + // Read HTTP response headers until we hit "\r\n\r\n" + char header_buf[1024]; + size_t header_len = 0; + while (header_len < sizeof(header_buf) - 1) { + char c; + int r = lwip_recv(fd, &c, 1, 0); + if (r <= 0) { + close(fd); + return -1; + } + header_buf[header_len++] = c; + header_buf[header_len] = '\0'; + + if (header_len >= 4 && strcmp(header_buf + header_len - 4, "\r\n\r\n") == 0) { + break; + } + } + + // Verify HTTP 101 Switching Protocols response status + if (strstr(header_buf, "HTTP/1.1 101") == NULL && strstr(header_buf, "HTTP/1.0 101") == NULL) { + close(fd); + return -1; + } + + return fd; +} + +int ws_send(int fd, const uint8_t* data, size_t len, bool binary) { + uint8_t header[10]; + size_t header_len = 0; + + header[0] = binary ? 0x82 : 0x81; + + if (len < 126) { + header[1] = 0x80 | (uint8_t)len; + header_len = 2; + } else { + header[1] = 0x80 | 126; + header[2] = (uint8_t)((len >> 8) & 0xFF); + header[3] = (uint8_t)(len & 0xFF); + header_len = 4; + } + + // Use fixed client mask for performance: 0x12, 0x34, 0x56, 0x78 + uint8_t mask[4] = { 0x12, 0x34, 0x56, 0x78 }; + memcpy(header + header_len, mask, 4); + header_len += 4; + + // Send WebSocket frame header + int sent = lwip_send(fd, header, header_len, 0); + if (sent < 0) return -1; + + // Mask the payload + uint8_t* masked = malloc(len); + if (masked == NULL) return -1; + for (size_t i = 0; i < len; ++i) { + masked[i] = data[i] ^ mask[i % 4]; + } + + // Send masked payload + sent = lwip_send(fd, masked, len, 0); + free(masked); + + return sent >= 0 ? 0 : -1; +} + +int ws_recv(int fd, int* out_opcode, uint8_t* payload, size_t max_len) { + uint8_t header[2]; + if (recv_all(fd, header, 2) < 0) { + return -1; + } + + int opcode = header[0] & 0x0F; + if (out_opcode != NULL) { + *out_opcode = opcode; + } + + int masked = (header[1] & 0x80) != 0; + size_t len = header[1] & 0x7F; + + if (len == 126) { + uint8_t ext_len[2]; + if (recv_all(fd, ext_len, 2) < 0) return -1; + len = ((size_t)ext_len[0] << 8) | ext_len[1]; + } else if (len == 127) { + uint8_t ext_len[8]; + if (recv_all(fd, ext_len, 8) < 0) return -1; + // Parse 64-bit length into size_t + len = ((size_t)ext_len[4] << 24) | ((size_t)ext_len[5] << 16) | ((size_t)ext_len[6] << 8) | ext_len[7]; + } + + if (masked) { + uint8_t mask[4]; + if (recv_all(fd, mask, 4) < 0) return -1; + + if (len > max_len) { + ESP_LOGE("websocket", "ws_recv overflow (masked): len=%u, max_len=%u", (unsigned)len, (unsigned)max_len); + // Buffer overflow, skip payload to align stream + size_t to_discard = len; + uint8_t discard_buf[256]; + while (to_discard > 0) { + size_t chunk = to_discard < sizeof(discard_buf) ? to_discard : sizeof(discard_buf); + if (recv_all(fd, discard_buf, chunk) < 0) return -1; + to_discard -= chunk; + } + return -2; + } + + if (recv_all(fd, payload, len) < 0) return -1; + for (size_t i = 0; i < len; ++i) { + payload[i] ^= mask[i % 4]; + } + } else { + if (len > max_len) { + ESP_LOGE("websocket", "ws_recv overflow (unmasked): len=%u, max_len=%u", (unsigned)len, (unsigned)max_len); + size_t to_discard = len; + uint8_t discard_buf[256]; + while (to_discard > 0) { + size_t chunk = to_discard < sizeof(discard_buf) ? to_discard : sizeof(discard_buf); + if (recv_all(fd, discard_buf, chunk) < 0) return -1; + to_discard -= chunk; + } + return -2; + } + + if (recv_all(fd, payload, len) < 0) return -1; + } + + return (int)len; +} + +void ws_close(int fd) { + if (fd >= 0) { + close(fd); + } +} + +int ws_send_pong(int fd, const uint8_t* payload, size_t len) { + uint8_t header[10]; + size_t header_len = 0; + + header[0] = 0x8A; // FIN | PONG (0x0A) + + if (len < 126) { + header[1] = 0x80 | (uint8_t)len; + header_len = 2; + } else { + header[1] = 0x80 | 126; + header[2] = (uint8_t)((len >> 8) & 0xFF); + header[3] = (uint8_t)(len & 0xFF); + header_len = 4; + } + + uint8_t mask[4] = { 0x12, 0x34, 0x56, 0x78 }; + memcpy(header + header_len, mask, 4); + header_len += 4; + + int sent = lwip_send(fd, header, header_len, 0); + if (sent < 0) return -1; + + if (len > 0 && payload != NULL) { + uint8_t* masked = malloc(len); + if (masked == NULL) return -1; + for (size_t i = 0; i < len; ++i) { + masked[i] = payload[i] ^ mask[i % 4]; + } + sent = lwip_send(fd, masked, len, 0); + free(masked); + } + + return sent >= 0 ? 0 : -1; +} diff --git a/Apps/ReynaBot/main/Source/websocket.h b/Apps/ReynaBot/main/Source/websocket.h new file mode 100644 index 0000000..396d430 --- /dev/null +++ b/Apps/ReynaBot/main/Source/websocket.h @@ -0,0 +1,59 @@ +#pragma once + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Connect to a WebSocket server. + * @param host Server IP address (e.g. "192.168.68.126") + * @param port Port number (e.g. 8642) + * @param path WebSocket path (e.g. "/api/esp32/voice/ws") + * @param device_id Unique device identifier + * @param auth_key Hermes Bearer API key + * @return Socket file descriptor on success, or -1 on failure + */ +int ws_connect(const char* host, int port, const char* path, const char* device_id, const char* auth_key); + +/** + * Send a WebSocket frame. + * @param fd Socket file descriptor + * @param data Data payload to send + * @param len Length of the data payload + * @param binary True for binary frame, false for text frame + * @return 0 on success, or -1 on failure + */ +int ws_send(int fd, const uint8_t* data, size_t len, bool binary); + +/** + * Receive a WebSocket frame. + * @param fd Socket file descriptor + * @param out_opcode Pointer to store the received opcode (e.g. 0x01 text, 0x02 binary) + * @param payload Buffer to store the received payload + * @param max_len Maximum length of the payload buffer + * @return Received payload length on success, -1 on connection failure, or -2 on buffer overflow + */ +int ws_recv(int fd, int* out_opcode, uint8_t* payload, size_t max_len); + +/** + * Close a WebSocket connection. + * @param fd Socket file descriptor + */ +void ws_close(int fd); + +/** + * Send a WebSocket PONG frame. + * @param fd Socket file descriptor + * @param payload Payload to reflect + * @param len Length of payload + * @return 0 on success, or -1 on failure + */ +int ws_send_pong(int fd, const uint8_t* payload, size_t len); + +#ifdef __cplusplus +} +#endif diff --git a/Apps/ReynaBot/manifest.properties b/Apps/ReynaBot/manifest.properties new file mode 100644 index 0000000..8214320 --- /dev/null +++ b/Apps/ReynaBot/manifest.properties @@ -0,0 +1,10 @@ +[manifest] +version=0.1 +[target] +sdk=0.7.0-dev +platforms=esp32s3 +[app] +id=one.tactility.reynabot +versionName=1.0.0 +versionCode=1 +name=ReynaBot