adolforeyna/tactility_apps
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(reynabot): Improve UI and update voice endpoint to kids gateway

Browse Source
This commit is contained in:
Adolfo Reyna
2026-07-01 13:45:04 -04:00
parent 98ae52d3a8
commit a4be0d73ae
6 changed files with 1033 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Apps/ReynaBot/main/Source/main.c
+700
View File
@@ -0,0 +1,700 @@
#include <tt_app.h>
#include <tt_lvgl.h>
#include <tt_lvgl_toolbar.h>
#include <tt_wifi.h>
#include <tactility/device.h>
#include <tactility/drivers/i2s_controller.h>
#include "websocket.h"
#include <cJSON.h>
#include <string.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <errno.h>
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "esp_log.h"
#define TAG "ReynaBot"
typedef enum {
STATE_IDLE,
STATE_CONNECTING,
STATE_LISTENING,
STATE_THINKING,
STATE_SPEAKING,
STATE_ERROR
} ReynaBotState;
typedef struct {
AppHandle app;
bool visible;
ReynaBotState state;
char server_url[128];
char device_id[64];
char api_key[128];
char last_transcript[256];
char last_response[512];
char error_message[128];
// UI components
lv_obj_t* lbl_status;
lv_obj_t* lbl_transcript;
lv_obj_t* lbl_response;
lv_obj_t* btn_ptt;
lv_obj_t* btn_ptt_label;
// PTT control
bool is_pressed;
bool start_session;
bool stop_session;
bool cancel_session;
// FreeRTOS Tasks
TaskHandle_t worker_task;
TaskHandle_t rx_task;
// Hardware
struct Device* i2s_dev;
// WebSocket
int ws_fd;
bool ws_connected;
bool ws_done;
bool ui_update_pending;
} ReynaBotCtx;
/* ─── Forward Declarations ─── */
static void reynabot_task(void* arg);
static void reynabot_rx_task(void* arg);
static void update_ui(ReynaBotCtx* ctx);
static void load_config(ReynaBotCtx* ctx);
/* ─── Helper for URL parsing ─── */
static bool parse_ws_url(const char* url, char* host, int* port, char* path) {
if (strncmp(url, "ws://", 5) != 0) return false;
const char* p = url + 5;
const char* colon = strchr(p, ':');
const char* slash = strchr(p, '/');
if (slash == NULL) {
strcpy(path, "/");
} else {
strcpy(path, slash);
}
if (colon != NULL && (slash == NULL || colon < slash)) {
int host_len = colon - p;
memcpy(host, p, host_len);
host[host_len] = '\0';
*port = atoi(colon + 1);
} else {
int host_len = slash ? (slash - p) : strlen(p);
memcpy(host, p, host_len);
host[host_len] = '\0';
*port = 80;
}
return true;
}
/* ─── Config Loading/Saving ─── */
static void load_config(ReynaBotCtx* ctx) {
// Default fallback config
snprintf(ctx->server_url, sizeof(ctx->server_url), "ws://192.168.68.126:8643/api/esp32/voice/ws");
snprintf(ctx->device_id, sizeof(ctx->device_id), "reynabot_screen");
snprintf(ctx->api_key, sizeof(ctx->api_key), "hmek_sXB7921bZ9FXTVKARExqUZ7ttBxtEoURHRU0JCB-gNY");
char path[256];
size_t path_size = sizeof(path);
tt_app_get_user_data_child_path(ctx->app, "config.json", path, &path_size);
FILE* file = fopen(path, "r");
if (file == NULL) {
// Create user data directory if it doesn't exist
char dir_path[256];
size_t dir_size = sizeof(dir_path);
tt_app_get_user_data_path(ctx->app, dir_path, &dir_size);
mkdir(dir_path, 0755);
// Write default config file
file = fopen(path, "w");
if (file != NULL) {
fprintf(file, "{\n \"server_url\": \"ws://192.168.68.126:8643/api/esp32/voice/ws\",\n \"device_id\": \"reynabot_screen\",\n \"api_key\": \"mcT1YA1vOr9wXSiHpCYalweEGGZKX-PIfZv2drp8BSg\"\n}\n");
fclose(file);
}
ESP_LOGI(TAG, "Created default config.json at %s", path);
return;
}
fseek(file, 0, SEEK_END);
long size = ftell(file);
fseek(file, 0, SEEK_SET);
if (size > 0 && size < 4096) {
char* buf = malloc(size + 1);
if (buf != NULL) {
size_t read_bytes = fread(buf, 1, size, file);
buf[read_bytes] = '\0';
cJSON* json = cJSON_Parse(buf);
if (json != NULL) {
cJSON* url_item = cJSON_GetObjectItem(json, "server_url");
if (url_item != NULL && url_item->valuestring != NULL) {
strncpy(ctx->server_url, url_item->valuestring, sizeof(ctx->server_url) - 1);
}
cJSON* dev_item = cJSON_GetObjectItem(json, "device_id");
if (dev_item != NULL && dev_item->valuestring != NULL) {
strncpy(ctx->device_id, dev_item->valuestring, sizeof(ctx->device_id) - 1);
}
cJSON* key_item = cJSON_GetObjectItem(json, "api_key");
if (key_item != NULL && key_item->valuestring != NULL) {
strncpy(ctx->api_key, key_item->valuestring, sizeof(ctx->api_key) - 1);
}
cJSON_Delete(json);
}
free(buf);
}
}
fclose(file);
ESP_LOGI(TAG, "Loaded config: server=%s device=%s", ctx->server_url, ctx->device_id);
}
/* ─── JSON Message Processing ─── */
static void parse_json_message(ReynaBotCtx* ctx, const char* json_str) {
cJSON* json = cJSON_Parse(json_str);
if (json == NULL) return;
cJSON* evt_item = cJSON_GetObjectItem(json, "event");
if (evt_item != NULL && evt_item->valuestring != NULL) {
const char* evt = evt_item->valuestring;
ESP_LOGI(TAG, "WS Event: %s", evt);
if (strcmp(evt, "ready") == 0) {
ESP_LOGI(TAG, "Server ready");
} else if (strcmp(evt, "listening") == 0) {
ctx->state = STATE_LISTENING;
ctx->ui_update_pending = true;
} else if (strcmp(evt, "transcript") == 0) {
cJSON* txt_item = cJSON_GetObjectItem(json, "text");
if (txt_item != NULL && txt_item->valuestring != NULL) {
strncpy(ctx->last_transcript, txt_item->valuestring, sizeof(ctx->last_transcript) - 1);
ctx->ui_update_pending = true;
}
} else if (strcmp(evt, "thinking") == 0) {
ctx->state = STATE_THINKING;
ctx->ui_update_pending = true;
} else if (strcmp(evt, "response_text") == 0) {
cJSON* txt_item = cJSON_GetObjectItem(json, "text");
if (txt_item != NULL && txt_item->valuestring != NULL) {
strncpy(ctx->last_response, txt_item->valuestring, sizeof(ctx->last_response) - 1);
ctx->ui_update_pending = true;
}
} else if (strcmp(evt, "audio_start") == 0) {
ctx->state = STATE_SPEAKING;
ctx->ui_update_pending = true;
// I2S is already configured globally at session startup
} else if (strcmp(evt, "audio_end") == 0) {
ESP_LOGI(TAG, "Audio response ended");
} else if (strcmp(evt, "done") == 0) {
ctx->ws_done = true;
} else if (strcmp(evt, "error") == 0) {
cJSON* msg_item = cJSON_GetObjectItem(json, "message");
if (msg_item != NULL && msg_item->valuestring != NULL) {
strncpy(ctx->error_message, msg_item->valuestring, sizeof(ctx->error_message) - 1);
} else {
snprintf(ctx->error_message, sizeof(ctx->error_message), "Unknown server error");
}
ctx->state = STATE_ERROR;
ctx->ui_update_pending = true;
}
}
cJSON_Delete(json);
}
/* ─── WebSocket RX (Receive) Task ─── */
static void reynabot_rx_task(void* arg) {
ReynaBotCtx* ctx = (ReynaBotCtx*)arg;
uint8_t* rx_buf = malloc(8192);
if (rx_buf == NULL) {
ESP_LOGE(TAG, "Failed to allocate RX buffer");
ctx->ws_done = true;
ctx->rx_task = NULL;
vTaskDelete(NULL);
return;
}
int opcode;
ESP_LOGI(TAG, "WS Receive task started");
while (ctx->ws_fd >= 0) {
int r = ws_recv(ctx->ws_fd, &opcode, rx_buf, 8191);
if (r < 0) {
if (r == -1) {
ESP_LOGI(TAG, "WS connection closed or read error, errno=%d", errno);
} else {
ESP_LOGE(TAG, "WS rx buffer overflow");
}
ctx->ws_done = true;
break;
}
if (opcode == 0x01) { // Text frame (JSON)
rx_buf[r] = '\0';
parse_json_message(ctx, (char*)rx_buf);
} else if (opcode == 0x02) { // Binary frame (Audio data)
if (ctx->state == STATE_SPEAKING && ctx->i2s_dev != NULL) {
const uint8_t* payload_ptr = rx_buf;
size_t payload_len = r;
// Skip WAV header if present in the first chunk
if (payload_len > 44 && memcmp(payload_ptr, "RIFF", 4) == 0) {
payload_ptr += 44;
payload_len -= 44;
}
size_t written = 0;
device_lock(ctx->i2s_dev);
i2s_controller_write(ctx->i2s_dev, payload_ptr, payload_len, &written, pdMS_TO_TICKS(100));
device_unlock(ctx->i2s_dev);
}
} else if (opcode == 0x09) { // PING frame
ESP_LOGI(TAG, "WS PING received, sending PONG");
ws_send_pong(ctx->ws_fd, rx_buf, r);
}
}
free(rx_buf);
ctx->rx_task = NULL;
vTaskDelete(NULL);
}
/* ─── UI Press-to-Talk Event Callback ─── */
static void ptt_event_cb(lv_event_t* e) {
ReynaBotCtx* ctx = (ReynaBotCtx*)lv_event_get_user_data(e);
lv_event_code_t code = lv_event_get_code(e);
if (code == LV_EVENT_PRESSED) {
ctx->is_pressed = true;
if (ctx->state == STATE_IDLE || ctx->state == STATE_ERROR) {
ctx->start_session = true;
}
} else if (code == LV_EVENT_RELEASED || code == LV_EVENT_PRESS_LOST) {
ctx->is_pressed = false;
if (ctx->state == STATE_LISTENING) {
ctx->stop_session = true;
} else if (ctx->state == STATE_CONNECTING) {
ctx->cancel_session = true;
}
}
}
/* ─── Core UI Update Function ─── */
static void update_ui(ReynaBotCtx* ctx) {
if (!ctx->visible) return;
if (tt_lvgl_lock(pdMS_TO_TICKS(500))) {
if (!ctx->visible) {
tt_lvgl_unlock();
return;
}
switch (ctx->state) {
case STATE_IDLE:
lv_label_set_text(ctx->lbl_status, "Ready");
lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0x9E9E9E), LV_PART_MAIN);
lv_label_set_text(ctx->btn_ptt_label, LV_SYMBOL_AUDIO " Hold to Talk");
lv_obj_set_style_bg_color(ctx->btn_ptt, lv_color_hex(0x6200EE), LV_PART_MAIN);
lv_obj_remove_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN);
break;
case STATE_CONNECTING:
lv_label_set_text(ctx->lbl_status, "Connecting...");
lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0xFFC107), LV_PART_MAIN);
lv_label_set_text(ctx->btn_ptt_label, "Connecting...");
lv_obj_set_style_bg_color(ctx->btn_ptt, lv_color_hex(0x757575), LV_PART_MAIN);
lv_obj_remove_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN);
break;
case STATE_LISTENING:
lv_label_set_text(ctx->lbl_status, "Listening...");
lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0x00E676), LV_PART_MAIN);
lv_label_set_text(ctx->btn_ptt_label, "Release to Stop");
lv_obj_set_style_bg_color(ctx->btn_ptt, lv_color_hex(0xD50000), LV_PART_MAIN);
lv_obj_remove_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN);
break;
case STATE_THINKING:
lv_label_set_text(ctx->lbl_status, "Thinking...");
lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0x00B0FF), LV_PART_MAIN);
lv_obj_add_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN);
break;
case STATE_SPEAKING:
lv_label_set_text(ctx->lbl_status, "Speaking...");
lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0xAA00FF), LV_PART_MAIN);
lv_obj_add_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN);
break;
case STATE_ERROR:
lv_label_set_text(ctx->lbl_status, "Error");
lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0xFF1744), LV_PART_MAIN);
lv_label_set_text(ctx->btn_ptt_label, LV_SYMBOL_REFRESH " Try Again");
lv_obj_set_style_bg_color(ctx->btn_ptt, lv_color_hex(0xFF1744), LV_PART_MAIN);
lv_obj_remove_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN);
// Show the full error message in the bot's response bubble
if (ctx->error_message[0] != '\0') {
snprintf(ctx->last_response, sizeof(ctx->last_response), "Error: %s", ctx->error_message);
}
break;
}
lv_label_set_text(ctx->lbl_transcript, ctx->last_transcript[0] != '\0' ? ctx->last_transcript : "(Your question will appear here)");
lv_label_set_text(ctx->lbl_response, ctx->last_response[0] != '\0' ? ctx->last_response : "(Answer will appear here)");
tt_lvgl_unlock();
}
}
/* ─── Streaming & Main Worker Task ─── */
static void reynabot_task(void* arg) {
ReynaBotCtx* ctx = (ReynaBotCtx*)arg;
char host[64];
char path[128];
int port = 80;
ctx->ws_fd = -1;
while (ctx->visible) {
if (ctx->start_session) {
ctx->start_session = false;
ctx->stop_session = false;
ctx->cancel_session = false;
ctx->state = STATE_CONNECTING;
ctx->last_transcript[0] = '\0';
ctx->last_response[0] = '\0';
ctx->error_message[0] = '\0';
ctx->ws_done = false;
update_ui(ctx);
if (!parse_ws_url(ctx->server_url, host, &port, path)) {
ctx->state = STATE_ERROR;
snprintf(ctx->error_message, sizeof(ctx->error_message), "Invalid Server URL");
update_ui(ctx);
continue;
}
ESP_LOGI(TAG, "Connecting: host=%s port=%d path=%s", host, port, path);
int fd = ws_connect(host, port, path, ctx->device_id, ctx->api_key);
if (fd < 0) {
ctx->state = STATE_ERROR;
snprintf(ctx->error_message, sizeof(ctx->error_message), "Connect failed");
update_ui(ctx);
continue;
}
ctx->ws_fd = fd;
ctx->ws_connected = true;
// Configure I2S on demand for this session (16 kHz, 16-bit, mono)
struct I2sConfig session_cfg = {
.communication_format = I2S_FORMAT_STAND_I2S,
.sample_rate = 16000,
.bits_per_sample = 16,
.channel_left = 0,
.channel_right = I2S_CHANNEL_NONE
};
if (ctx->i2s_dev != NULL) {
device_lock(ctx->i2s_dev);
i2s_controller_set_config(ctx->i2s_dev, &session_cfg);
device_unlock(ctx->i2s_dev);
}
// Spawn background RX task to read and parse events
xTaskCreate(reynabot_rx_task, "reynabot_rx", 4096, ctx, 6, &ctx->rx_task);
// Send start event handshake
char start_json[256];
snprintf(start_json, sizeof(start_json),
"{\"event\":\"start\",\"device_id\":\"%s\",\"sample_rate\":16000,\"channels\":1,\"sample_width\":2,\"format\":\"pcm_s16le\"}",
ctx->device_id);
if (ws_send(ctx->ws_fd, (const uint8_t*)start_json, strlen(start_json), false) < 0) {
ctx->state = STATE_ERROR;
snprintf(ctx->error_message, sizeof(ctx->error_message), "Handshake send failed");
ws_close(ctx->ws_fd);
ctx->ws_fd = -1;
ctx->ws_connected = false;
update_ui(ctx);
continue;
}
// Wait for server to switch us to LISTENING
int timeout_ms = 4000;
while (ctx->state != STATE_LISTENING && timeout_ms > 0 && !ctx->cancel_session && !ctx->ws_done) {
vTaskDelay(pdMS_TO_TICKS(50));
timeout_ms -= 50;
}
if (ctx->state != STATE_LISTENING) {
if (ctx->state != STATE_ERROR) {
ctx->state = STATE_ERROR;
snprintf(ctx->error_message, sizeof(ctx->error_message), "Handshake timeout");
}
ws_close(ctx->ws_fd);
ctx->ws_fd = -1;
ctx->ws_connected = false;
update_ui(ctx);
continue;
}
// I2S is already configured globally at session startup
uint8_t* buffer = malloc(1024);
if (buffer == NULL) {
ESP_LOGE(TAG, "Failed to allocate record buffer");
ctx->state = STATE_ERROR;
snprintf(ctx->error_message, sizeof(ctx->error_message), "Out of memory");
ws_close(ctx->ws_fd);
ctx->ws_fd = -1;
ctx->ws_connected = false;
update_ui(ctx);
continue;
}
size_t total_sent_bytes = 0;
// Stream audio loop while PTT is held
while (ctx->is_pressed && !ctx->stop_session && !ctx->cancel_session && !ctx->ws_done && total_sent_bytes < 320000) {
size_t bytes_read = 0;
error_t r = i2s_controller_read(ctx->i2s_dev, buffer, 1024, &bytes_read, pdMS_TO_TICKS(100));
if (r == ERROR_NONE && bytes_read > 0) {
if (ws_send(ctx->ws_fd, buffer, bytes_read, true) < 0) {
ESP_LOGE(TAG, "Audio stream send failed");
break;
}
total_sent_bytes += bytes_read;
}
}
free(buffer);
if (ctx->cancel_session || total_sent_bytes < 3200) {
ESP_LOGI(TAG, "Cancelling audio session");
const char* cancel_json = "{\"event\":\"cancel\"}";
ws_send(ctx->ws_fd, (const uint8_t*)cancel_json, strlen(cancel_json), false);
ctx->state = STATE_IDLE;
} else {
const char* stop_json = "{\"event\":\"stop\"}";
ws_send(ctx->ws_fd, (const uint8_t*)stop_json, strlen(stop_json), false);
ctx->state = STATE_THINKING;
update_ui(ctx);
// Wait for response to finish (increase to 90s to match socket timeout)
int wait_timeout_ms = 90000;
while (!ctx->ws_done && ctx->state != STATE_ERROR && wait_timeout_ms > 0) {
vTaskDelay(pdMS_TO_TICKS(100));
wait_timeout_ms -= 100;
if (ctx->ui_update_pending) {
ctx->ui_update_pending = false;
update_ui(ctx);
}
}
if (!ctx->ws_done && ctx->state != STATE_ERROR) {
ctx->state = STATE_ERROR;
snprintf(ctx->error_message, sizeof(ctx->error_message), "Response timeout");
}
}
// Clean up socket
int fd_to_close = ctx->ws_fd;
ctx->ws_fd = -1;
ctx->ws_connected = false;
ws_close(fd_to_close);
// Reset I2S controller to release DMA and stop white noise
if (ctx->i2s_dev != NULL) {
device_lock(ctx->i2s_dev);
i2s_controller_reset(ctx->i2s_dev);
device_unlock(ctx->i2s_dev);
}
// Wait for receive task to exit
int rx_timeout = 100;
while (ctx->rx_task != NULL && rx_timeout > 0) {
vTaskDelay(pdMS_TO_TICKS(10));
rx_timeout--;
}
if (ctx->state != STATE_ERROR) {
ctx->state = STATE_IDLE;
}
update_ui(ctx);
}
vTaskDelay(pdMS_TO_TICKS(100));
if (ctx->ui_update_pending) {
ctx->ui_update_pending = false;
update_ui(ctx);
}
}
ctx->worker_task = NULL;
vTaskDelete(NULL);
}
/* ─── App Lifecycle ─── */
static void* create_data(void) {
ReynaBotCtx* ctx = calloc(1, sizeof(ReynaBotCtx));
if (ctx != NULL) {
ctx->state = STATE_IDLE;
ctx->ws_fd = -1;
}
return ctx;
}
static void destroy_data(void* data) {
free(data);
}
static void on_create(AppHandle app, void* data) {
ReynaBotCtx* ctx = (ReynaBotCtx*)data;
ctx->app = app;
}
static void on_destroy(AppHandle app, void* data) {
// No-op
}
static void on_show(AppHandle app, void* data, lv_obj_t* parent) {
ReynaBotCtx* ctx = (ReynaBotCtx*)data;
ctx->visible = true;
load_config(ctx);
// Find I2S controller
ctx->i2s_dev = device_find_by_name("i2s0");
if (ctx->i2s_dev == NULL) {
ESP_LOGE(TAG, "I2S controller 'i2s0' not found!");
}
// Style the parent screen
lv_obj_set_flex_flow(parent, LV_FLEX_FLOW_COLUMN);
lv_obj_set_flex_align(parent, LV_FLEX_ALIGN_START, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER);
lv_obj_set_style_pad_all(parent, 10, LV_PART_MAIN);
lv_obj_set_style_pad_row(parent, 12, LV_PART_MAIN);
lv_obj_set_style_bg_color(parent, lv_color_hex(0x0C0B12), LV_PART_MAIN); // Rich dark background
// Add App Toolbar
lv_obj_t* toolbar = tt_lvgl_toolbar_create_for_app(parent, app);
lv_obj_align(toolbar, LV_ALIGN_TOP_MID, 0, 0);
// Status Label inside toolbar (replaces the static "ReynaBot Voice" label)
ctx->lbl_status = lv_label_create(toolbar);
lv_label_set_text(ctx->lbl_status, "Ready");
lv_obj_add_flag(ctx->lbl_status, LV_OBJ_FLAG_FLOATING);
lv_obj_align(ctx->lbl_status, LV_ALIGN_RIGHT_MID, -10, 0);
lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0x9E9E9E), LV_PART_MAIN);
// Conversation bubble container
lv_obj_t* conv_card = lv_obj_create(parent);
lv_obj_set_width(conv_card, lv_pct(95));
lv_obj_set_flex_grow(conv_card, 1);
lv_obj_set_flex_flow(conv_card, LV_FLEX_FLOW_COLUMN);
lv_obj_set_flex_align(conv_card, LV_FLEX_ALIGN_START, LV_FLEX_ALIGN_START, LV_FLEX_ALIGN_START);
lv_obj_set_style_pad_all(conv_card, 12, LV_PART_MAIN);
lv_obj_set_style_pad_row(conv_card, 10, LV_PART_MAIN);
lv_obj_set_style_bg_color(conv_card, lv_color_hex(0x13121A), LV_PART_MAIN);
lv_obj_set_style_border_color(conv_card, lv_color_hex(0x23212C), LV_PART_MAIN);
lv_obj_set_style_border_width(conv_card, 1, LV_PART_MAIN);
lv_obj_set_style_radius(conv_card, 12, LV_PART_MAIN);
// User bubble
lv_obj_t* user_label_title = lv_label_create(conv_card);
lv_label_set_text(user_label_title, "👤 You:");
lv_obj_set_style_text_color(user_label_title, lv_color_hex(0x00E676), LV_PART_MAIN);
ctx->lbl_transcript = lv_label_create(conv_card);
lv_label_set_text(ctx->lbl_transcript, "(Your question will appear here)");
lv_label_set_long_mode(ctx->lbl_transcript, LV_LABEL_LONG_WRAP);
lv_obj_set_width(ctx->lbl_transcript, lv_pct(100));
lv_obj_set_style_text_color(ctx->lbl_transcript, lv_color_hex(0xE0E0E0), LV_PART_MAIN);
// Separator line
lv_obj_t* sep = lv_line_create(conv_card);
static lv_point_precise_t line_points[] = { {0, 0}, {300, 0} };
lv_line_set_points(sep, line_points, 2);
lv_obj_set_style_line_color(sep, lv_color_hex(0x2D2B36), LV_PART_MAIN);
lv_obj_set_style_line_width(sep, 1, LV_PART_MAIN);
lv_obj_set_width(sep, lv_pct(100));
// Assistant bubble
lv_obj_t* bot_label_title = lv_label_create(conv_card);
lv_label_set_text(bot_label_title, "🤖 ReynaBot:");
lv_obj_set_style_text_color(bot_label_title, lv_color_hex(0xAA00FF), LV_PART_MAIN);
ctx->lbl_response = lv_label_create(conv_card);
lv_label_set_text(ctx->lbl_response, "(Answer will appear here)");
lv_label_set_long_mode(ctx->lbl_response, LV_LABEL_LONG_WRAP);
lv_obj_set_width(ctx->lbl_response, lv_pct(100));
lv_obj_set_style_text_color(ctx->lbl_response, lv_color_hex(0xE0E0E0), LV_PART_MAIN);
// Push-to-Talk Button
ctx->btn_ptt = lv_btn_create(parent);
lv_obj_set_size(ctx->btn_ptt, 180, 50);
lv_obj_set_style_radius(ctx->btn_ptt, 25, LV_PART_MAIN);
lv_obj_set_style_bg_color(ctx->btn_ptt, lv_color_hex(0x6200EE), LV_PART_MAIN);
ctx->btn_ptt_label = lv_label_create(ctx->btn_ptt);
lv_label_set_text(ctx->btn_ptt_label, LV_SYMBOL_AUDIO " Hold to Talk");
lv_obj_center(ctx->btn_ptt_label);
lv_obj_add_event_cb(ctx->btn_ptt, ptt_event_cb, LV_EVENT_ALL, ctx);
// Launch main worker task
xTaskCreate(reynabot_task, "reynabot_worker", 6144, ctx, 5, &ctx->worker_task);
}
static void on_hide(AppHandle app, void* data) {
ReynaBotCtx* ctx = (ReynaBotCtx*)data;
if (ctx == NULL) return;
ESP_LOGI(TAG, "on_hide: cleaning up");
ctx->visible = false;
if (ctx->ws_fd >= 0) {
int fd_to_close = ctx->ws_fd;
ctx->ws_fd = -1;
ws_close(fd_to_close);
}
// Reset I2S controller to stop DMA and looping noise
if (ctx->i2s_dev != NULL) {
device_lock(ctx->i2s_dev);
i2s_controller_reset(ctx->i2s_dev);
device_unlock(ctx->i2s_dev);
}
// Wait briefly for tasks to exit
int timeout = 100;
while ((ctx->worker_task != NULL || ctx->rx_task != NULL) && timeout > 0) {
vTaskDelay(pdMS_TO_TICKS(10));
timeout--;
}
}
int main(int argc, char* argv[]) {
(void)argc;
(void)argv;
tt_app_register((AppRegistration) {
.createData = create_data,
.destroyData = destroy_data,
.onCreate = on_create,
.onDestroy = on_destroy,
.onShow = on_show,
.onHide = on_hide
});
return 0;
}
Apps/ReynaBot/main/Source/websocket.c
+240
View File
@@ -0,0 +1,240 @@
#include "websocket.h"
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <lwip/sockets.h>
#include <lwip/inet.h>
#include <esp_log.h>
static int recv_all(int fd, void* buf, size_t len) {
size_t total = 0;
char* p = (char*)buf;
while (total < len) {
int r = lwip_recv(fd, p + total, len - total, 0);
if (r <= 0) {
return -1;
}
total += r;
}
return 0;
}
static uint16_t my_htons(uint16_t val) {
return (uint16_t)(((val & 0xff) << 8) | ((val & 0xff00) >> 8));
}
int ws_connect(const char* host, int port, const char* path, const char* device_id, const char* auth_key) {
int fd = lwip_socket(AF_INET, SOCK_STREAM, 0);
if (fd < 0) return -1;
struct sockaddr_in addr;
memset(&addr, 0, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_port = my_htons(port);
addr.sin_addr.s_addr = ipaddr_addr(host);
if (lwip_connect(fd, (struct sockaddr*)&addr, sizeof(addr)) < 0) {
close(fd);
return -1;
}
// Set socket receive timeout (e.g. 90 seconds) to prevent blocking indefinitely
struct timeval tv;
tv.tv_sec = 90;
tv.tv_usec = 0;
lwip_setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv));
// Send HTTP upgrade handshake request
char req[1024];
snprintf(req, sizeof(req),
"GET %s HTTP/1.1\r\n"
"Host: %s:%d\r\n"
"Upgrade: websocket\r\n"
"Connection: Upgrade\r\n"
"Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n"
"Sec-WebSocket-Version: 13\r\n"
"Authorization: Bearer %s\r\n"
"X-Device-ID: %s\r\n"
"\r\n",
path, host, port, auth_key, device_id);
if (lwip_send(fd, req, strlen(req), 0) < 0) {
close(fd);
return -1;
}
// Read HTTP response headers until we hit "\r\n\r\n"
char header_buf[1024];
size_t header_len = 0;
while (header_len < sizeof(header_buf) - 1) {
char c;
int r = lwip_recv(fd, &c, 1, 0);
if (r <= 0) {
close(fd);
return -1;
}
header_buf[header_len++] = c;
header_buf[header_len] = '\0';
if (header_len >= 4 && strcmp(header_buf + header_len - 4, "\r\n\r\n") == 0) {
break;
}
}
// Verify HTTP 101 Switching Protocols response status
if (strstr(header_buf, "HTTP/1.1 101") == NULL && strstr(header_buf, "HTTP/1.0 101") == NULL) {
close(fd);
return -1;
}
return fd;
}
int ws_send(int fd, const uint8_t* data, size_t len, bool binary) {
uint8_t header[10];
size_t header_len = 0;
header[0] = binary ? 0x82 : 0x81;
if (len < 126) {
header[1] = 0x80 | (uint8_t)len;
header_len = 2;
} else {
header[1] = 0x80 | 126;
header[2] = (uint8_t)((len >> 8) & 0xFF);
header[3] = (uint8_t)(len & 0xFF);
header_len = 4;
}
// Use fixed client mask for performance: 0x12, 0x34, 0x56, 0x78
uint8_t mask[4] = { 0x12, 0x34, 0x56, 0x78 };
memcpy(header + header_len, mask, 4);
header_len += 4;
// Send WebSocket frame header
int sent = lwip_send(fd, header, header_len, 0);
if (sent < 0) return -1;
// Mask the payload
uint8_t* masked = malloc(len);
if (masked == NULL) return -1;
for (size_t i = 0; i < len; ++i) {
masked[i] = data[i] ^ mask[i % 4];
}
// Send masked payload
sent = lwip_send(fd, masked, len, 0);
free(masked);
return sent >= 0 ? 0 : -1;
}
int ws_recv(int fd, int* out_opcode, uint8_t* payload, size_t max_len) {
uint8_t header[2];
if (recv_all(fd, header, 2) < 0) {
return -1;
}
int opcode = header[0] & 0x0F;
if (out_opcode != NULL) {
*out_opcode = opcode;
}
int masked = (header[1] & 0x80) != 0;
size_t len = header[1] & 0x7F;
if (len == 126) {
uint8_t ext_len[2];
if (recv_all(fd, ext_len, 2) < 0) return -1;
len = ((size_t)ext_len[0] << 8) | ext_len[1];
} else if (len == 127) {
uint8_t ext_len[8];
if (recv_all(fd, ext_len, 8) < 0) return -1;
// Parse 64-bit length into size_t
len = ((size_t)ext_len[4] << 24) | ((size_t)ext_len[5] << 16) | ((size_t)ext_len[6] << 8) | ext_len[7];
}
if (masked) {
uint8_t mask[4];
if (recv_all(fd, mask, 4) < 0) return -1;
if (len > max_len) {
ESP_LOGE("websocket", "ws_recv overflow (masked): len=%u, max_len=%u", (unsigned)len, (unsigned)max_len);
// Buffer overflow, skip payload to align stream
size_t to_discard = len;
uint8_t discard_buf[256];
while (to_discard > 0) {
size_t chunk = to_discard < sizeof(discard_buf) ? to_discard : sizeof(discard_buf);
if (recv_all(fd, discard_buf, chunk) < 0) return -1;
to_discard -= chunk;
}
return -2;
}
if (recv_all(fd, payload, len) < 0) return -1;
for (size_t i = 0; i < len; ++i) {
payload[i] ^= mask[i % 4];
}
} else {
if (len > max_len) {
ESP_LOGE("websocket", "ws_recv overflow (unmasked): len=%u, max_len=%u", (unsigned)len, (unsigned)max_len);
size_t to_discard = len;
uint8_t discard_buf[256];
while (to_discard > 0) {
size_t chunk = to_discard < sizeof(discard_buf) ? to_discard : sizeof(discard_buf);
if (recv_all(fd, discard_buf, chunk) < 0) return -1;
to_discard -= chunk;
}
return -2;
}
if (recv_all(fd, payload, len) < 0) return -1;
}
return (int)len;
}
void ws_close(int fd) {
if (fd >= 0) {
close(fd);
}
}
int ws_send_pong(int fd, const uint8_t* payload, size_t len) {
uint8_t header[10];
size_t header_len = 0;
header[0] = 0x8A; // FIN | PONG (0x0A)
if (len < 126) {
header[1] = 0x80 | (uint8_t)len;
header_len = 2;
} else {
header[1] = 0x80 | 126;
header[2] = (uint8_t)((len >> 8) & 0xFF);
header[3] = (uint8_t)(len & 0xFF);
header_len = 4;
}
uint8_t mask[4] = { 0x12, 0x34, 0x56, 0x78 };
memcpy(header + header_len, mask, 4);
header_len += 4;
int sent = lwip_send(fd, header, header_len, 0);
if (sent < 0) return -1;
if (len > 0 && payload != NULL) {
uint8_t* masked = malloc(len);
if (masked == NULL) return -1;
for (size_t i = 0; i < len; ++i) {
masked[i] = payload[i] ^ mask[i % 4];
}
sent = lwip_send(fd, masked, len, 0);
free(masked);
}
return sent >= 0 ? 0 : -1;
}
Apps/ReynaBot/main/Source/websocket.h
+59
View File
@@ -0,0 +1,59 @@
#pragma once
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#ifdef __cplusplus
extern "C" {
#endif
/**
* Connect to a WebSocket server.
* @param host Server IP address (e.g. "192.168.68.126")
* @param port Port number (e.g. 8642)
* @param path WebSocket path (e.g. "/api/esp32/voice/ws")
* @param device_id Unique device identifier
* @param auth_key Hermes Bearer API key
* @return Socket file descriptor on success, or -1 on failure
*/
int ws_connect(const char* host, int port, const char* path, const char* device_id, const char* auth_key);
/**
* Send a WebSocket frame.
* @param fd Socket file descriptor
* @param data Data payload to send
* @param len Length of the data payload
* @param binary True for binary frame, false for text frame
* @return 0 on success, or -1 on failure
*/
int ws_send(int fd, const uint8_t* data, size_t len, bool binary);
/**
* Receive a WebSocket frame.
* @param fd Socket file descriptor
* @param out_opcode Pointer to store the received opcode (e.g. 0x01 text, 0x02 binary)
* @param payload Buffer to store the received payload
* @param max_len Maximum length of the payload buffer
* @return Received payload length on success, -1 on connection failure, or -2 on buffer overflow
*/
int ws_recv(int fd, int* out_opcode, uint8_t* payload, size_t max_len);
/**
* Close a WebSocket connection.
* @param fd Socket file descriptor
*/
void ws_close(int fd);
/**
* Send a WebSocket PONG frame.
* @param fd Socket file descriptor
* @param payload Payload to reflect
* @param len Length of payload
* @return 0 on success, or -1 on failure
*/
int ws_send_pong(int fd, const uint8_t* payload, size_t len);
#ifdef __cplusplus
}
#endif