adolforeyna/tactility_apps
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
59612020bbf5c2f5ebaa7646320b62046b6a112b
tactility_apps/Apps/ReynaBot/main/Source/main.c
T
Adolfo Reyna 59612020bb chore: checkpoint app work before firmware sync
2026-09-07 23:01:26 -04:00

971 lines
39 KiB
C
Raw Blame History

#include <tt_app.h>
#include <tt_lvgl.h>
#include <tt_lvgl_toolbar.h>
#include <tactility/device.h>
#include <tactility/drivers/audio_stream.h>
#include "websocket.h"
#include <cJSON.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/stat.h>
#include <errno.h>
/* Exported by Tactility firmware 0.8.0-dev; absent from the CDN SDK header. */
struct Device* device_find_by_name(const char* name);
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "esp_log.h"
#include "esp_random.h"
#define TAG "ReynaBot"
#define AUDIO_SAMPLE_RATE 16000U
#define AUDIO_BITS_PER_SAMPLE 16U
#define MAX_PCM_CHUNK_BYTES 16384U
#define MAX_RESPONSE_AUDIO_BYTES 65536U
typedef enum {
STATE_IDLE,
STATE_CONNECTING,
STATE_LISTENING,
STATE_THINKING,
STATE_SPEAKING,
STATE_ERROR
} ReynaBotState;
typedef struct {
AppHandle app;
bool visible;
ReynaBotState state;
char server_url[128];
char device_id[64];
char api_key[128];
char last_transcript[256];
char last_response[512];
char error_message[128];
// UI components
lv_obj_t* lbl_status;
lv_obj_t* lbl_transcript;
lv_obj_t* lbl_response;
lv_obj_t* btn_ptt;
lv_obj_t* btn_ptt_label;
lv_obj_t* btn_grace;
lv_obj_t* btn_elias;
bool speaker_selected;
lv_obj_t* speaker_select_cont;
lv_obj_t* app_main_cont;
// PTT control
bool is_pressed;
bool start_session;
bool stop_session;
bool cancel_session;
// FreeRTOS Tasks
TaskHandle_t worker_task;
TaskHandle_t rx_task;
// Hardware/audio service
struct Device* audio_stream_dev;
AudioStreamHandle input_handle;
AudioStreamHandle output_handle;
uint32_t output_sample_rate;
uint8_t output_channels;
uint8_t output_bits_per_sample;
bool output_stream_pending;
size_t expected_audio_bytes;
uint32_t expected_audio_rate;
uint8_t expected_audio_channels;
uint8_t expected_audio_bits;
bool stop_sent;
int ws_fd;
bool ws_connected;
bool ws_done;
bool ui_update_pending;
} ReynaBotCtx;
/* ─── Forward Declarations ─── */
static void reynabot_task(void* arg);
static void reynabot_rx_task(void* arg);
static void update_ui(ReynaBotCtx* ctx);
static void load_config(ReynaBotCtx* ctx);
static bool find_audio_stream_device(ReynaBotCtx* ctx) {
struct Device* dev = device_find_by_name("audio-stream");
if (dev != NULL) {
ctx->audio_stream_dev = dev;
return true;
}
// The 0.8.0-dev firmware registers this shared service by name. The
// CDN SDK headers do not expose the deprecated type-search helper, so do
// not reference it from a dynamically linked app.
return false;
}
static void close_input_stream(ReynaBotCtx* ctx) {
if (ctx->input_handle != NULL) {
audio_stream_close(ctx->input_handle);
ctx->input_handle = NULL;
}
}
static void close_output_stream(ReynaBotCtx* ctx) {
if (ctx->output_handle != NULL) {
audio_stream_close(ctx->output_handle);
ctx->output_handle = NULL;
}
ctx->output_stream_pending = false;
}
static bool open_input_stream(ReynaBotCtx* ctx) {
if (ctx->audio_stream_dev == NULL) return false;
close_input_stream(ctx);
struct AudioStreamConfig config = {
.sample_rate = 16000,
.bits_per_sample = 16,
.channels = 1,
};
error_t err = audio_stream_open_input(ctx->audio_stream_dev, &config, &ctx->input_handle);
if (err != ERROR_NONE) {
ctx->input_handle = NULL;
ESP_LOGE(TAG, "audio_stream_open_input failed: %d", err);
return false;
}
audio_stream_set_mute(ctx->audio_stream_dev, AUDIO_CODEC_DIR_INPUT, false);
audio_stream_set_volume(ctx->audio_stream_dev, AUDIO_CODEC_DIR_INPUT, 100.0f);
ESP_LOGI(TAG, "Microphone opened via audio-stream at 16000 Hz mono");
return true;
}
static bool open_output_stream(ReynaBotCtx* ctx, uint32_t sample_rate, uint8_t channels, uint8_t bits_per_sample) {
if (ctx->audio_stream_dev == NULL || sample_rate == 0 || channels == 0 || bits_per_sample != 16) return false;
close_output_stream(ctx);
struct AudioStreamConfig config = {
.sample_rate = sample_rate,
.bits_per_sample = bits_per_sample,
.channels = channels,
};
error_t err = audio_stream_open_output(ctx->audio_stream_dev, &config, &ctx->output_handle);
if (err != ERROR_NONE) {
ctx->output_handle = NULL;
ESP_LOGE(TAG, "audio_stream_open_output failed: %d rate=%u ch=%u", err, (unsigned)sample_rate, channels);
return false;
}
audio_stream_set_mute(ctx->audio_stream_dev, AUDIO_CODEC_DIR_OUTPUT, false);
audio_stream_set_volume(ctx->audio_stream_dev, AUDIO_CODEC_DIR_OUTPUT, 100.0f);
ctx->output_sample_rate = sample_rate;
ctx->output_channels = channels;
ctx->output_bits_per_sample = bits_per_sample;
ctx->output_stream_pending = true;
ESP_LOGI(TAG, "Speaker opened via audio-stream at %u Hz mono", (unsigned)sample_rate);
return true;
}
/* ─── Helper for URL parsing ─── */
static bool parse_ws_url(const char* url, char* host, int* port, char* path) {
if (strncmp(url, "ws://", 5) != 0) return false;
const char* p = url + 5;
const char* colon = strchr(p, ':');
const char* slash = strchr(p, '/');
if (slash == NULL) {
strcpy(path, "/");
} else {
strcpy(path, slash);
}
if (colon != NULL && (slash == NULL || colon < slash)) {
int host_len = colon - p;
memcpy(host, p, host_len);
host[host_len] = '\0';
*port = atoi(colon + 1);
} else {
int host_len = slash ? (slash - p) : strlen(p);
memcpy(host, p, host_len);
host[host_len] = '\0';
*port = 80;
}
return true;
}
/* ─── Config Loading/Saving ─── */
static void load_config(ReynaBotCtx* ctx) {
// Default fallback config
snprintf(ctx->server_url, sizeof(ctx->server_url), "ws://192.168.68.112:8643/api/esp32/voice/ws");
snprintf(ctx->device_id, sizeof(ctx->device_id), "reynabot_screen");
ctx->api_key[0] = '\0';
char path[256];
size_t path_size = sizeof(path);
tt_app_get_user_data_child_path(ctx->app, "config.json", path, &path_size);
FILE* file = fopen(path, "r");
if (file == NULL) {
// Create user data directory if it doesn't exist
char dir_path[256];
size_t dir_size = sizeof(dir_path);
tt_app_get_user_data_path(ctx->app, dir_path, &dir_size);
mkdir(dir_path, 0755);
// Write default config file
file = fopen(path, "w");
if (file != NULL) {
fprintf(file, "{\n \"server_url\": \"ws://192.168.68.112:8643/api/esp32/voice/ws\",\n \"device_id\": \"reynabot_screen\",\n \"api_key\": \"\"\n}\n");
fclose(file);
}
ESP_LOGI(TAG, "Created default config.json at %s", path);
return;
}
fseek(file, 0, SEEK_END);
long size = ftell(file);
fseek(file, 0, SEEK_SET);
if (size > 0 && size < 4096) {
char* buf = malloc(size + 1);
if (buf != NULL) {
size_t read_bytes = fread(buf, 1, size, file);
buf[read_bytes] = '\0';
cJSON* json = cJSON_Parse(buf);
if (json != NULL) {
cJSON* url_item = cJSON_GetObjectItem(json, "server_url");
if (url_item != NULL && url_item->valuestring != NULL) {
strncpy(ctx->server_url, url_item->valuestring, sizeof(ctx->server_url) - 1);
}
cJSON* dev_item = cJSON_GetObjectItem(json, "device_id");
if (dev_item != NULL && dev_item->valuestring != NULL) {
strncpy(ctx->device_id, dev_item->valuestring, sizeof(ctx->device_id) - 1);
}
cJSON* key_item = cJSON_GetObjectItem(json, "api_key");
if (key_item != NULL && key_item->valuestring != NULL) {
strncpy(ctx->api_key, key_item->valuestring, sizeof(ctx->api_key) - 1);
}
cJSON_Delete(json);
}
free(buf);
}
}
fclose(file);
ESP_LOGI(TAG, "Loaded config: server=%s device=%s", ctx->server_url, ctx->device_id);
}
/* ─── JSON Message Processing ─── */
static void parse_json_message(ReynaBotCtx* ctx, const char* json_str) {
cJSON* json = cJSON_Parse(json_str);
if (json == NULL) return;
cJSON* evt_item = cJSON_GetObjectItem(json, "event");
if (evt_item != NULL && evt_item->valuestring != NULL) {
const char* evt = evt_item->valuestring;
ESP_LOGI(TAG, "WS Event: %s", evt);
if (strcmp(evt, "ready") == 0) {
ESP_LOGI(TAG, "Server ready");
} else if (strcmp(evt, "state") == 0) {
cJSON* state_item = cJSON_GetObjectItem(json, "state");
if (state_item != NULL && cJSON_IsString(state_item)) {
if (strcmp(state_item->valuestring, "listening") == 0) {
if (ctx->stop_sent) {
ctx->ws_done = true;
} else {
ctx->state = STATE_LISTENING;
}
ctx->ui_update_pending = true;
} else if (strcmp(state_item->valuestring, "processing") == 0) {
ctx->state = STATE_THINKING;
ctx->ui_update_pending = true;
}
}
} else if (strcmp(evt, "listening") == 0) {
ctx->state = STATE_LISTENING;
ctx->ui_update_pending = true;
} else if (strcmp(evt, "transcript") == 0) {
cJSON* txt_item = cJSON_GetObjectItem(json, "text");
if (txt_item != NULL && txt_item->valuestring != NULL) {
strncpy(ctx->last_transcript, txt_item->valuestring, sizeof(ctx->last_transcript) - 1);
ctx->ui_update_pending = true;
}
} else if (strcmp(evt, "thinking") == 0) {
ctx->state = STATE_THINKING;
ctx->ui_update_pending = true;
} else if (strcmp(evt, "response_text") == 0) {
cJSON* txt_item = cJSON_GetObjectItem(json, "text");
if (txt_item != NULL && txt_item->valuestring != NULL) {
strncpy(ctx->last_response, txt_item->valuestring, sizeof(ctx->last_response) - 1);
ctx->ui_update_pending = true;
}
} else if (strcmp(evt, "audio") == 0) {
cJSON* format_item = cJSON_GetObjectItem(json, "format");
cJSON* rate_item = cJSON_GetObjectItem(json, "sample_rate");
cJSON* channels_item = cJSON_GetObjectItem(json, "channels");
cJSON* width_item = cJSON_GetObjectItem(json, "sample_width");
cJSON* length_item = cJSON_GetObjectItem(json, "byte_length");
bool valid = format_item != NULL && cJSON_IsString(format_item) &&
strcmp(format_item->valuestring, "pcm_s16le") == 0 &&
rate_item != NULL && cJSON_IsNumber(rate_item) && rate_item->valueint > 0 && rate_item->valueint <= 48000 &&
channels_item != NULL && cJSON_IsNumber(channels_item) && channels_item->valueint == 1 &&
width_item != NULL && cJSON_IsNumber(width_item) && width_item->valueint == 2 &&
length_item != NULL && cJSON_IsNumber(length_item) && length_item->valueint > 0 &&
length_item->valueint <= MAX_RESPONSE_AUDIO_BYTES && (length_item->valueint % 2) == 0;
if (valid) {
ctx->expected_audio_rate = (uint32_t)rate_item->valueint;
ctx->expected_audio_channels = (uint8_t)channels_item->valueint;
ctx->expected_audio_bits = (uint8_t)width_item->valueint;
ctx->expected_audio_bytes = (size_t)length_item->valueint;
if (!open_output_stream(ctx, ctx->expected_audio_rate, ctx->expected_audio_channels, ctx->expected_audio_bits)) {
snprintf(ctx->error_message, sizeof(ctx->error_message), "Audio output unavailable");
ctx->state = STATE_ERROR;
ctx->expected_audio_bytes = 0;
} else {
ctx->state = STATE_SPEAKING;
}
ctx->ui_update_pending = true;
} else {
snprintf(ctx->error_message, sizeof(ctx->error_message), "Invalid audio metadata");
ctx->state = STATE_ERROR;
ctx->expected_audio_bytes = 0;
ctx->ui_update_pending = true;
}
} else if (strcmp(evt, "audio_start") == 0) {
/* Legacy event: metadata must still arrive as `audio` before binary PCM. */
ctx->state = STATE_SPEAKING;
ctx->ui_update_pending = true;
} else if (strcmp(evt, "audio_end") == 0) {
ESP_LOGI(TAG, "Audio response ended");
} else if (strcmp(evt, "done") == 0) {
ctx->ws_done = true;
} else if (strcmp(evt, "error") == 0) {
cJSON* msg_item = cJSON_GetObjectItem(json, "message");
if (msg_item != NULL && msg_item->valuestring != NULL) {
strncpy(ctx->error_message, msg_item->valuestring, sizeof(ctx->error_message) - 1);
} else {
snprintf(ctx->error_message, sizeof(ctx->error_message), "Unknown server error");
}
ctx->state = STATE_ERROR;
ctx->ui_update_pending = true;
}
}
cJSON_Delete(json);
}
/* ─── WebSocket RX (Receive) Task ─── */
static void reynabot_rx_task(void* arg) {
ReynaBotCtx* ctx = (ReynaBotCtx*)arg;
uint8_t* rx_buf = malloc(MAX_RESPONSE_AUDIO_BYTES + 1U);
if (rx_buf == NULL) {
ESP_LOGE(TAG, "Failed to allocate RX buffer");
ctx->ws_done = true;
ctx->rx_task = NULL;
vTaskDelete(NULL);
return;
}
int opcode;
ESP_LOGI(TAG, "WS Receive task started");
while (ctx->ws_fd >= 0) {
int r = ws_recv(ctx->ws_fd, &opcode, rx_buf, MAX_RESPONSE_AUDIO_BYTES);
if (r < 0) {
if (r == -1) {
ESP_LOGI(TAG, "WS connection closed or read error, errno=%d", errno);
} else {
ESP_LOGE(TAG, "WS rx buffer overflow");
}
ctx->ws_done = true;
break;
}
if (opcode == 0x01) { // Text frame (JSON)
rx_buf[r] = '\0';
parse_json_message(ctx, (char*)rx_buf);
} else if (opcode == 0x02) { // Audio response PCM
if (ctx->output_handle == NULL || ctx->expected_audio_bytes == 0 || (size_t)r != ctx->expected_audio_bytes) {
ESP_LOGE(TAG, "Audio frame does not match metadata: got=%d expected=%u", r, (unsigned)ctx->expected_audio_bytes);
ctx->state = STATE_ERROR;
snprintf(ctx->error_message, sizeof(ctx->error_message), "Invalid audio frame");
ctx->ui_update_pending = true;
} else {
size_t written_total = 0;
while (written_total < (size_t)r) {
size_t written = 0;
error_t err = audio_stream_write(ctx->output_handle, rx_buf + written_total,
(size_t)r - written_total, &written, pdMS_TO_TICKS(3000));
if (err != ERROR_NONE || written == 0) {
ESP_LOGE(TAG, "audio_stream_write failed: %d written=%u", err, (unsigned)written);
ctx->state = STATE_ERROR;
snprintf(ctx->error_message, sizeof(ctx->error_message), "Audio playback failed");
ctx->ui_update_pending = true;
break;
}
written_total += written;
}
close_output_stream(ctx);
ctx->expected_audio_bytes = 0;
ctx->state = STATE_THINKING;
ctx->ui_update_pending = true;
}
} else if (opcode == 0x09) { // PING frame
ESP_LOGI(TAG, "WS PING received, sending PONG");
ws_send_pong(ctx->ws_fd, rx_buf, r);
}
}
free(rx_buf);
ctx->rx_task = NULL;
vTaskDelete(NULL);
}
/* ─── UI Speaker Selection Event Callback ─── */
static void speaker_event_cb(lv_event_t* e) {
ReynaBotCtx* ctx = (ReynaBotCtx*)lv_event_get_user_data(e);
lv_obj_t* target = lv_event_get_target(e);
lv_event_code_t code = lv_event_get_code(e);
if (code == LV_EVENT_CLICKED && !ctx->speaker_selected) {
if (target == ctx->btn_grace) {
strcpy(ctx->device_id, "grace-esp32");
} else if (target == ctx->btn_elias) {
strcpy(ctx->device_id, "elias-esp32");
}
ctx->speaker_selected = true;
// Hide selection screen, show main screen
lv_obj_add_flag(ctx->speaker_select_cont, LV_OBJ_FLAG_HIDDEN);
lv_obj_remove_flag(ctx->app_main_cont, LV_OBJ_FLAG_HIDDEN);
// Trigger UI update to refresh status and PTT button
ctx->ui_update_pending = true;
}
}
/* ─── UI Press-to-Talk Event Callback ─── */
static void ptt_event_cb(lv_event_t* e) {
ReynaBotCtx* ctx = (ReynaBotCtx*)lv_event_get_user_data(e);
lv_event_code_t code = lv_event_get_code(e);
if (code == LV_EVENT_PRESSED) {
ctx->is_pressed = true;
if (ctx->state == STATE_IDLE || ctx->state == ERROR_NONE) {
ctx->start_session = true;
}
} else if (code == LV_EVENT_RELEASED || code == LV_EVENT_PRESS_LOST) {
ctx->is_pressed = false;
if (ctx->state == STATE_LISTENING) {
ctx->stop_session = true;
} else if (ctx->state == STATE_CONNECTING) {
ctx->cancel_session = true;
}
}
}
/* ─── Core UI Update Function ─── */
static void update_ui(ReynaBotCtx* ctx) {
if (!ctx->visible) return;
if (tt_lvgl_lock(pdMS_TO_TICKS(500))) {
if (!ctx->visible) {
tt_lvgl_unlock();
return;
}
if (!ctx->speaker_selected) {
lv_label_set_text(ctx->lbl_status, "Choose Speaker");
lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0x9E9E9E), LV_PART_MAIN);
tt_lvgl_unlock();
return;
}
switch (ctx->state) {
case STATE_IDLE:
lv_label_set_text(ctx->lbl_status, "Ready");
lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0x9E9E9E), LV_PART_MAIN);
lv_label_set_text(ctx->btn_ptt_label, LV_SYMBOL_AUDIO " Hold to Talk");
lv_obj_set_style_bg_color(ctx->btn_ptt, lv_color_hex(0x6200EE), LV_PART_MAIN);
lv_obj_remove_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN);
break;
case STATE_CONNECTING:
lv_label_set_text(ctx->lbl_status, "Connecting...");
lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0xFFC107), LV_PART_MAIN);
lv_label_set_text(ctx->btn_ptt_label, "Connecting...");
lv_obj_set_style_bg_color(ctx->btn_ptt, lv_color_hex(0x757575), LV_PART_MAIN);
lv_obj_remove_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN);
break;
case STATE_LISTENING:
lv_label_set_text(ctx->lbl_status, "Listening...");
lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0x00E676), LV_PART_MAIN);
lv_label_set_text(ctx->btn_ptt_label, "Release to Stop");
lv_obj_set_style_bg_color(ctx->btn_ptt, lv_color_hex(0xD50000), LV_PART_MAIN);
lv_obj_remove_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN);
break;
case STATE_THINKING:
lv_label_set_text(ctx->lbl_status, "Thinking...");
lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0x00B0FF), LV_PART_MAIN);
lv_obj_add_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN);
break;
case STATE_SPEAKING:
lv_label_set_text(ctx->lbl_status, "Speaking...");
lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0xAA00FF), LV_PART_MAIN);
lv_obj_add_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN);
break;
case STATE_ERROR:
lv_label_set_text(ctx->lbl_status, "Error");
lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0xFF1744), LV_PART_MAIN);
lv_label_set_text(ctx->btn_ptt_label, LV_SYMBOL_REFRESH " Try Again");
lv_obj_set_style_bg_color(ctx->btn_ptt, lv_color_hex(0xFF1744), LV_PART_MAIN);
lv_obj_remove_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN);
// Show the full error message in the bot's response bubble
if (ctx->error_message[0] != '\0') {
snprintf(ctx->last_response, sizeof(ctx->last_response), "Error: %s", ctx->error_message);
}
break;
}
lv_label_set_text(ctx->lbl_transcript, ctx->last_transcript[0] != '\0' ? ctx->last_transcript : "(Your question will appear here)");
lv_label_set_text(ctx->lbl_response, ctx->last_response[0] != '\0' ? ctx->last_response : "(Answer will appear here)");
tt_lvgl_unlock();
}
}
/* ─── Streaming & Main Worker Task ─── */
static void reynabot_task(void* arg) {
ReynaBotCtx* ctx = (ReynaBotCtx*)arg;
char host[64];
char path[128];
int port = 80;
ctx->ws_fd = -1;
while (ctx->visible) {
if (ctx->start_session) {
ctx->start_session = false;
ctx->stop_session = false;
ctx->cancel_session = false;
ctx->stop_sent = false;
ctx->expected_audio_bytes = 0;
close_input_stream(ctx);
close_output_stream(ctx);
ctx->last_transcript[0] = '\0';
ctx->last_response[0] = '\0';
ctx->error_message[0] = '\0';
ctx->ws_done = false;
update_ui(ctx);
if (!parse_ws_url(ctx->server_url, host, &port, path)) {
ctx->state = STATE_ERROR;
snprintf(ctx->error_message, sizeof(ctx->error_message), "Invalid Server URL");
update_ui(ctx);
continue;
}
ESP_LOGI(TAG, "Connecting: host=%s port=%d path=%s", host, port, path);
int fd = ws_connect(host, port, path, ctx->device_id, ctx->api_key);
if (fd < 0) {
ctx->state = STATE_ERROR;
snprintf(ctx->error_message, sizeof(ctx->error_message), "Connect failed");
update_ui(ctx);
continue;
}
ctx->ws_fd = fd;
ctx->ws_connected = true;
// Spawn background RX task to read and parse events.
xTaskCreate(reynabot_rx_task, "reynabot_rx", 4096, ctx, 6, &ctx->rx_task);
// The Kids LAN adapter requires protocol v1 and a fresh session id.
char session_id[80];
snprintf(session_id, sizeof(session_id), "reynabot-%08x%08x",
(unsigned)esp_random(), (unsigned)esp_random());
char start_json[384];
snprintf(start_json, sizeof(start_json),
"{\"v\":1,\"event\":\"start\",\"session_id\":\"%s\",\"device_id\":\"%s\",\"audio\":{\"format\":\"pcm_s16le\",\"sample_rate\":16000,\"channels\":1,\"sample_width\":2}}",
session_id, ctx->device_id);
if (ws_send(ctx->ws_fd, (const uint8_t*)start_json, strlen(start_json), false) < 0) {
ctx->state = STATE_ERROR;
snprintf(ctx->error_message, sizeof(ctx->error_message), "Handshake send failed");
ws_close(ctx->ws_fd);
ctx->ws_fd = -1;
ctx->ws_connected = false;
update_ui(ctx);
continue;
}
// Wait for server to switch us to LISTENING
int timeout_ms = 4000;
while (ctx->state != STATE_LISTENING && timeout_ms > 0 && !ctx->cancel_session && !ctx->ws_done) {
vTaskDelay(pdMS_TO_TICKS(50));
timeout_ms -= 50;
}
if (ctx->state != STATE_LISTENING) {
if (ctx->state != STATE_ERROR) {
ctx->state = STATE_ERROR;
snprintf(ctx->error_message, sizeof(ctx->error_message), "Handshake timeout");
}
ws_close(ctx->ws_fd);
ctx->ws_fd = -1;
ctx->ws_connected = false;
update_ui(ctx);
continue;
}
if (!open_input_stream(ctx)) {
ctx->state = STATE_ERROR;
snprintf(ctx->error_message, sizeof(ctx->error_message), "Microphone unavailable");
ws_close(ctx->ws_fd);
ctx->ws_fd = -1;
ctx->ws_connected = false;
update_ui(ctx);
continue;
}
uint8_t* buffer = malloc(MAX_PCM_CHUNK_BYTES);
if (buffer == NULL) {
ESP_LOGE(TAG, "Failed to allocate record buffer");
close_input_stream(ctx);
ctx->state = STATE_ERROR;
snprintf(ctx->error_message, sizeof(ctx->error_message), "Out of memory");
ws_close(ctx->ws_fd);
ctx->ws_fd = -1;
ctx->ws_connected = false;
update_ui(ctx);
continue;
}
size_t total_sent_bytes = 0;
// Stream 16 kHz mono PCM while PTT is held.
while (ctx->is_pressed && !ctx->stop_session && !ctx->cancel_session && !ctx->ws_done && total_sent_bytes < 320000) {
size_t bytes_read = 0;
error_t r = audio_stream_read(ctx->input_handle, buffer, MAX_PCM_CHUNK_BYTES,
&bytes_read, pdMS_TO_TICKS(200));
if (r == ERROR_NONE && bytes_read > 0 && bytes_read <= MAX_PCM_CHUNK_BYTES && (bytes_read % 2U) == 0) {
if (ws_send(ctx->ws_fd, buffer, bytes_read, true) < 0) {
ESP_LOGE(TAG, "Audio stream send failed");
break;
}
total_sent_bytes += bytes_read;
}
}
close_input_stream(ctx);
free(buffer);
if (ctx->cancel_session || total_sent_bytes < 3200) {
ESP_LOGI(TAG, "Cancelling audio session");
const char* cancel_json = "{\"event\":\"cancel\"}";
ws_send(ctx->ws_fd, (const uint8_t*)cancel_json, strlen(cancel_json), false);
ctx->stop_sent = true;
ctx->state = STATE_IDLE;
} else {
const char* stop_json = "{\"event\":\"stop\"}";
ws_send(ctx->ws_fd, (const uint8_t*)stop_json, strlen(stop_json), false);
ctx->stop_sent = true;
ctx->state = STATE_THINKING;
update_ui(ctx);
// Wait for response to finish (increase to 90s to match socket timeout)
int wait_timeout_ms = 90000;
while (!ctx->ws_done && ctx->state != STATE_ERROR && wait_timeout_ms > 0) {
vTaskDelay(pdMS_TO_TICKS(100));
wait_timeout_ms -= 100;
if (ctx->ui_update_pending) {
ctx->ui_update_pending = false;
update_ui(ctx);
}
}
if (!ctx->ws_done && ctx->state != STATE_ERROR) {
ctx->state = STATE_ERROR;
snprintf(ctx->error_message, sizeof(ctx->error_message), "Response timeout");
}
}
// Clean up socket
int fd_to_close = ctx->ws_fd;
ctx->ws_fd = -1;
ctx->ws_connected = false;
ws_close(fd_to_close);
close_input_stream(ctx);
close_output_stream(ctx);
// Wait for receive task to exit
int rx_timeout = 100;
while (ctx->rx_task != NULL && rx_timeout > 0) {
vTaskDelay(pdMS_TO_TICKS(10));
rx_timeout--;
}
if (ctx->state != STATE_ERROR) {
ctx->state = STATE_IDLE;
}
update_ui(ctx);
}
vTaskDelay(pdMS_TO_TICKS(100));
if (ctx->ui_update_pending) {
ctx->ui_update_pending = false;
update_ui(ctx);
}
}
ctx->worker_task = NULL;
vTaskDelete(NULL);
}
/* ─── App Lifecycle ─── */
static void* create_data(void) {
ReynaBotCtx* ctx = calloc(1, sizeof(ReynaBotCtx));
if (ctx != NULL) {
ctx->state = STATE_IDLE;
ctx->ws_fd = -1;
}
return ctx;
}
static void destroy_data(void* data) {
free(data);
}
static void on_create(AppHandle app, void* data) {
ReynaBotCtx* ctx = (ReynaBotCtx*)data;
ctx->app = app;
}
static void on_destroy(AppHandle app, void* data) {
// No-op
}
static void on_show(AppHandle app, void* data, lv_obj_t* parent) {
ReynaBotCtx* ctx = (ReynaBotCtx*)data;
ctx->visible = true;
load_config(ctx);
// Find the firmware Audio System service; it owns codec/native-rate conversion.
if (!find_audio_stream_device(ctx)) {
ESP_LOGE(TAG, "audio-stream device not found!");
}
// Style the parent screen
lv_obj_set_flex_flow(parent, LV_FLEX_FLOW_COLUMN);
lv_obj_set_flex_align(parent, LV_FLEX_ALIGN_START, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER);
lv_obj_set_style_pad_all(parent, 10, LV_PART_MAIN);
lv_obj_set_style_pad_row(parent, 12, LV_PART_MAIN);
lv_obj_set_style_bg_color(parent, lv_color_hex(0x0C0B12), LV_PART_MAIN); // Rich dark background
// Add App Toolbar
lv_obj_t* toolbar = tt_lvgl_toolbar_create_for_app(parent, app);
lv_obj_align(toolbar, LV_ALIGN_TOP_MID, 0, 0);
// Status Label inside toolbar (replaces the static "ReynaBot Voice" label)
ctx->lbl_status = lv_label_create(toolbar);
lv_label_set_text(ctx->lbl_status, "Ready");
lv_obj_add_flag(ctx->lbl_status, LV_OBJ_FLAG_FLOATING);
lv_obj_align(ctx->lbl_status, LV_ALIGN_RIGHT_MID, -10, 0);
lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0x9E9E9E), LV_PART_MAIN);
// Initial state
ctx->speaker_selected = false;
// Default to Grace if device_id is not already configured to elias-esp32
if (strcmp(ctx->device_id, "elias-esp32") != 0) {
strcpy(ctx->device_id, "grace-esp32");
}
// 1. Speaker Selection Container (Startup Screen)
ctx->speaker_select_cont = lv_obj_create(parent);
lv_obj_set_width(ctx->speaker_select_cont, lv_pct(95));
lv_obj_set_flex_grow(ctx->speaker_select_cont, 1);
lv_obj_set_flex_flow(ctx->speaker_select_cont, LV_FLEX_FLOW_COLUMN);
lv_obj_set_flex_align(ctx->speaker_select_cont, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER);
lv_obj_set_style_pad_all(ctx->speaker_select_cont, 0, LV_PART_MAIN);
lv_obj_set_style_bg_opa(ctx->speaker_select_cont, 0, LV_PART_MAIN);
lv_obj_set_style_border_width(ctx->speaker_select_cont, 0, LV_PART_MAIN);
lv_obj_set_style_pad_row(ctx->speaker_select_cont, 16, LV_PART_MAIN);
// Title Prompt
lv_obj_t* prompt_lbl = lv_label_create(ctx->speaker_select_cont);
lv_label_set_text(prompt_lbl, "Who is speaking?");
lv_obj_set_style_text_color(prompt_lbl, lv_color_hex(0xFFFFFF), LV_PART_MAIN);
// Row of two selection buttons
lv_obj_t* speaker_row = lv_obj_create(ctx->speaker_select_cont);
lv_obj_set_size(speaker_row, lv_pct(100), 140);
lv_obj_set_flex_flow(speaker_row, LV_FLEX_FLOW_ROW);
lv_obj_set_flex_align(speaker_row, LV_FLEX_ALIGN_SPACE_BETWEEN, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER);
lv_obj_set_style_pad_all(speaker_row, 0, LV_PART_MAIN);
lv_obj_set_style_bg_opa(speaker_row, 0, LV_PART_MAIN);
lv_obj_set_style_border_width(speaker_row, 0, LV_PART_MAIN);
// Grace Button (Girl)
ctx->btn_grace = lv_btn_create(speaker_row);
lv_obj_set_size(ctx->btn_grace, lv_pct(46), 130);
lv_obj_set_flex_flow(ctx->btn_grace, LV_FLEX_FLOW_COLUMN);
lv_obj_set_flex_align(ctx->btn_grace, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER);
lv_obj_set_style_pad_all(ctx->btn_grace, 8, LV_PART_MAIN);
lv_obj_set_style_pad_row(ctx->btn_grace, 8, LV_PART_MAIN);
lv_obj_set_style_radius(ctx->btn_grace, 12, LV_PART_MAIN);
lv_obj_set_style_bg_color(ctx->btn_grace, lv_color_hex(0x1C1B22), LV_PART_MAIN);
lv_obj_set_style_border_width(ctx->btn_grace, 1, LV_PART_MAIN);
lv_obj_set_style_border_color(ctx->btn_grace, lv_color_hex(0x2D2B36), LV_PART_MAIN);
lv_obj_add_event_cb(ctx->btn_grace, speaker_event_cb, LV_EVENT_CLICKED, ctx);
lv_obj_t* icon_grace = lv_image_create(ctx->btn_grace);
lv_obj_set_size(icon_grace, 80, 80);
char path_grace[256] = "A:";
size_t sz_grace = sizeof(path_grace) - 2;
tt_app_get_assets_child_path(ctx->app, "girl.png", path_grace + 2, &sz_grace);
lv_image_set_src(icon_grace, path_grace);
lv_obj_t* lbl_grace = lv_label_create(ctx->btn_grace);
lv_label_set_text(lbl_grace, "Grace");
lv_obj_set_style_text_color(lbl_grace, lv_color_hex(0xFFFFFF), LV_PART_MAIN);
// Elias Button (Boy)
ctx->btn_elias = lv_btn_create(speaker_row);
lv_obj_set_size(ctx->btn_elias, lv_pct(46), 130);
lv_obj_set_flex_flow(ctx->btn_elias, LV_FLEX_FLOW_COLUMN);
lv_obj_set_flex_align(ctx->btn_elias, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER);
lv_obj_set_style_pad_all(ctx->btn_elias, 8, LV_PART_MAIN);
lv_obj_set_style_pad_row(ctx->btn_elias, 8, LV_PART_MAIN);
lv_obj_set_style_radius(ctx->btn_elias, 12, LV_PART_MAIN);
lv_obj_set_style_bg_color(ctx->btn_elias, lv_color_hex(0x1C1B22), LV_PART_MAIN);
lv_obj_set_style_border_width(ctx->btn_elias, 1, LV_PART_MAIN);
lv_obj_set_style_border_color(ctx->btn_elias, lv_color_hex(0x2D2B36), LV_PART_MAIN);
lv_obj_add_event_cb(ctx->btn_elias, speaker_event_cb, LV_EVENT_CLICKED, ctx);
lv_obj_t* icon_elias = lv_image_create(ctx->btn_elias);
lv_obj_set_size(icon_elias, 80, 80);
char path_elias[256] = "A:";
size_t sz_elias = sizeof(path_elias) - 2;
tt_app_get_assets_child_path(ctx->app, "boy.png", path_elias + 2, &sz_elias);
lv_image_set_src(icon_elias, path_elias);
lv_obj_t* lbl_elias = lv_label_create(ctx->btn_elias);
lv_label_set_text(lbl_elias, "Elias");
lv_obj_set_style_text_color(lbl_elias, lv_color_hex(0xFFFFFF), LV_PART_MAIN);
// 2. Main App Container (Usual App Screen)
ctx->app_main_cont = lv_obj_create(parent);
lv_obj_set_width(ctx->app_main_cont, lv_pct(95));
lv_obj_set_flex_grow(ctx->app_main_cont, 1);
lv_obj_set_flex_flow(ctx->app_main_cont, LV_FLEX_FLOW_COLUMN);
lv_obj_set_flex_align(ctx->app_main_cont, LV_FLEX_ALIGN_START, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER);
lv_obj_set_style_pad_all(ctx->app_main_cont, 0, LV_PART_MAIN);
lv_obj_set_style_pad_row(ctx->app_main_cont, 12, LV_PART_MAIN);
lv_obj_set_style_bg_opa(ctx->app_main_cont, 0, LV_PART_MAIN);
lv_obj_set_style_border_width(ctx->app_main_cont, 0, LV_PART_MAIN);
lv_obj_add_flag(ctx->app_main_cont, LV_OBJ_FLAG_HIDDEN); // Hidden by default
// Conversation bubble container
lv_obj_t* conv_card = lv_obj_create(ctx->app_main_cont);
lv_obj_set_width(conv_card, lv_pct(100));
lv_obj_set_flex_grow(conv_card, 1);
lv_obj_set_flex_flow(conv_card, LV_FLEX_FLOW_COLUMN);
lv_obj_set_flex_align(conv_card, LV_FLEX_ALIGN_START, LV_FLEX_ALIGN_START, LV_FLEX_ALIGN_START);
lv_obj_set_style_pad_all(conv_card, 12, LV_PART_MAIN);
lv_obj_set_style_pad_row(conv_card, 10, LV_PART_MAIN);
lv_obj_set_style_bg_color(conv_card, lv_color_hex(0x13121A), LV_PART_MAIN);
lv_obj_set_style_border_color(conv_card, lv_color_hex(0x23212C), LV_PART_MAIN);
lv_obj_set_style_border_width(conv_card, 1, LV_PART_MAIN);
lv_obj_set_style_radius(conv_card, 12, LV_PART_MAIN);
// User bubble
lv_obj_t* user_label_title = lv_label_create(conv_card);
lv_label_set_text(user_label_title, "👤 You:");
lv_obj_set_style_text_color(user_label_title, lv_color_hex(0x00E676), LV_PART_MAIN);
ctx->lbl_transcript = lv_label_create(conv_card);
lv_label_set_text(ctx->lbl_transcript, "(Your question will appear here)");
lv_label_set_long_mode(ctx->lbl_transcript, LV_LABEL_LONG_WRAP);
lv_obj_set_width(ctx->lbl_transcript, lv_pct(100));
lv_obj_set_style_text_color(ctx->lbl_transcript, lv_color_hex(0xE0E0E0), LV_PART_MAIN);
// Separator line
lv_obj_t* sep = lv_line_create(conv_card);
static lv_point_precise_t line_points[] = { {0, 0}, {300, 0} };
lv_line_set_points(sep, line_points, 2);
lv_obj_set_style_line_color(sep, lv_color_hex(0x2D2B36), LV_PART_MAIN);
lv_obj_set_style_line_width(sep, 1, LV_PART_MAIN);
lv_obj_set_width(sep, lv_pct(100));
// Assistant bubble
lv_obj_t* bot_label_title = lv_label_create(conv_card);
lv_label_set_text(bot_label_title, "🤖 ReynaBot:");
lv_obj_set_style_text_color(bot_label_title, lv_color_hex(0xAA00FF), LV_PART_MAIN);
ctx->lbl_response = lv_label_create(conv_card);
lv_label_set_text(ctx->lbl_response, "(Answer will appear here)");
lv_label_set_long_mode(ctx->lbl_response, LV_LABEL_LONG_WRAP);
lv_obj_set_width(ctx->lbl_response, lv_pct(100));
lv_obj_set_style_text_color(ctx->lbl_response, lv_color_hex(0xE0E0E0), LV_PART_MAIN);
// Push-to-Talk Button
ctx->btn_ptt = lv_btn_create(ctx->app_main_cont);
lv_obj_set_size(ctx->btn_ptt, 180, 50);
lv_obj_set_style_radius(ctx->btn_ptt, 25, LV_PART_MAIN);
lv_obj_set_style_bg_color(ctx->btn_ptt, lv_color_hex(0x6200EE), LV_PART_MAIN);
ctx->btn_ptt_label = lv_label_create(ctx->btn_ptt);
lv_label_set_text(ctx->btn_ptt_label, LV_SYMBOL_AUDIO " Hold to Talk");
lv_obj_center(ctx->btn_ptt_label);
lv_obj_add_event_cb(ctx->btn_ptt, ptt_event_cb, LV_EVENT_ALL, ctx);
// Launch main worker task
xTaskCreate(reynabot_task, "reynabot_worker", 6144, ctx, 5, &ctx->worker_task);
}
static void on_hide(AppHandle app, void* data) {
ReynaBotCtx* ctx = (ReynaBotCtx*)data;
if (ctx == NULL) return;
ESP_LOGI(TAG, "on_hide: cleaning up");
ctx->visible = false;
if (ctx->ws_fd >= 0) {
int fd_to_close = ctx->ws_fd;
ctx->ws_fd = -1;
ws_close(fd_to_close);
}
close_input_stream(ctx);
close_output_stream(ctx);
// Wait briefly for tasks to exit
int timeout = 100;
while ((ctx->worker_task != NULL || ctx->rx_task != NULL) && timeout > 0) {
vTaskDelay(pdMS_TO_TICKS(10));
timeout--;
}
}
int main(int argc, char* argv[]) {
(void)argc;
(void)argv;
tt_app_register((AppRegistration) {
.createData = create_data,
.destroyData = destroy_data,
.onCreate = on_create,
.onDestroy = on_destroy,
.onShow = on_show,
.onHide = on_hide
});
return 0;
}
Reference in New Issue View Git Blame Copy Permalink