diff --git a/Apps/Breakout/main/Source/Breakout.cpp b/Apps/Breakout/main/Source/Breakout.cpp index 4c9553a..37c458f 100644 --- a/Apps/Breakout/main/Source/Breakout.cpp +++ b/Apps/Breakout/main/Source/Breakout.cpp @@ -12,8 +12,8 @@ #include #include -#include -#include +#include +#include constexpr auto* TAG = "Breakout"; diff --git a/Apps/LiveCaptions/main/Source/main.c b/Apps/LiveCaptions/main/Source/main.c index 05ce6c5..d4ac02e 100644 --- a/Apps/LiveCaptions/main/Source/main.c +++ b/Apps/LiveCaptions/main/Source/main.c @@ -25,10 +25,11 @@ struct Device* device_find_by_name(const char* name); struct Device* device_find_first_by_type(const struct DeviceType* type); #define TAG "LiveCaptions" -#define DEFAULT_ENDPOINT "ws://192.168.68.102:8642/api/esp32/voice/ws" +#define DEFAULT_ENDPOINT "ws://192.168.68.102:8645/api/esp32/captions/ws" #define DEFAULT_DEVICE_ID "tactility-14c19d1a790" #define PCM_BUFFER_BYTES 1024U #define EVENT_BUFFER_BYTES 4096U +#define DISPLAY_WORD_LIMIT 50 typedef enum { CAPTION_IDLE, @@ -59,31 +60,15 @@ typedef struct { TaskHandle_t receiver; SemaphoreHandle_t socket_lock; SemaphoreHandle_t audio_lock; - lv_obj_t* state_label; lv_obj_t* caption_label; - lv_obj_t* start_button; - lv_obj_t* stop_button; + lv_obj_t* status_label; } CaptionContext; static void update_ui(CaptionContext* ctx) { if (!ctx->visible || !tt_lvgl_lock(pdMS_TO_TICKS(100))) return; - const char* state = "READY"; - if (ctx->state == CAPTION_CONNECTING) state = "CONNECTING"; - else if (ctx->state == CAPTION_LISTENING) state = "LISTENING"; - else if (ctx->state == CAPTION_PROCESSING) state = "FINALIZING"; - else if (ctx->state == CAPTION_FAILED) state = "FAILED"; - lv_label_set_text(ctx->state_label, state); - lv_label_set_text(ctx->caption_label, ctx->caption[0] ? ctx->caption : ctx->detail); - if (ctx->state == CAPTION_IDLE || ctx->state == CAPTION_FAILED) { - lv_obj_clear_state(ctx->start_button, LV_STATE_DISABLED); - } else { - lv_obj_add_state(ctx->start_button, LV_STATE_DISABLED); - } - if (ctx->state == CAPTION_LISTENING || ctx->state == CAPTION_PROCESSING) { - lv_obj_clear_state(ctx->stop_button, LV_STATE_DISABLED); - } else { - lv_obj_add_state(ctx->stop_button, LV_STATE_DISABLED); - } + lv_label_set_text(ctx->caption_label, ctx->caption); + const char* status = ctx->state == CAPTION_LISTENING ? "Connected" : (ctx->state == CAPTION_CONNECTING ? "Connecting" : (ctx->state == CAPTION_FAILED ? "Failed — Reconnecting" : "Connecting")); + lv_label_set_text(ctx->status_label, status); tt_lvgl_unlock(); } @@ -162,14 +147,29 @@ static void append_final_caption(const char* text) { fclose(log); } +static bool is_space_char(char value) { return value == ' ' || value == '\n' || value == '\r' || value == ' '; } + +static void copy_recent_words(char* output, size_t output_size, const char* text) { + const char* starts[DISPLAY_WORD_LIMIT]; + int count = 0; + bool in_word = false; + for (const char* cursor = text; *cursor; ++cursor) { + if (is_space_char(*cursor)) { in_word = false; continue; } + if (!in_word) { starts[count % DISPLAY_WORD_LIMIT] = cursor; ++count; in_word = true; } + } + const char* first = count > DISPLAY_WORD_LIMIT ? starts[count % DISPLAY_WORD_LIMIT] : text; + snprintf(output, output_size, "%s", first); +} + static void display_caption(CaptionContext* ctx, const char* text, bool final) { if (text == NULL || !*text) return; - snprintf(ctx->caption, sizeof(ctx->caption), "%s", text); + copy_recent_words(ctx->caption, sizeof(ctx->caption), text); if (final && strcmp(ctx->last_final, text) != 0) { snprintf(ctx->last_final, sizeof(ctx->last_final), "%s", text); append_final_caption(text); - set_state(ctx, CAPTION_IDLE, "Saved final caption to SD card"); - } else update_ui(ctx); + ctx->state = CAPTION_IDLE; + } + update_ui(ctx); } static void handle_event(CaptionContext* ctx, const char* json) { @@ -183,7 +183,14 @@ static void handle_event(CaptionContext* ctx, const char* json) { set_state(ctx, CAPTION_CONNECTING, "Starting caption stream"); } else if (strcmp(name, "listening") == 0) { set_state(ctx, CAPTION_LISTENING, "Listening — press Stop when finished"); - } else if (strcmp(name, "draft") == 0 || strcmp(name, "interim_transcript") == 0) { + } else if (strcmp(name, "state") == 0) { + cJSON* remote_state = cJSON_GetObjectItem(root, "state"); + if (cJSON_IsString(remote_state) && strcmp(remote_state->valuestring, "listening") == 0) { + set_state(ctx, CAPTION_LISTENING, "Listening"); + } else if (cJSON_IsString(remote_state) && strcmp(remote_state->valuestring, "processing") == 0) { + set_state(ctx, CAPTION_PROCESSING, "Captioning…"); + } + } else if (strcmp(name, "draft") == 0 || strcmp(name, "interim_transcript") == 0 || strcmp(name, "review") == 0) { if (cJSON_IsString(text)) display_caption(ctx, text->valuestring, false); } else if (strcmp(name, "transcript") == 0 || strcmp(name, "final") == 0) { cJSON* is_final = cJSON_GetObjectItem(root, "isFinal"); @@ -248,7 +255,7 @@ static void worker_task(void* argument) { ctx->fd = ws_connect(host, port, path, ctx->device_id, ctx->api_key); if (ctx->fd < 0) { set_state(ctx, CAPTION_FAILED, "Fail to connect"); ctx->worker = NULL; vTaskDelete(NULL); } char start[256]; - snprintf(start, sizeof(start), "{\"event\":\"start\",\"device_id\":\"%s\",\"format\":\"pcm_s16le\",\"sample_rate\":16000,\"channels\":1,\"sample_width\":2,\"session_id\":\"cap-%08lx\"}", ctx->device_id, (unsigned long)esp_random()); + snprintf(start, sizeof(start), "{\"v\":1,\"event\":\"start\",\"session_id\":\"cap-%08lx\",\"device_id\":\"%s\",\"audio\":{\"format\":\"pcm_s16le\",\"sample_rate\":16000,\"channels\":1,\"sample_width\":2}}", (unsigned long)esp_random(), ctx->device_id); if (send_locked(ctx, (const uint8_t*)start, strlen(start), false) != 0 || !open_input_stream(ctx)) { set_state(ctx, CAPTION_FAILED, "Fail to connect"); ws_close(ctx->fd); ctx->fd = -1; ctx->worker = NULL; vTaskDelete(NULL); } @@ -271,26 +278,17 @@ static void worker_task(void* argument) { for (unsigned i = 0; ctx->session_active && i < 150; ++i) vTaskDelay(pdMS_TO_TICKS(100)); } if (ctx->fd >= 0) { ws_send_close(ctx->fd); ws_close(ctx->fd); ctx->fd = -1; } - if (ctx->socket_failed && ctx->visible) set_state(ctx, CAPTION_FAILED, "Fail to connect"); - else if (ctx->state == CAPTION_PROCESSING) set_state(ctx, CAPTION_IDLE, "No final caption received"); + bool reconnect = ctx->socket_failed && ctx->visible; + if (reconnect) set_state(ctx, CAPTION_FAILED, "Reconnecting"); + else if (ctx->state == CAPTION_PROCESSING) set_state(ctx, CAPTION_IDLE, ""); ctx->worker = NULL; + if (reconnect) { + vTaskDelay(pdMS_TO_TICKS(3000)); + if (ctx->visible) { ctx->socket_failed = false; ctx->capture_audio = true; xTaskCreate(worker_task, "caption_tx", 8192, ctx, 5, &ctx->worker); } + } vTaskDelete(NULL); } -static void on_start(lv_event_t* event) { - CaptionContext* ctx = lv_event_get_user_data(event); - if (ctx->worker != NULL || ctx->state == CAPTION_LISTENING || ctx->state == CAPTION_PROCESSING) return; - ctx->caption[0] = '\0'; ctx->last_final[0] = '\0'; ctx->socket_failed = false; ctx->stop_requested = false; ctx->capture_audio = true; - xTaskCreate(worker_task, "caption_tx", 8192, ctx, 5, &ctx->worker); -} - -static void on_stop(lv_event_t* event) { - CaptionContext* ctx = lv_event_get_user_data(event); - if (ctx->state != CAPTION_LISTENING) return; - ctx->capture_audio = false; ctx->stop_requested = true; - set_state(ctx, CAPTION_PROCESSING, "Final captioning…"); -} - static void* create_data(void) { CaptionContext* ctx = calloc(1, sizeof(*ctx)); if (ctx) ctx->fd = -1; return ctx; } static void destroy_data(void* data) { free(data); } static void on_create(AppHandle app, void* data) { ((CaptionContext*)data)->app = app; } @@ -298,19 +296,20 @@ static void on_create(AppHandle app, void* data) { ((CaptionContext*)data)->app static void on_show(AppHandle app, void* data, lv_obj_t* parent) { CaptionContext* ctx = data; ctx->visible = true; load_config(ctx); find_audio_stream_device(ctx); ctx->socket_lock = xSemaphoreCreateMutex(); ctx->audio_lock = xSemaphoreCreateMutex(); - tt_lvgl_toolbar_create_for_app(parent, app); - ctx->state_label = lv_label_create(parent); lv_obj_align(ctx->state_label, LV_ALIGN_TOP_MID, 0, 38); - ctx->caption_label = lv_label_create(parent); lv_obj_set_width(ctx->caption_label, lv_pct(88)); - lv_label_set_long_mode(ctx->caption_label, LV_LABEL_LONG_WRAP); lv_obj_set_style_text_align(ctx->caption_label, LV_TEXT_ALIGN_CENTER, 0); - lv_obj_align(ctx->caption_label, LV_ALIGN_CENTER, 0, -8); - ctx->start_button = lv_btn_create(parent); lv_obj_set_size(ctx->start_button, 100, 42); lv_obj_align(ctx->start_button, LV_ALIGN_BOTTOM_LEFT, 22, -18); - lv_obj_t* start_text = lv_label_create(ctx->start_button); lv_label_set_text(start_text, "Start"); lv_obj_center(start_text); - lv_obj_add_event_cb(ctx->start_button, on_start, LV_EVENT_CLICKED, ctx); - ctx->stop_button = lv_btn_create(parent); lv_obj_set_size(ctx->stop_button, 100, 42); lv_obj_align(ctx->stop_button, LV_ALIGN_BOTTOM_RIGHT, -22, -18); - lv_obj_t* stop_text = lv_label_create(ctx->stop_button); lv_label_set_text(stop_text, "Stop"); lv_obj_center(stop_text); - lv_obj_add_event_cb(ctx->stop_button, on_stop, LV_EVENT_CLICKED, ctx); - if (ctx->stream_dev == NULL || ctx->socket_lock == NULL || ctx->audio_lock == NULL) set_state(ctx, CAPTION_FAILED, "Audio service unavailable"); - else set_state(ctx, CAPTION_IDLE, "Press Start to caption"); + lv_obj_t* toolbar = tt_lvgl_toolbar_create_for_app(parent, app); + lv_obj_align(toolbar, LV_ALIGN_TOP_MID, 0, 0); + ctx->caption_label = lv_label_create(parent); + lv_obj_set_width(ctx->caption_label, lv_pct(88)); + lv_label_set_long_mode(ctx->caption_label, LV_LABEL_LONG_WRAP); + lv_obj_set_style_text_align(ctx->caption_label, LV_TEXT_ALIGN_CENTER, 0); + lv_obj_align(ctx->caption_label, LV_ALIGN_CENTER, 0, 0); + ctx->status_label = lv_label_create(parent); + lv_obj_align(ctx->status_label, LV_ALIGN_BOTTOM_MID, 0, -10); + ctx->caption[0] = '\0'; + if (ctx->stream_dev != NULL && ctx->socket_lock != NULL && ctx->audio_lock != NULL) { + ctx->capture_audio = true; ctx->stop_requested = false; ctx->socket_failed = false; + xTaskCreate(worker_task, "caption_tx", 8192, ctx, 5, &ctx->worker); + } } static void on_hide(AppHandle app, void* data) { diff --git a/Apps/Mp3Player/main/Source/main.c b/Apps/Mp3Player/main/Source/main.c index 3998441..a916e94 100644 --- a/Apps/Mp3Player/main/Source/main.c +++ b/Apps/Mp3Player/main/Source/main.c @@ -1,14 +1,21 @@ #include #include #include +#include #include #include #include +/* Exported by firmware 0.8.0-dev although absent from the CDN SDK header. */ +struct Device* device_find_by_name(const char* name); + #include #include #include +#include +#include +#include #include "freertos/FreeRTOS.h" #include "freertos/task.h" @@ -21,24 +28,49 @@ #define TAG "Mp3Player" #define MP3_INPUT_BUFFER_SIZE 16384 +// ─── Scroll animation flag ─── +#define ENABLE_TITLE_SCROLL_ANIM_WHEN_IDLE 1 + +#define MAX_HISTORY 20 +#define SEEK_SECONDS 15 + typedef enum { STATE_IDLE, STATE_PLAYING, STATE_PAUSED } PlaybackState; +#define MAX_HISTORY 20 + +typedef struct { + char filepath[512]; + int pos_sec; + int total_sec; + time_t last_played; +} HistoryEntry; + typedef struct { struct Device* stream_dev; AudioStreamHandle stream_handle; char filepath[512]; PlaybackState state; + AppHandle app_handle; // UI elements lv_obj_t* lbl_title; lv_obj_t* lbl_status; lv_obj_t* bar_progress; + lv_obj_t* lbl_time; lv_obj_t* btn_play_pause; lv_obj_t* btn_stop; + lv_obj_t* btn_seek_back; + lv_obj_t* btn_seek_fwd; + lv_obj_t* list_history; + lv_obj_t* history_container; + lv_obj_t* center_box; + lv_obj_t* volume_box; + lv_obj_t* slider_volume; + lv_obj_t* bottom_box; // Playback state variables FILE* file; @@ -55,45 +87,419 @@ typedef struct { // Progress calculation size_t file_size; size_t bytes_read_total; - + int pos_sec; + int total_sec; + long total_decoded_samples; + int start_pos_sec; + bool finished_naturally; + int last_progress_pct; + + // Seek support + long id3_skip; + long file_data_size; + volatile int seek_request_sec; // -1 = none + + // History - in RAM (10KB), stored in PSRAM via g_ctx + HistoryEntry history[MAX_HISTORY]; + int history_count; + TaskHandle_t playback_task_handle; } AppCtx; static AppCtx g_ctx; +/* ─── Safe persistence helpers ─── */ +static void ensure_user_data_dir(AppHandle app) { + char dir_path[256]; + size_t len = sizeof(dir_path); + tt_app_get_user_data_path(app, dir_path, &len); + // mkdir -p recursively + char tmp[256]; + strncpy(tmp, dir_path, sizeof(tmp)-1); + tmp[sizeof(tmp)-1]='\0'; + for (char* p = tmp+1; *p; p++) { + if (*p == '/') { + *p = '\0'; + mkdir(tmp, 0755); + *p = '/'; + } + } + mkdir(tmp, 0755); + ESP_LOGI(TAG, "Ensured user data dir %s", dir_path); +} + +static void get_history_path(AppHandle app, char* buf, size_t buf_len) { + size_t len = buf_len; + tt_app_get_user_data_child_path(app, "play_history.txt", buf, &len); +} + +static void get_volume_path(AppHandle app, char* buf, size_t buf_len) { + size_t len = buf_len; + tt_app_get_user_data_child_path(app, "volume.txt", buf, &len); +} + +static void load_volume(AppCtx* ctx) { + char path[256]; + get_volume_path(ctx->app_handle, path, sizeof(path)); + FILE* f = fopen(path, "r"); + if (!f) { + ctx->volume = 80; + return; + } + int vol = 80; + if (fscanf(f, "%d", &vol) == 1) { + if (vol < 0) vol = 0; + if (vol > 100) vol = 100; + ctx->volume = vol; + } + fclose(f); + ESP_LOGI(TAG, "Loaded volume %d", ctx->volume); +} + +static void save_volume(AppCtx* ctx) { + char path[256]; + get_volume_path(ctx->app_handle, path, sizeof(path)); + // Ensure dir exists + ensure_user_data_dir(ctx->app_handle); + FILE* f = fopen(path, "w"); + if (!f) { + ESP_LOGW(TAG, "Failed to save volume to %s", path); + return; + } + fprintf(f, "%d\n", ctx->volume); + fclose(f); +} + +static void load_history(AppCtx* ctx) { + ctx->history_count = 0; + char path[256]; + get_history_path(ctx->app_handle, path, sizeof(path)); + FILE* f = fopen(path, "r"); + if (!f) { + ESP_LOGI(TAG, "No history file"); + return; + } + // Use heap buffer to avoid large stack allocation in LVGL task + char* line = (char*)malloc(1024); + if (!line) { + fclose(f); + return; + } + while (fgets(line, 1024, f) && ctx->history_count < MAX_HISTORY) { + char* p1 = strchr(line, '|'); + if (!p1) continue; + *p1 = '\0'; + char* p2 = strchr(p1+1, '|'); + if (!p2) continue; + *p2 = '\0'; + char* p3 = strchr(p2+1, '|'); + if (!p3) continue; + *p3 = '\0'; + long ts = atol(line); + int pos = atoi(p1+1); + int total = atoi(p2+1); + char filepath[512]; + strncpy(filepath, p3+1, sizeof(filepath)-1); + filepath[sizeof(filepath)-1] = '\0'; + size_t len = strlen(filepath); + while (len > 0 && (filepath[len-1]=='\n' || filepath[len-1]=='\r')) { + filepath[len-1]='\0'; len--; + } + if (filepath[0]=='\0') continue; + HistoryEntry* e = &ctx->history[ctx->history_count++]; + strncpy(e->filepath, filepath, sizeof(e->filepath)-1); + e->filepath[sizeof(e->filepath)-1]='\0'; + e->pos_sec = pos; + e->total_sec = total; + e->last_played = (time_t)ts; + } + free(line); + fclose(f); + ESP_LOGI(TAG, "Loaded %d history entries", ctx->history_count); + // Sort by last_played descending + for (int i=0;ihistory_count-1;i++) { + for (int j=0;jhistory_count-i-1;j++) { + if (ctx->history[j].last_played < ctx->history[j+1].last_played) { + HistoryEntry tmp = ctx->history[j]; + ctx->history[j] = ctx->history[j+1]; + ctx->history[j+1] = tmp; + } + } + } +} + +static void save_history(AppCtx* ctx) { + char path[256]; + get_history_path(ctx->app_handle, path, sizeof(path)); + ensure_user_data_dir(ctx->app_handle); + FILE* f = fopen(path, "w"); + if (!f) { + ESP_LOGW(TAG, "Failed to save history to %s", path); + return; + } + for (int i=0;ihistory_count;i++) { + HistoryEntry* e = &ctx->history[i]; + fprintf(f, "%ld|%d|%d|%s\n", (long)e->last_played, e->pos_sec, e->total_sec, e->filepath); + } + fclose(f); + ESP_LOGI(TAG, "Saved %d history entries", ctx->history_count); +} + +static void update_history_entry(AppCtx* ctx, const char* filepath, int pos_sec, int total_sec) { + if (!filepath || filepath[0]=='\0') return; + time_t now = time(NULL); + int idx = -1; + for (int i=0;ihistory_count;i++) { + if (strcmp(ctx->history[i].filepath, filepath)==0) { idx=i; break; } + } + if (idx>=0) { + ctx->history[idx].pos_sec = pos_sec; + ctx->history[idx].total_sec = total_sec; + ctx->history[idx].last_played = now; + } else { + if (ctx->history_count >= MAX_HISTORY) ctx->history_count = MAX_HISTORY-1; + for (int i=ctx->history_count;i>0;i--) ctx->history[i]=ctx->history[i-1]; + HistoryEntry* e=&ctx->history[0]; + strncpy(e->filepath, filepath, sizeof(e->filepath)-1); + e->filepath[sizeof(e->filepath)-1]='\0'; + e->pos_sec=pos_sec; e->total_sec=total_sec; e->last_played=now; + ctx->history_count++; + } + // Re-sort + for (int i=0;ihistory_count-1;i++) { + for (int j=0;jhistory_count-i-1;j++) { + if (ctx->history[j].last_played < ctx->history[j+1].last_played) { + HistoryEntry tmp=ctx->history[j]; + ctx->history[j]=ctx->history[j+1]; + ctx->history[j+1]=tmp; + } + } + } +} + +static bool get_last_played(AppCtx* ctx, char* out_path, size_t out_len, int* out_pos) { + if (ctx->history_count==0) return false; + for (int i=0;ihistory_count;i++) { + HistoryEntry* e=&ctx->history[i]; + struct stat st; + if (stat(e->filepath,&st)!=0) continue; + if (e->total_sec>0 && e->pos_sec >= e->total_sec-10) continue; + if (e->pos_sec <=5) continue; + strncpy(out_path, e->filepath, out_len-1); + out_path[out_len-1]='\0'; + if (out_pos) *out_pos=e->pos_sec; + return true; + } + // Fallback to most recent + HistoryEntry* e=&ctx->history[0]; + struct stat st; + if (stat(e->filepath,&st)!=0) return false; + strncpy(out_path, e->filepath, out_len-1); + out_path[out_len-1]='\0'; + if (out_pos) *out_pos=e->pos_sec; + return true; +} + +/* ─── Forward decls ─── */ +static void update_ui(AppCtx* ctx); +static void mp3_playback_task(void* arg); +static void refresh_history_list(AppCtx* ctx); +static void show_history_screen(AppCtx* ctx); +static void hide_history_screen(AppCtx* ctx); + +/* ─── History UI ─── */ +static void load_and_play_file(AppCtx* ctx, const char* filepath, int start_pos); + +static void on_history_selected(lv_event_t* e) { + int index = (int)(intptr_t)lv_event_get_user_data(e); + AppCtx* c = &g_ctx; + if (index < 0 || index >= c->history_count) return; + HistoryEntry* he = &c->history[index]; + int resume_pos = he->pos_sec; + if (resume_pos <= 5) resume_pos = 0; + if (he->total_sec > 0 && resume_pos >= he->total_sec - 10) resume_pos = 0; + ESP_LOGI(TAG, "History tap %d: %s resume %d (total %d)", index, he->filepath, resume_pos, he->total_sec); + // Hide history full screen and go back to player + hide_history_screen(c); + load_and_play_file(c, he->filepath, resume_pos); +} + +static void refresh_history_list(AppCtx* ctx) { + if (!ctx->list_history) return; + lv_obj_clean(ctx->list_history); + if (ctx->history_count == 0) { + lv_list_add_text(ctx->list_history, "No history yet"); + return; + } + for (int i=0;ihistory_count;i++) { + HistoryEntry* he = &ctx->history[i]; + const char* slash = strrchr(he->filepath, '/'); + const char* fname = slash ? slash+1 : he->filepath; + char label[640]; + if (he->total_sec >= 600) { + int pos_m = he->pos_sec / 60; + int pos_s = he->pos_sec % 60; + int tot_m = he->total_sec / 60; + int tot_s = he->total_sec % 60; + snprintf(label, sizeof(label), "%s\n%02d:%02d / %02d:%02d", fname, pos_m, pos_s, tot_m, tot_s); + } else { + snprintf(label, sizeof(label), "%s", fname); + } + lv_obj_t* btn = lv_list_add_btn(ctx->list_history, LV_SYMBOL_AUDIO, label); + // Force DOT to avoid scroll anim glitch (safe: last child is label) + uint32_t cnt = lv_obj_get_child_cnt(btn); + if (cnt > 0) { + lv_obj_t* lbl = lv_obj_get_child(btn, cnt-1); + if (lbl) lv_label_set_long_mode(lbl, LV_LABEL_LONG_DOT); + } + // Highlight currently playing file + if (ctx->filepath[0] != '\0' && strcmp(ctx->filepath, he->filepath) == 0) { + lv_obj_add_state(btn, LV_STATE_CHECKED); + } + lv_obj_add_event_cb(btn, on_history_selected, LV_EVENT_CLICKED, (void*)(intptr_t)i); + } +} + +static void show_history_screen(AppCtx* ctx) { + if (!ctx->history_container || !ctx->center_box || !ctx->bottom_box) return; + // Refresh to show latest and highlight + refresh_history_list(ctx); + // Expand history to full screen + lv_obj_set_size(ctx->history_container, lv_pct(100), lv_pct(100)); + lv_obj_align(ctx->history_container, LV_ALIGN_CENTER, 0, 0); + lv_obj_set_style_bg_color(ctx->history_container, lv_color_hex(0x11111B), 0); + lv_obj_set_style_bg_opa(ctx->history_container, LV_OPA_COVER, 0); + lv_obj_remove_flag(ctx->history_container, LV_OBJ_FLAG_HIDDEN); + lv_obj_move_foreground(ctx->history_container); + // Hide player UI + lv_obj_add_flag(ctx->center_box, LV_OBJ_FLAG_HIDDEN); + lv_obj_add_flag(ctx->volume_box, LV_OBJ_FLAG_HIDDEN); + lv_obj_add_flag(ctx->bottom_box, LV_OBJ_FLAG_HIDDEN); +} + +static void hide_history_screen(AppCtx* ctx) { + if (!ctx->history_container || !ctx->center_box || !ctx->bottom_box) return; + // Restore small history container hidden state (or keep as small below center) + // For full-screen mode, we hide history container and show player + lv_obj_add_flag(ctx->history_container, LV_OBJ_FLAG_HIDDEN); + lv_obj_remove_flag(ctx->center_box, LV_OBJ_FLAG_HIDDEN); + lv_obj_remove_flag(ctx->volume_box, LV_OBJ_FLAG_HIDDEN); + lv_obj_remove_flag(ctx->bottom_box, LV_OBJ_FLAG_HIDDEN); + // Restore small size for when it's shown inline (if needed) + lv_obj_set_size(ctx->history_container, lv_pct(92), lv_pct(28)); + lv_obj_align(ctx->history_container, LV_ALIGN_CENTER, 0, 35); +} + +static void load_and_play_file(AppCtx* ctx, const char* filepath, int start_pos) { + if (!filepath || filepath[0]=='\0') return; + // Stop current playback if any + if (ctx->state != STATE_IDLE) { + ctx->state = STATE_IDLE; + int tries=0; + while (ctx->playback_task_handle != NULL && tries<50) { + vTaskDelay(pdMS_TO_TICKS(50)); + tries++; + } + if (ctx->playback_task_handle) { + vTaskDelete(ctx->playback_task_handle); + ctx->playback_task_handle = NULL; + } + if (ctx->stream_handle) { + audio_stream_close(ctx->stream_handle); + ctx->stream_handle = NULL; + } + } + strncpy(ctx->filepath, filepath, sizeof(ctx->filepath)-1); + ctx->filepath[sizeof(ctx->filepath)-1]='\0'; + ctx->start_pos_sec = start_pos; + const char* slash = strrchr(filepath, '/'); + const char* fname = slash ? slash+1 : filepath; + if (ctx->lbl_title) { + lv_label_set_text(ctx->lbl_title, fname); + } + ctx->state = STATE_PLAYING; + update_ui(ctx); + xTaskCreate(mp3_playback_task, "mp3_play", 6144, ctx, 6, &ctx->playback_task_handle); +} + /* ─── Update UI ─── */ +static void set_title_scroll_enabled(AppCtx* ctx, bool enable) { + if (!ctx->lbl_title) return; + if (enable) { +#if ENABLE_TITLE_SCROLL_ANIM_WHEN_IDLE + lv_label_set_long_mode(ctx->lbl_title, LV_LABEL_LONG_SCROLL_CIRCULAR); +#else + lv_label_set_long_mode(ctx->lbl_title, LV_LABEL_LONG_DOT); +#endif + } else { + lv_label_set_long_mode(ctx->lbl_title, LV_LABEL_LONG_DOT); + } +} + static void update_ui(AppCtx* ctx) { if (!ctx->lbl_status) return; switch (ctx->state) { case STATE_PLAYING: + set_title_scroll_enabled(ctx, false); lv_label_set_text(ctx->lbl_status, "Playing..."); - lv_label_set_text(lv_obj_get_child(ctx->btn_play_pause, 0), LV_SYMBOL_PAUSE " Pause"); - lv_obj_clear_state(ctx->btn_play_pause, LV_STATE_DISABLED); - lv_obj_clear_state(ctx->btn_stop, LV_STATE_DISABLED); + if (ctx->btn_play_pause) { + lv_obj_t* lbl = lv_obj_get_child(ctx->btn_play_pause, 0); + if (lbl) lv_label_set_text(lbl, LV_SYMBOL_PAUSE); + } + if (ctx->btn_play_pause) lv_obj_clear_state(ctx->btn_play_pause, LV_STATE_DISABLED); + if (ctx->btn_stop) lv_obj_clear_state(ctx->btn_stop, LV_STATE_DISABLED); + if (ctx->btn_seek_back) lv_obj_clear_state(ctx->btn_seek_back, LV_STATE_DISABLED); + if (ctx->btn_seek_fwd) lv_obj_clear_state(ctx->btn_seek_fwd, LV_STATE_DISABLED); break; case STATE_PAUSED: + set_title_scroll_enabled(ctx, false); lv_label_set_text(ctx->lbl_status, "Paused"); - lv_label_set_text(lv_obj_get_child(ctx->btn_play_pause, 0), LV_SYMBOL_PLAY " Play"); - lv_obj_clear_state(ctx->btn_play_pause, LV_STATE_DISABLED); - lv_obj_clear_state(ctx->btn_stop, LV_STATE_DISABLED); + if (ctx->btn_play_pause) { + lv_obj_t* lbl = lv_obj_get_child(ctx->btn_play_pause, 0); + if (lbl) lv_label_set_text(lbl, LV_SYMBOL_PLAY); + } + if (ctx->btn_play_pause) lv_obj_clear_state(ctx->btn_play_pause, LV_STATE_DISABLED); + if (ctx->btn_stop) lv_obj_clear_state(ctx->btn_stop, LV_STATE_DISABLED); + if (ctx->btn_seek_back) lv_obj_clear_state(ctx->btn_seek_back, LV_STATE_DISABLED); + if (ctx->btn_seek_fwd) lv_obj_clear_state(ctx->btn_seek_fwd, LV_STATE_DISABLED); break; case STATE_IDLE: default: + set_title_scroll_enabled(ctx, true); if (ctx->filepath[0] != '\0') { lv_label_set_text(ctx->lbl_status, "Ready"); - lv_obj_clear_state(ctx->btn_play_pause, LV_STATE_DISABLED); + if (ctx->btn_play_pause) lv_obj_clear_state(ctx->btn_play_pause, LV_STATE_DISABLED); } else { lv_label_set_text(ctx->lbl_status, "No file loaded"); - lv_obj_add_state(ctx->btn_play_pause, LV_STATE_DISABLED); + if (ctx->btn_play_pause) lv_obj_add_state(ctx->btn_play_pause, LV_STATE_DISABLED); } - lv_label_set_text(lv_obj_get_child(ctx->btn_play_pause, 0), LV_SYMBOL_PLAY " Play"); - lv_obj_add_state(ctx->btn_stop, LV_STATE_DISABLED); + if (ctx->btn_play_pause) { + lv_obj_t* lbl = lv_obj_get_child(ctx->btn_play_pause, 0); + if (lbl) lv_label_set_text(lbl, LV_SYMBOL_PLAY); + } + if (ctx->btn_stop) lv_obj_add_state(ctx->btn_stop, LV_STATE_DISABLED); + if (ctx->btn_seek_back) lv_obj_add_state(ctx->btn_seek_back, LV_STATE_DISABLED); + if (ctx->btn_seek_fwd) lv_obj_add_state(ctx->btn_seek_fwd, LV_STATE_DISABLED); lv_bar_set_value(ctx->bar_progress, 0, LV_ANIM_OFF); + if (ctx->lbl_time) { + lv_label_set_text(ctx->lbl_time, "00:00 / 00:00"); + } break; } } +/* ─── Seek handling ─── */ +static void request_seek(AppCtx* ctx, int delta_sec) { + if (ctx->filepath[0] == '\0') return; + if (ctx->state == STATE_IDLE) return; // no task to seek + int target = ctx->pos_sec + delta_sec; + if (target < 0) target = 0; + if (ctx->total_sec > 0 && target > ctx->total_sec) target = ctx->total_sec - 1; + ctx->seek_request_sec = target; + ESP_LOGI(TAG, "Seek requested: %d -> %d (delta %d)", ctx->pos_sec, target, delta_sec); +} + /* ─── Playback Task ─── */ static void mp3_playback_task(void* arg) { AppCtx* ctx = (AppCtx*)arg; @@ -115,6 +521,46 @@ static void mp3_playback_task(void* arg) { ctx->file_size = ftell(ctx->file); fseek(ctx->file, 0, SEEK_SET); ctx->bytes_read_total = 0; + ctx->pos_sec = 0; + ctx->total_sec = 0; + ctx->total_decoded_samples = 0; + ctx->finished_naturally = false; + ctx->last_progress_pct = -1; + ctx->seek_request_sec = -1; + + // Handle ID3 and resume seek + uint8_t id3hdr[10]; + ctx->id3_skip = 0; + if (fread(id3hdr,1,10,ctx->file)==10 && memcmp(id3hdr,"ID3",3)==0) { + int id3size = (id3hdr[6]&0x7F)<<21 | (id3hdr[7]&0x7F)<<14 | (id3hdr[8]&0x7F)<<7 | (id3hdr[9]&0x7F); + ctx->id3_skip = id3size + 10; + fseek(ctx->file, ctx->id3_skip, SEEK_SET); + } else { + fseek(ctx->file, 0, SEEK_SET); + } + ctx->file_data_size = ctx->file_size - ctx->id3_skip; + if (ctx->file_data_size>0) ctx->total_sec = (int)(ctx->file_data_size / 16000); + + if (ctx->start_pos_sec > 5) { + long seek_bytes; + if (ctx->total_sec > 0 && ctx->file_data_size > 0) { + // Proportional seek based on estimated duration + seek_bytes = (long)((int64_t)ctx->start_pos_sec * ctx->file_data_size / ctx->total_sec); + } else { + seek_bytes = (long)ctx->start_pos_sec * 16000; + } + if (seek_bytes < 0) seek_bytes = 0; + if (seek_bytes >= ctx->file_data_size) seek_bytes = ctx->file_data_size - 1024; + if (seek_bytes < ctx->file_data_size) { + fseek(ctx->file, ctx->id3_skip + seek_bytes, SEEK_SET); + ctx->bytes_read_total = ctx->id3_skip + seek_bytes; + // total_decoded_samples is in samples, not bytes - use estimated sample rate + // 44100 Hz is common, use that for initial estimate; will be corrected after first frame + ctx->total_decoded_samples = (long)ctx->start_pos_sec * 44100; + ctx->pos_sec = ctx->start_pos_sec; + ESP_LOGI(TAG, "Resuming %s from %d sec (seek_bytes %ld, file_data %ld, total_est %d)", ctx->filepath, ctx->start_pos_sec, seek_bytes, ctx->file_data_size, ctx->total_sec); + } + } mp3dec_init(&ctx->decoder); ctx->buffered_bytes = 0; @@ -123,15 +569,50 @@ static void mp3_playback_task(void* arg) { ctx->channels = 0; ctx->stream_handle = NULL; - ESP_LOGI(TAG, "Starting MP3 playback: %s (size: %d bytes)", ctx->filepath, ctx->file_size); + ESP_LOGI(TAG, "Starting MP3 playback: %s (size: %d bytes) resume=%d", ctx->filepath, ctx->file_size, ctx->start_pos_sec); while (ctx->state != STATE_IDLE) { + // ─── Handle seek request ─── + if (ctx->seek_request_sec >= 0) { + int target = ctx->seek_request_sec; + ctx->seek_request_sec = -1; + long seek_bytes; + if (ctx->total_sec > 0 && ctx->file_data_size > 0) { + seek_bytes = (long)((int64_t)target * ctx->file_data_size / ctx->total_sec); + } else { + seek_bytes = (long)target * 16000; + } + if (seek_bytes < 0) seek_bytes = 0; + if (seek_bytes >= ctx->file_data_size) seek_bytes = ctx->file_data_size - 1024; + fseek(ctx->file, ctx->id3_skip + seek_bytes, SEEK_SET); + ctx->buffered_bytes = 0; + ctx->eof = false; + ctx->bytes_read_total = ctx->id3_skip + seek_bytes; + // Reset decoder for clean seek + mp3dec_init(&ctx->decoder); + if (ctx->sample_rate > 0) { + ctx->total_decoded_samples = (long)target * ctx->sample_rate; + } else { + ctx->total_decoded_samples = (long)target * 44100; + } + ctx->pos_sec = target; + ctx->last_progress_pct = -1; + ESP_LOGI(TAG, "Seeked to %d sec (bytes %ld)", target, seek_bytes); + // Update UI progress + if (ctx->file_size > 0) { + int pct = (int)(ctx->bytes_read_total * 100 / ctx->file_size); + if (pct < 0) pct = 0; + if (pct > 100) pct = 100; + tt_lvgl_lock(portMAX_DELAY); + lv_bar_set_value(ctx->bar_progress, pct, LV_ANIM_OFF); + tt_lvgl_unlock(); + } + } + if (ctx->state == STATE_PAUSED) { if (ctx->stream_handle != NULL) { - // Close stream immediately on pause to stop DMA audio_stream_close(ctx->stream_handle); ctx->stream_handle = NULL; - // Clear cached format to force re-open on resume ctx->sample_rate = 0; ctx->channels = 0; } @@ -139,15 +620,12 @@ static void mp3_playback_task(void* arg) { continue; } - // Fill input buffer if (!ctx->eof && ctx->buffered_bytes < MP3_INPUT_BUFFER_SIZE) { size_t to_read = MP3_INPUT_BUFFER_SIZE - ctx->buffered_bytes; size_t read_bytes = fread(ctx->input_buf + ctx->buffered_bytes, 1, to_read, ctx->file); ctx->buffered_bytes += read_bytes; ctx->bytes_read_total += read_bytes; - if (read_bytes == 0) { - ctx->eof = true; - } + if (read_bytes == 0) ctx->eof = true; } if (ctx->buffered_bytes == 0 && ctx->eof) { @@ -155,16 +633,12 @@ static void mp3_playback_task(void* arg) { break; } - // Decode one frame mp3dec_frame_info_t info; memset(&info, 0, sizeof(info)); int samples = mp3dec_decode_frame(&ctx->decoder, ctx->input_buf, (int)ctx->buffered_bytes, ctx->pcm_buf, &info); if (info.frame_bytes <= 0) { - if (ctx->eof) { - break; - } - // Move 1 byte forward to resync + if (ctx->eof) break; memmove(ctx->input_buf, ctx->input_buf + 1, --ctx->buffered_bytes); continue; } @@ -174,19 +648,21 @@ static void mp3_playback_task(void* arg) { memmove(ctx->input_buf, ctx->input_buf + consumed, ctx->buffered_bytes); if (samples > 0) { - // Configure audio-stream if format changed + ctx->total_decoded_samples += samples; + if (info.hz > 0) { + ctx->pos_sec = ctx->total_decoded_samples / info.hz; + } + if (ctx->sample_rate != info.hz || ctx->channels != info.channels) { if (ctx->stream_handle != NULL) { audio_stream_close(ctx->stream_handle); ctx->stream_handle = NULL; } - struct AudioStreamConfig config = { .sample_rate = (uint32_t)info.hz, .bits_per_sample = 16, .channels = (uint8_t)info.channels }; - error_t err = audio_stream_open_output(ctx->stream_dev, &config, &ctx->stream_handle); if (err != ERROR_NONE) { ESP_LOGE(TAG, "Failed to open audio stream: %d", err); @@ -197,7 +673,6 @@ static void mp3_playback_task(void* arg) { ESP_LOGI(TAG, "Audio stream opened: %d Hz, %d channels", info.hz, info.channels); } - // Adjust volume (after resampler) int vol = ctx->volume; int16_t* samples_ptr = (int16_t*)ctx->pcm_buf; size_t sample_count = (size_t)samples * info.channels; @@ -206,7 +681,6 @@ static void mp3_playback_task(void* arg) { samples_ptr[i] = (int16_t)scaled; } - // Play audio via audio-stream (resampled to native 44100) size_t offset = 0; size_t data_size = sample_count * sizeof(int16_t); bool write_err = false; @@ -220,32 +694,60 @@ static void mp3_playback_task(void* arg) { } offset += written; } - if (write_err) { - break; - } + if (write_err) break; } - // Update progress UI + // Progress bar: update when % changes (throttled) + int pct = -1; if (ctx->file_size > 0) { - int pct = (int)((ctx->bytes_read_total - ctx->buffered_bytes) * 100 / ctx->file_size); + pct = (int)((ctx->bytes_read_total - ctx->buffered_bytes) * 100 / ctx->file_size); if (pct < 0) pct = 0; if (pct > 100) pct = 100; + } + // Time label: update every second (fast) rather than only on % change + // This fixes user report: label updated every 10-15s for long files + static int last_time_ui_sec = -1; + bool need_time_update = (ctx->pos_sec != last_time_ui_sec); + bool need_bar_update = (pct >=0 && pct != ctx->last_progress_pct); + + if (need_bar_update || need_time_update) { + char time_buf[32]; + int cur_m = ctx->pos_sec / 60; + int cur_s = ctx->pos_sec % 60; + int tot_m = ctx->total_sec / 60; + int tot_s = ctx->total_sec % 60; + if (tot_m > 99) { tot_m = 99; tot_s = 59; } + snprintf(time_buf, sizeof(time_buf), "%02d:%02d / %02d:%02d", cur_m, cur_s, tot_m, tot_s); + tt_lvgl_lock(portMAX_DELAY); - lv_bar_set_value(ctx->bar_progress, pct, LV_ANIM_OFF); + if (need_bar_update && pct >=0) { + lv_bar_set_value(ctx->bar_progress, pct, LV_ANIM_OFF); + ctx->last_progress_pct = pct; + } + if (need_time_update) { + if (ctx->lbl_time) lv_label_set_text(ctx->lbl_time, time_buf); + last_time_ui_sec = ctx->pos_sec; + } tt_lvgl_unlock(); } } fclose(ctx->file); ctx->file = NULL; + + if (ctx->pos_sec > 0 && ctx->total_sec >0 && ctx->pos_sec >= ctx->total_sec - 10) { + ctx->pos_sec = 0; + ctx->finished_naturally = true; + } else { + ctx->finished_naturally = false; + } - // Close audio stream to clean up DMA / resampler task if (ctx->stream_handle != NULL) { audio_stream_close(ctx->stream_handle); ctx->stream_handle = NULL; } - ESP_LOGI(TAG, "Playback task finished"); + ESP_LOGI(TAG, "Playback task finished pos=%d total=%d naturally=%d", ctx->pos_sec, ctx->total_sec, ctx->finished_naturally); tt_lvgl_lock(portMAX_DELAY); ctx->state = STATE_IDLE; @@ -256,6 +758,18 @@ static void mp3_playback_task(void* arg) { vTaskDelete(NULL); } +static void wait_for_playback_task_to_exit(AppCtx* ctx) { + if (ctx->playback_task_handle == NULL) return; + + ctx->state = STATE_IDLE; + while (ctx->playback_task_handle != NULL) { + // Let the playback worker take LVGL for its final teardown update. + tt_lvgl_unlock(); + vTaskDelay(pdMS_TO_TICKS(10)); + tt_lvgl_lock(portMAX_DELAY); + } +} + /* ─── Callbacks ─── */ static void on_play_pause_click(lv_event_t* e) { AppCtx* ctx = (AppCtx*)lv_event_get_user_data(e); @@ -268,6 +782,10 @@ static void on_play_pause_click(lv_event_t* e) { } else if (ctx->state == STATE_PLAYING) { ctx->state = STATE_PAUSED; update_ui(ctx); + update_history_entry(ctx, ctx->filepath, ctx->pos_sec, ctx->total_sec); + save_history(ctx); + refresh_history_list(ctx); + ESP_LOGI(TAG, "History updated on pause: %s pos %d total %d", ctx->filepath, ctx->pos_sec, ctx->total_sec); } else if (ctx->state == STATE_PAUSED) { ctx->state = STATE_PLAYING; update_ui(ctx); @@ -277,68 +795,192 @@ static void on_play_pause_click(lv_event_t* e) { static void on_stop_click(lv_event_t* e) { AppCtx* ctx = (AppCtx*)lv_event_get_user_data(e); if (ctx->state == STATE_IDLE) return; - + update_history_entry(ctx, ctx->filepath, ctx->pos_sec, ctx->total_sec); + save_history(ctx); + refresh_history_list(ctx); + ESP_LOGI(TAG, "History updated on stop: %s pos %d total %d", ctx->filepath, ctx->pos_sec, ctx->total_sec); ctx->state = STATE_IDLE; update_ui(ctx); } +static void on_volume_slider_changed(lv_event_t* e) { + AppCtx* ctx = (AppCtx*)lv_event_get_user_data(e); + if (!ctx || !ctx->slider_volume) return; + ctx->volume = lv_slider_get_value(ctx->slider_volume); + if (ctx->volume < 0) ctx->volume = 0; + if (ctx->volume > 100) ctx->volume = 100; + save_volume(ctx); +} + +static void on_seek_back_click(lv_event_t* e) { + AppCtx* ctx = (AppCtx*)lv_event_get_user_data(e); + request_seek(ctx, -SEEK_SECONDS); +} + +static void on_seek_fwd_click(lv_event_t* e) { + AppCtx* ctx = (AppCtx*)lv_event_get_user_data(e); + request_seek(ctx, SEEK_SECONDS); +} + +static void on_close_click(lv_event_t* e) { + (void)e; + wait_for_playback_task_to_exit(&g_ctx); + tt_app_stop(); +} + +static void on_library_click(lv_event_t* e) { + (void)e; + show_history_screen(&g_ctx); +} + +static void on_history_cancel_click(lv_event_t* e) { + AppCtx* ctx = (AppCtx*)lv_event_get_user_data(e); + hide_history_screen(ctx); +} + +/* Consumed once by the host-side device test; never exposed in the player UI. */ +static void on_dev_autoclose_timer(lv_timer_t* timer) { + lv_timer_del(timer); + ESP_LOGI(TAG, "DEV_TEST invoking the normal Close path"); + on_close_click(NULL); +} + +static void schedule_dev_autoclose_if_requested(AppCtx* ctx) { + char marker_path[512] = {0}; + size_t marker_path_size = sizeof(marker_path); + tt_app_get_user_data_child_path(ctx->app_handle, "mp3player_dev_autoclose_ms", marker_path, &marker_path_size); + if (marker_path_size == 0) return; + + FILE* marker = fopen(marker_path, "r"); + if (!marker) return; + int delay_ms = 0; + int parsed = fscanf(marker, "%d", &delay_ms); + fclose(marker); + unlink(marker_path); + if (parsed != 1 || delay_ms < 1000 || delay_ms > 30000) { + ESP_LOGW(TAG, "DEV_TEST ignored invalid autoclose request"); + return; + } + + ESP_LOGI(TAG, "DEV_TEST scheduling normal Close in %d ms", delay_ms); + lv_timer_create(on_dev_autoclose_timer, delay_ms, NULL); +} + /* ─── App Lifecycle ─── */ static void onShowApp(AppHandle app, void* data, lv_obj_t* parent) { memset(&g_ctx, 0, sizeof(g_ctx)); g_ctx.volume = 80; + g_ctx.app_handle = app; + g_ctx.seek_request_sec = -1; - // Allocate memory buffers g_ctx.input_buf = (uint8_t*)malloc(MP3_INPUT_BUFFER_SIZE); g_ctx.pcm_buf = (mp3d_sample_t*)malloc(MINIMP3_MAX_SAMPLES_PER_FRAME * sizeof(mp3d_sample_t)); - // Find audio-stream device (resampling layer over ES8311 codec) - g_ctx.stream_dev = device_find_by_name("audio-stream"); - if (!g_ctx.stream_dev) { - ESP_LOGE(TAG, "Audio-stream device not found!"); + for (int tries=0; tries<5; tries++) { + struct Device* dev = device_find_by_name("audio-stream0"); + if (dev) { g_ctx.stream_dev = dev; break; } + ESP_LOGW(TAG, "Audio-stream not found, retry %d/5", tries); + vTaskDelay(pdMS_TO_TICKS(200)); } + if (!g_ctx.stream_dev) { + ESP_LOGE(TAG, "Audio-stream device not found after retries!"); + } + + // Load persisted volume and history (safe with heap buffer) + load_volume(&g_ctx); + load_history(&g_ctx); - // Parse launch parameters BundleHandle bundle = tt_app_get_parameters(app); + bool has_file_param = false; if (bundle) { - char file_param[256] = {0}; - if (tt_bundle_opt_string(bundle, "file", file_param, sizeof(file_param))) { - // Safely resolve SD card path + char file_param[512] = {0}; + if (tt_bundle_opt_string(bundle, "file", file_param, sizeof(file_param)) || + tt_bundle_opt_string(bundle, "path", file_param, sizeof(file_param)) || + tt_bundle_opt_string(bundle, "filepath", file_param, sizeof(file_param))) { if (strncmp(file_param, "/sdcard", 7) == 0) { strncpy(g_ctx.filepath, file_param, sizeof(g_ctx.filepath) - 1); } else { if (file_param[0] == '/') { - snprintf(g_ctx.filepath, sizeof(g_ctx.filepath), "/sdcard%s", file_param); + size_t needed = 7 + strlen(file_param) + 1; + if (needed <= sizeof(g_ctx.filepath)) { + strcpy(g_ctx.filepath, "/sdcard"); + strcat(g_ctx.filepath, file_param); + } else { + strncpy(g_ctx.filepath, file_param, sizeof(g_ctx.filepath)-1); + } } else { - snprintf(g_ctx.filepath, sizeof(g_ctx.filepath), "/sdcard/%s", file_param); + size_t needed = 8 + strlen(file_param) + 1; + if (needed <= sizeof(g_ctx.filepath)) { + strcpy(g_ctx.filepath, "/sdcard/"); + strcat(g_ctx.filepath, file_param); + } else { + strncpy(g_ctx.filepath, file_param, sizeof(g_ctx.filepath)-1); + } } } + g_ctx.filepath[sizeof(g_ctx.filepath)-1]=0; + has_file_param = true; + } + int32_t pos = 0; + if (tt_bundle_opt_int32(bundle, "pos_sec", &pos) || + tt_bundle_opt_int32(bundle, "position", &pos) || + tt_bundle_opt_int32(bundle, "timestamp", &pos)) { + g_ctx.start_pos_sec = pos; + } + int32_t vol = 0; + if (tt_bundle_opt_int32(bundle, "volume", &vol)) { + g_ctx.volume = vol; + } + } + + // Try resume from history (only 10min+ entries) + if (!has_file_param) { + char last_path[512]; + int last_pos = 0; + if (get_last_played(&g_ctx, last_path, sizeof(last_path), &last_pos)) { + strncpy(g_ctx.filepath, last_path, sizeof(g_ctx.filepath)-1); + g_ctx.filepath[sizeof(g_ctx.filepath)-1]=0; + g_ctx.start_pos_sec = last_pos; + ESP_LOGI(TAG, "Resuming last play %s at %d sec from history", g_ctx.filepath, last_pos); + has_file_param = true; + } + } + // Fallback test files if still no param + if (!has_file_param) { + const char *test_files[] = { + "/sdcard/dm/462_Gadgets_Escape_S01E02.mp3", + "/sdcard/dm/463_Discovering_Discovery_Mountain_S01E01.mp3", + "/sdcard/download/test2.mp3", + "/sdcard/download/test.mp3" + }; + for (int i=0; i<4; i++) { + struct stat st; + if (stat(test_files[i], &st)==0 && st.st_size>1024) { + strncpy(g_ctx.filepath, test_files[i], sizeof(g_ctx.filepath)-1); + g_ctx.filepath[sizeof(g_ctx.filepath)-1]=0; + ESP_LOGI(TAG, "No file param, using test file %s", g_ctx.filepath); + has_file_param = true; + break; + } } } // ─── UI Layout ─── - lv_obj_t* toolbar = tt_lvgl_toolbar_create_for_app(parent, app); - lv_obj_align(toolbar, LV_ALIGN_TOP_MID, 0, 0); - - // Player Card (rounded box) - lv_obj_t* card = lv_obj_create(parent); - lv_obj_set_size(card, lv_pct(90), lv_pct(70)); - lv_obj_align(card, LV_ALIGN_CENTER, 0, 15); - lv_obj_set_style_radius(card, 15, 0); - lv_obj_set_style_bg_color(card, lv_color_hex(0x1E1E2E), 0); - lv_obj_set_style_border_color(card, lv_color_hex(0x313244), 0); - lv_obj_set_style_border_width(card, 2, 0); - lv_obj_set_flex_flow(card, LV_FLEX_FLOW_COLUMN); - lv_obj_set_flex_align(card, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER); - lv_obj_set_style_pad_all(card, 15, 0); - lv_obj_set_style_pad_gap(card, 15, 0); - - // Audio Icon - lv_obj_t* icon = lv_label_create(card); - lv_label_set_text(icon, LV_SYMBOL_AUDIO); - lv_obj_set_style_text_color(icon, lv_color_hex(0x89B4FA), 0); - - // Track Title - g_ctx.lbl_title = lv_label_create(card); + // Full-screen player with only the requested bottom control row. + lv_obj_set_style_bg_color(parent, lv_color_hex(0x11111B), 0); + lv_obj_set_style_bg_opa(parent, LV_OPA_COVER, 0); + + // Center content vertically (title, status, progress, time) + lv_obj_t* center_box = lv_obj_create(parent); + lv_obj_remove_style_all(center_box); + lv_obj_set_size(center_box, lv_pct(90), lv_pct(45)); + lv_obj_align(center_box, LV_ALIGN_CENTER, 0, -60); + lv_obj_set_flex_flow(center_box, LV_FLEX_FLOW_COLUMN); + lv_obj_set_flex_align(center_box, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER); + lv_obj_set_style_pad_gap(center_box, 12, 0); + g_ctx.center_box = center_box; + + g_ctx.lbl_title = lv_label_create(center_box); lv_obj_set_width(g_ctx.lbl_title, lv_pct(95)); lv_obj_set_style_text_align(g_ctx.lbl_title, LV_TEXT_ALIGN_CENTER, 0); lv_obj_set_style_text_color(g_ctx.lbl_title, lv_color_hex(0xCDD6F4), 0); @@ -346,89 +988,235 @@ static void onShowApp(AppHandle app, void* data, lv_obj_t* parent) { const char* last_slash = strrchr(g_ctx.filepath, '/'); const char* filename = last_slash ? last_slash + 1 : g_ctx.filepath; lv_label_set_text(g_ctx.lbl_title, filename); +#if ENABLE_TITLE_SCROLL_ANIM_WHEN_IDLE lv_label_set_long_mode(g_ctx.lbl_title, LV_LABEL_LONG_SCROLL_CIRCULAR); +#else + lv_label_set_long_mode(g_ctx.lbl_title, LV_LABEL_LONG_DOT); +#endif } else { lv_label_set_text(g_ctx.lbl_title, "No File Parameter"); + lv_label_set_long_mode(g_ctx.lbl_title, LV_LABEL_LONG_DOT); } - // Playback Status - g_ctx.lbl_status = lv_label_create(card); + g_ctx.lbl_status = lv_label_create(center_box); + lv_label_set_long_mode(g_ctx.lbl_status, LV_LABEL_LONG_DOT); lv_obj_set_style_text_color(g_ctx.lbl_status, lv_color_hex(0xA6ADC8), 0); - // Progress Bar - g_ctx.bar_progress = lv_bar_create(card); - lv_obj_set_size(g_ctx.bar_progress, lv_pct(85), 8); + g_ctx.bar_progress = lv_bar_create(center_box); + lv_obj_set_size(g_ctx.bar_progress, lv_pct(88), 8); lv_bar_set_range(g_ctx.bar_progress, 0, 100); lv_bar_set_value(g_ctx.bar_progress, 0, LV_ANIM_OFF); lv_obj_set_style_bg_color(g_ctx.bar_progress, lv_color_hex(0x45475A), LV_PART_MAIN); lv_obj_set_style_bg_color(g_ctx.bar_progress, lv_color_hex(0x89B4FA), LV_PART_INDICATOR); - - // Controls Container - lv_obj_t* ctrl_box = lv_obj_create(card); + + g_ctx.lbl_time = lv_label_create(center_box); + lv_obj_set_style_text_color(g_ctx.lbl_time, lv_color_hex(0xA6ADC8), 0); + lv_obj_set_style_text_font(g_ctx.lbl_time, lv_theme_get_font_small(center_box), 0); + lv_label_set_text(g_ctx.lbl_time, "00:00 / 00:00"); + + // History full-screen container (hidden by default) + lv_obj_t* history_cont = lv_obj_create(parent); + lv_obj_remove_style_all(history_cont); + lv_obj_set_size(history_cont, lv_pct(100), lv_pct(100)); + lv_obj_align(history_cont, LV_ALIGN_CENTER, 0, 0); + lv_obj_set_style_bg_color(history_cont, lv_color_hex(0x11111B), 0); + lv_obj_set_style_bg_opa(history_cont, LV_OPA_COVER, 0); + lv_obj_set_flex_flow(history_cont, LV_FLEX_FLOW_COLUMN); + lv_obj_set_flex_align(history_cont, LV_FLEX_ALIGN_START, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_START); + lv_obj_set_style_pad_all(history_cont, 10, 0); + lv_obj_set_style_pad_gap(history_cont, 8, 0); + lv_obj_add_flag(history_cont, LV_OBJ_FLAG_HIDDEN); + g_ctx.history_container = history_cont; + + // History header with title + cancel button + lv_obj_t* hist_header = lv_obj_create(history_cont); + lv_obj_remove_style_all(hist_header); + lv_obj_set_size(hist_header, lv_pct(95), 32); + lv_obj_set_flex_flow(hist_header, LV_FLEX_FLOW_ROW); + lv_obj_set_flex_align(hist_header, LV_FLEX_ALIGN_SPACE_BETWEEN, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER); + + lv_obj_t* lbl_hist = lv_label_create(hist_header); + lv_label_set_text(lbl_hist, "History"); + lv_obj_set_style_text_color(lbl_hist, lv_color_hex(0xCDD6F4), 0); + lv_obj_set_style_text_font(lbl_hist, lv_theme_get_font_small(hist_header), 0); + + lv_obj_t* btn_cancel = lv_btn_create(hist_header); + lv_obj_set_size(btn_cancel, 60, 28); + lv_obj_set_style_radius(btn_cancel, 8, 0); + lv_obj_set_style_bg_color(btn_cancel, lv_color_hex(0x313244), 0); + lv_obj_t* lbl_cancel = lv_label_create(btn_cancel); + lv_label_set_text(lbl_cancel, "Back"); + lv_obj_center(lbl_cancel); + lv_obj_add_event_cb(btn_cancel, on_history_cancel_click, LV_EVENT_CLICKED, &g_ctx); + + g_ctx.list_history = lv_list_create(history_cont); + lv_obj_set_size(g_ctx.list_history, lv_pct(95), lv_pct(85)); + lv_obj_set_style_bg_color(g_ctx.list_history, lv_color_hex(0x1E1E2E), 0); + lv_obj_set_style_border_color(g_ctx.list_history, lv_color_hex(0x313244), 0); + lv_obj_set_style_border_width(g_ctx.list_history, 1, 0); + lv_obj_set_style_radius(g_ctx.list_history, 8, 0); + lv_obj_set_style_pad_all(g_ctx.list_history, 2, 0); + + // Populate history list + refresh_history_list(&g_ctx); + + // Visible volume row restored above the fixed six-button bottom row. + lv_obj_t* vol_box = lv_obj_create(parent); + lv_obj_remove_style_all(vol_box); + lv_obj_set_size(vol_box, lv_pct(92), 26); + lv_obj_align(vol_box, LV_ALIGN_BOTTOM_MID, 0, -60); + g_ctx.volume_box = vol_box; + lv_obj_set_flex_flow(vol_box, LV_FLEX_FLOW_ROW); + lv_obj_set_flex_align(vol_box, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER); + lv_obj_set_style_pad_column(vol_box, 8, 0); + + lv_obj_t* lbl_volume = lv_label_create(vol_box); + lv_label_set_text(lbl_volume, "Volume"); + lv_obj_set_style_text_color(lbl_volume, lv_color_hex(0xCDD6F4), 0); + lv_obj_set_style_text_font(lbl_volume, lv_theme_get_font_small(vol_box), 0); + + g_ctx.slider_volume = lv_slider_create(vol_box); + lv_obj_set_size(g_ctx.slider_volume, 170, 12); + lv_slider_set_range(g_ctx.slider_volume, 0, 100); + lv_slider_set_value(g_ctx.slider_volume, g_ctx.volume, LV_ANIM_OFF); + lv_obj_set_style_bg_color(g_ctx.slider_volume, lv_color_hex(0x313244), LV_PART_MAIN); + lv_obj_set_style_bg_color(g_ctx.slider_volume, lv_color_hex(0x89B4FA), LV_PART_INDICATOR); + lv_obj_add_event_cb(g_ctx.slider_volume, on_volume_slider_changed, LV_EVENT_VALUE_CHANGED, &g_ctx); + + // Bottom-only controls: Close, -15s, Play/Pause, Stop, +15s, Folder. + lv_obj_t* bottom_box = lv_obj_create(parent); + lv_obj_remove_style_all(bottom_box); + lv_obj_set_size(bottom_box, lv_pct(100), 52); + lv_obj_align(bottom_box, LV_ALIGN_BOTTOM_MID, 0, -5); + g_ctx.bottom_box = bottom_box; + lv_obj_set_flex_flow(bottom_box, LV_FLEX_FLOW_COLUMN); + lv_obj_set_flex_align(bottom_box, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER); + lv_obj_set_style_pad_all(bottom_box, 5, 0); + + lv_obj_t* ctrl_box = lv_obj_create(bottom_box); lv_obj_remove_style_all(ctrl_box); - lv_obj_set_size(ctrl_box, lv_pct(90), 45); + lv_obj_set_size(ctrl_box, lv_pct(98), 42); lv_obj_set_flex_flow(ctrl_box, LV_FLEX_FLOW_ROW); lv_obj_set_flex_align(ctrl_box, LV_FLEX_ALIGN_SPACE_EVENLY, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER); - - // Play/Pause Button + + lv_obj_t* btn_close = lv_btn_create(ctrl_box); + lv_obj_set_size(btn_close, 34, 36); + lv_obj_set_style_radius(btn_close, 12, 0); + lv_obj_set_style_bg_color(btn_close, lv_color_hex(0x313244), 0); + lv_obj_t* lbl_close = lv_label_create(btn_close); + lv_label_set_text(lbl_close, LV_SYMBOL_CLOSE); + lv_obj_center(lbl_close); + lv_obj_add_event_cb(btn_close, on_close_click, LV_EVENT_CLICKED, &g_ctx); + + // -15s + g_ctx.btn_seek_back = lv_btn_create(ctrl_box); + lv_obj_set_size(g_ctx.btn_seek_back, 46, 36); + lv_obj_set_style_radius(g_ctx.btn_seek_back, 12, 0); + lv_obj_set_style_bg_color(g_ctx.btn_seek_back, lv_color_hex(0x313244), 0); + lv_obj_set_style_text_color(g_ctx.btn_seek_back, lv_color_hex(0xCDD6F4), 0); + lv_obj_t* lbl_back = lv_label_create(g_ctx.btn_seek_back); + lv_label_set_text(lbl_back, "-15s"); + lv_obj_center(lbl_back); + lv_obj_add_event_cb(g_ctx.btn_seek_back, on_seek_back_click, LV_EVENT_CLICKED, &g_ctx); + g_ctx.btn_play_pause = lv_btn_create(ctrl_box); - lv_obj_set_size(g_ctx.btn_play_pause, 100, 36); + lv_obj_set_size(g_ctx.btn_play_pause, 44, 36); lv_obj_set_style_radius(g_ctx.btn_play_pause, 18, 0); lv_obj_set_style_bg_color(g_ctx.btn_play_pause, lv_color_hex(0x89B4FA), 0); lv_obj_set_style_text_color(g_ctx.btn_play_pause, lv_color_hex(0x11111B), 0); lv_obj_t* lbl_play = lv_label_create(g_ctx.btn_play_pause); - lv_label_set_text(lbl_play, LV_SYMBOL_PLAY " Play"); + lv_label_set_text(lbl_play, LV_SYMBOL_PLAY); lv_obj_center(lbl_play); lv_obj_add_event_cb(g_ctx.btn_play_pause, on_play_pause_click, LV_EVENT_CLICKED, &g_ctx); - - // Stop Button + g_ctx.btn_stop = lv_btn_create(ctrl_box); - lv_obj_set_size(g_ctx.btn_stop, 100, 36); - lv_obj_set_style_radius(g_ctx.btn_stop, 18, 0); + lv_obj_set_size(g_ctx.btn_stop, 38, 36); + lv_obj_set_style_radius(g_ctx.btn_stop, 12, 0); lv_obj_set_style_bg_color(g_ctx.btn_stop, lv_color_hex(0xF38BA8), 0); lv_obj_set_style_text_color(g_ctx.btn_stop, lv_color_hex(0x11111B), 0); lv_obj_t* lbl_stop = lv_label_create(g_ctx.btn_stop); - lv_label_set_text(lbl_stop, LV_SYMBOL_STOP " Stop"); + lv_label_set_text(lbl_stop, LV_SYMBOL_STOP); lv_obj_center(lbl_stop); lv_obj_add_event_cb(g_ctx.btn_stop, on_stop_click, LV_EVENT_CLICKED, &g_ctx); + + // +15s + g_ctx.btn_seek_fwd = lv_btn_create(ctrl_box); + lv_obj_set_size(g_ctx.btn_seek_fwd, 46, 36); + lv_obj_set_style_radius(g_ctx.btn_seek_fwd, 12, 0); + lv_obj_set_style_bg_color(g_ctx.btn_seek_fwd, lv_color_hex(0x313244), 0); + lv_obj_set_style_text_color(g_ctx.btn_seek_fwd, lv_color_hex(0xCDD6F4), 0); + lv_obj_t* lbl_fwd = lv_label_create(g_ctx.btn_seek_fwd); + lv_label_set_text(lbl_fwd, "+15s"); + lv_obj_center(lbl_fwd); + lv_obj_add_event_cb(g_ctx.btn_seek_fwd, on_seek_fwd_click, LV_EVENT_CLICKED, &g_ctx); + + // Folder opens the persisted in-app playback history. + lv_obj_t* btn_hist = lv_btn_create(ctrl_box); + lv_obj_set_size(btn_hist, 34, 36); + lv_obj_set_style_radius(btn_hist, 12, 0); + lv_obj_set_style_bg_color(btn_hist, lv_color_hex(0x313244), 0); + lv_obj_t* lbl_hist_btn = lv_label_create(btn_hist); + lv_label_set_text(lbl_hist_btn, LV_SYMBOL_DIRECTORY); + lv_obj_center(lbl_hist_btn); + lv_obj_add_event_cb(btn_hist, on_library_click, LV_EVENT_CLICKED, &g_ctx); if (!g_ctx.stream_dev) { lv_label_set_text(g_ctx.lbl_status, "Error: Audio Not Found"); lv_obj_add_state(g_ctx.btn_play_pause, LV_STATE_DISABLED); lv_obj_add_state(g_ctx.btn_stop, LV_STATE_DISABLED); + lv_obj_add_state(g_ctx.btn_seek_back, LV_STATE_DISABLED); + lv_obj_add_state(g_ctx.btn_seek_fwd, LV_STATE_DISABLED); } else if (!g_ctx.input_buf || !g_ctx.pcm_buf) { lv_label_set_text(g_ctx.lbl_status, "Error: Out of Memory"); lv_obj_add_state(g_ctx.btn_play_pause, LV_STATE_DISABLED); lv_obj_add_state(g_ctx.btn_stop, LV_STATE_DISABLED); + lv_obj_add_state(g_ctx.btn_seek_back, LV_STATE_DISABLED); + lv_obj_add_state(g_ctx.btn_seek_fwd, LV_STATE_DISABLED); } else { g_ctx.state = STATE_IDLE; update_ui(&g_ctx); - - // Auto-play if a file parameter was provided if (g_ctx.filepath[0] != '\0') { g_ctx.state = STATE_PLAYING; update_ui(&g_ctx); xTaskCreate(mp3_playback_task, "mp3_play", 6144, &g_ctx, 6, &g_ctx.playback_task_handle); + schedule_dev_autoclose_if_requested(&g_ctx); } } } static void onHideApp(AppHandle app, void* data) { + (void)app; (void)data; + ESP_LOGI(TAG, "onHide start pos_sec=%d", g_ctx.pos_sec); + if (g_ctx.state != STATE_IDLE) { - g_ctx.state = STATE_IDLE; // Signal task to stop + g_ctx.state = STATE_IDLE; } - // Wait for the task to finish self-deletion to prevent resource leaks - while (g_ctx.playback_task_handle != NULL) { - vTaskDelay(pdMS_TO_TICKS(10)); + int tries=0; + while (g_ctx.playback_task_handle != NULL && tries<50) { + vTaskDelay(pdMS_TO_TICKS(50)); + tries++; } - - // Close any open audio stream + if (g_ctx.playback_task_handle != NULL) { + ESP_LOGW(TAG, "Playback task stuck, force deleting"); + vTaskDelete(g_ctx.playback_task_handle); + g_ctx.playback_task_handle = NULL; + } + + ESP_LOGI(TAG, "Playback task stopped, closing audio stream"); + if (g_ctx.stream_handle != NULL) { audio_stream_close(g_ctx.stream_handle); g_ctx.stream_handle = NULL; + vTaskDelay(pdMS_TO_TICKS(100)); } - + + // Persist the active track for normal relaunch, regardless of duration. + save_volume(&g_ctx); + update_history_entry(&g_ctx, g_ctx.filepath, g_ctx.pos_sec, g_ctx.total_sec); + save_history(&g_ctx); + ESP_LOGI(TAG, "History saved on hide: %s pos %d total %d", g_ctx.filepath, g_ctx.pos_sec, g_ctx.total_sec); + if (g_ctx.input_buf) { free(g_ctx.input_buf); g_ctx.input_buf = NULL; @@ -437,6 +1225,25 @@ static void onHideApp(AppHandle app, void* data) { free(g_ctx.pcm_buf); g_ctx.pcm_buf = NULL; } + + size_t free_internal = heap_caps_get_free_size(MALLOC_CAP_INTERNAL); + ESP_LOGI(TAG, "Heap after free buffers: internal=%d", free_internal); + + if (free_internal > 6000) { + BundleHandle result = tt_bundle_alloc(); + if (result) { + tt_bundle_put_int32(result, "pos_sec", g_ctx.pos_sec); + tt_bundle_put_int32(result, "timestamp", g_ctx.pos_sec); + tt_app_set_result(g_ctx.app_handle, APP_RESULT_OK, result); + ESP_LOGI(TAG, "Set result bundle pos_sec=%d", g_ctx.pos_sec); + } else { + tt_app_set_result(g_ctx.app_handle, APP_RESULT_OK, NULL); + } + } else { + tt_app_set_result(g_ctx.app_handle, APP_RESULT_OK, NULL); + } + + ESP_LOGI(TAG, "onHide returning pos_sec=%d total_sec=%d finished=%d", g_ctx.pos_sec, g_ctx.total_sec, g_ctx.finished_naturally); } int main(int argc, char* argv[]) { diff --git a/Apps/PipecatVoice/CMakeLists.txt b/Apps/PipecatVoice/CMakeLists.txt new file mode 100644 index 0000000..e0e5c5c --- /dev/null +++ b/Apps/PipecatVoice/CMakeLists.txt @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 3.20) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) + +if (DEFINED ENV{TACTILITY_SDK_PATH}) + set(TACTILITY_SDK_PATH $ENV{TACTILITY_SDK_PATH}) +else() + set(TACTILITY_SDK_PATH "../../release/TactilitySDK") + message(WARNING "TACTILITY_SDK_PATH is not set, defaulting to ${TACTILITY_SDK_PATH}") +endif() + +include("${TACTILITY_SDK_PATH}/TactilitySDK.cmake") +set(EXTRA_COMPONENT_DIRS ${TACTILITY_SDK_PATH}) + +project(PipecatVoice) +tactility_project(PipecatVoice) diff --git a/Apps/PipecatVoice/README.md b/Apps/PipecatVoice/README.md index 8f8781e..a49fda2 100644 --- a/Apps/PipecatVoice/README.md +++ b/Apps/PipecatVoice/README.md @@ -1,19 +1,30 @@ -# PipecatVoice — Tactility native voice companion +# Pipecat Voice -Native ESP32 ELF app (like VoiceRecorder/ReynaBot), not a web client. +Minimal native Tactility client for the LAN voice-adapter protocol v1. Opening the app immediately connects to `ws://192.168.68.102:8644/api/esp32/voice/ws`, sends the versioned `start` declaration, and continuously streams 16 kHz mono signed-16-bit PCM. There is no Connect button, speaker/transport picker, PTT control, text entry, transcript display, or credential on the device. -- Uses `audio-stream`/`i2s_controller` APIs (see VoiceRecorder) for 16kHz PCM mic + speaker -- Uses ReynaBot's `websocket.c/h` + `lwip` for WS transport -- Default gateway: Hermes WS `ws://:8642/api/esp32/voice/ws` (voice profile, limited tools, imperfect STT) -- Optional second target: Pipecat websocket transport `ws://:7861` (when server exposes `websocket` via `create_transport`) +The Mac-hosted adapter owns VAD, STT, Pipecat/Hermes orchestration, TTS, and credentials. It returns metadata followed by a single PCM response frame; the app pauses capture, plays that response at the declared rate through Tactility `i2s0`, then resumes capture. The app only displays connection/streaming/reconnect/configuration state and uses the standard toolbar to exit. -Build: +## Configuration + +`config.json` in app user data can override the non-secret endpoint and allowed adapter device id: + +```json +{"server_url":"ws://192.168.68.102:8644/api/esp32/voice/ws","device_id":"tactility-14c19d1a790"} ``` -unset PYTHONPATH; export IDF_PYTHON_ENV_PATH=~/.espressif/python_env/idf5.3_py3.9_env -. ~/esp/esp-idf/export.sh -export TACTILITY_SDK_PATH=~/Projects/electronics/tactility/tactility/Buildscripts/TactilitySDK # or firmware/release/TactilitySDK if built + +Only `ws://` private-LAN endpoints are accepted; loopback endpoints are rejected. Invalid configuration is a terminal actionable state. Network failures use bounded 1, 2, 4, 8, 16, then 30-second reconnect delays. Protocol and payload violations close the session and reconnect; stale audio is never queued. + +## Build and test + +```sh +cc -std=c11 -Wall -Wextra -Werror -I main/Source tests/test_voice_protocol.c main/Source/voice_protocol.c -o /tmp/pipecatvoice-protocol-test +/tmp/pipecatvoice-protocol-test + +unset PYTHONPATH PYTHONHOME +export IDF_PYTHON_ENV_PATH=/Users/adolforeyna/.espressif/python_env/idf5.3_py3.9_env +source /Users/adolforeyna/esp/esp-idf/export.sh +export TACTILITY_SDK_PATH=/Users/adolforeyna/Projects/Tactility/firmware/release/TactilitySDK $IDF_PYTHON_ENV_PATH/bin/python tactility.py Apps/PipecatVoice build esp32s3 --local-sdk -$IDF_PYTHON_ENV_PATH/bin/python tactility.py Apps/PipecatVoice install 192.168.68.112 esp32s3 ``` -Adapted from ReynaBot (PTT + I2S + WS JSON events: ready/listening/transcript/thinking/response_text/audio_start/audio_end/done/error). +The compatible adapter is documented at `/Users/adolforeyna/Projects/voice-assistant/hermes-esp32-voice-gateway/docs/lan-ws-pipecat-adapter.md`. It is the only supported endpoint; Pipecat `:7861` is not an optional device transport. \ No newline at end of file diff --git a/Apps/PipecatVoice/docs/smallwebrtc-esp32-feasibility-spike.md b/Apps/PipecatVoice/docs/smallwebrtc-esp32-feasibility-spike.md new file mode 100644 index 0000000..cf9f1a2 --- /dev/null +++ b/Apps/PipecatVoice/docs/smallwebrtc-esp32-feasibility-spike.md @@ -0,0 +1,89 @@ +# Pipecat SmallWebRTC ESP32 feasibility spike + +Date: 2026-08-10 + +## Decision + +**NO-GO for a Tactility external ELF on the current SDK; GO only for a separate, full ESP-IDF firmware application.** + +Pipecat has an official native ESP32 client, so the required transport is real and source-proven. It cannot presently be used as a Tactility runtime ELF without a firmware/SDK integration project: the client is a full ESP-IDF firmware with private, static component dependencies and system configuration that the ELF loader does not provide or export. Do not replace the rejected raw-WebSocket implementation with another transport until that integration is designed and proven. + +This is not a proposal to flash anything. No device was deployed or flashed during this spike. + +## Source-pinned native client + +| Item | Evidence | +| --- | --- | +| Client | `https://github.com/pipecat-ai/pipecat-esp32`, commit `e70e3b1f0576e502af9e390434e1a6e8a5cd0d2e`, cloned with all recursive submodules | +| Top-level licence | MIT (`LICENSE`, copyright Daily/OpenAI) | +| Pipecat server | local installed `pipecat-ai 1.7.0`, Python 3.11 virtual environment | +| Server ESP32 support | `SmallWebRTCRequestHandler(..., esp32_mode=True, host=...)` munges the SDP; `smallwebrtc_sdp_munging()` removes SHA-384/SHA-512 fingerprints and retains only the chosen host's ICE candidates | +| ESP-IDF build used | local ESP-IDF `v5.5.2`, environment `idf5.5_py3.9_env` | +| Native build result | upstream `esp32-s3-box-3` built successfully, including `peer`, `srtp`, `esp-libopus`, Wi-Fi, HTTP client, DTLS/SRTP, Opus, and the ESP-BOX-3 BSP | + +The official client is designed for ESP32-S3 and uses the `libpeer` API. Its `PeerConfiguration` sets `CODEC_OPUS`, creates a peer connection, installs ICE/data/audio callbacks, and invokes `peer_connection_create_offer()`. This is the source-proven native WebRTC implementation; it owns ICE, DTLS-SRTP, RTP, and Opus rather than hand-implementing any of them. + +## Required SmallWebRTC contract + +The product transport is SmallWebRTC HTTP signaling plus WebRTC media, never the old raw WebSocket PCM/JSON protocol: + +1. `POST /start` with `transport: "webrtc"`, `enableDefaultIceServers: false`, and optional `body`; retain the returned `sessionId`. +2. `POST /sessions/{sessionId}/api/offer` with `{ "sdp": ..., "type": "offer", "pc_id": optional, "restart_pc": optional, "requestData": optional }`; Pipecat returns SDP answer, type, and `pc_id`. +3. `PATCH /sessions/{sessionId}/api/offer` with `{ "pc_id": ..., "candidates": [{ "candidate": ..., "sdp_mid": ..., "sdp_mline_index": ... }] }` for trickle ICE. An empty candidate is the end-of-candidates marker. +4. Use the negotiated WebRTC audio track continuously. The runner starts the bot after the offer is processed. + +The local Pipecat source also supports the direct `/api/offer` route used by the current official ESP32 example. The session form above is the approved application contract because it supports Pipecat runner session lifecycle. The live endpoint returned HTTP 200 to `/status`, but it was not restarted with `--esp32`; therefore no live offer/candidate exchange is represented as ESP32 validation. + +## Audio adapter boundary + +The official client source (`media.cpp`) uses 16 kHz, mono, signed 16-bit PCM (`640` bytes = 320 samples = 20 ms) and encodes it as Opus for `peer_connection_send_audio()`. Inbound WebRTC audio reaches the `onaudiotrack` callback as Opus, is decoded to the same PCM shape, and is written to the speaker codec. + +For a future firmware-level integration, Tactility must keep ownership at the following boundary (no hard-coded board pins): + +- acquire the existing Tactility `audio_stream` / `i2s_controller` service; +- pull fixed 20 ms frames, 16 kHz mono S16LE, into the native client encoder; +- feed decoded remote S16LE frames to the existing output service; +- serialize I/O ownership, keep bounded queues, and drop stale audio rather than accumulating latency; +- close peer/media callbacks before releasing the audio device. + +The current kernel exports `audio_stream_open_input`, `audio_stream_open_output`, `audio_stream_read`, `audio_stream_write`, `audio_stream_close`, and `i2s_controller_read`/`i2s_controller_write`. Those APIs are the usable boundary, not a reason to configure physical pins in the app. + +## Full-firmware build evidence + +The following was run in a temporary checkout; non-secret placeholder Wi-Fi values were used and no flash command was run: + +```text +cd /tmp/pipecat-esp32-spike +# cloned pipecat-esp32 at e70e3b1... and initialized all recursive submodules +cd esp32-s3-box-3 +unset PYTHONPATH PYTHONHOME +export IDF_PYTHON_ENV_PATH=/Users/adolforeyna/.espressif/python_env/idf5.5_py3.9_env +export WIFI_SSID=spike +export WIFI_PASSWORD=spike +export PIPECAT_SMALLWEBRTC_URL=http://192.168.68.112:7860/api/offer +source /Users/adolforeyna/esp/esp-idf/export.sh +idf.py build +``` + +Actual result: `src.elf` and `src.bin` were produced; the IDF build ended with `Project build complete`. `src.bin` is **1,493,408 bytes** and the upstream 1.5 MiB app partition reported **79,712 bytes (5%) free**. `xtensa-esp32s3-elf-size src.elf` reported text `1,304,360`, data `201,012`, bss `2,863,205` (total `4,368,577`). The linked firmware has no undefined dynamic symbols. + +This is important capacity evidence: even before adapting it to the target board and Tactility services, the supported client nearly fills its own dedicated application partition and has a 2.86 MiB BSS footprint. + +## Why this does not link as a Tactility ELF + +Tactility's `TactilitySDK.cmake` calls `project_elf()`. Its loader CMake builds a PIC shared ELF with `-nostartfiles -nostdlib -shared -e app_main`, and links only `main` plus explicitly listed `ELF_COMPONENTS` / `ELF_LIBS`. The current PipecatVoice component declares only `REQUIRES TactilitySDK lwip`. + +The upstream client instead requires full firmware components including `peer`, `srtp`, `esp-libopus`, `esp_http_client`, `esp_wifi`, `nvs_flash`, `esp_psram`, `esp_netif`, mbedTLS, and ESP-BOX-3 BSP. The official `peer` static archive has unresolved references to the linked firmware environment such as `mbedtls_ssl_conf_dtls_srtp_protection_profiles`, `mbedtls_ssl_config_defaults`, `lwip_inet_ntop`, and socket/ICE helpers. The Tactility kernel export table contains the audio service APIs listed above but no `peer_connection`, `opus_*`, `srtp_*`, `mbedtls_*`, `esp_http_client*`, `esp_wifi*`, or `esp_netif*` exports. + +Attempting the ordinary app build also hit a concrete local SDK packaging blocker before linking: `tactility.py Apps/PipecatVoice build esp32s3 --local-sdk` reported that `Buildscripts/TactilitySDK/0.8.0-dev-esp32s3/TactilitySDK` is missing. This must be corrected for later normal ELF builds, but it is distinct from the component/loader incompatibility. + +Therefore copying the client sources or merely adding `REQUIRES peer` would not make a runnable ELF: it would either fail to find the private IDF component libraries during the ELF link or produce imports that the firmware loader cannot resolve. Statically embedding all dependencies is unproven and high-risk because of ELF size, duplicate runtime/library state, SDK configuration, and Wi-Fi/codec ownership conflicts. + +## Security, licensing, and follow-up gate + +- Do not log SDP, ICE details, credentials, or raw audio. The source-level HTTP helper currently logs offer/answer in debug mode; any reused code must remove that logging. +- The upstream defaults deliberately disable TLS certificate verification for its demo. Production must use HTTPS with a pinned/validated trust chain; do not inherit that setting. +- Preserve MIT notices for Pipecat ESP32 and audit each pinned submodule separately (`libpeer`, SRTP/libSRTP, Opus, and Espressif managed components have their own licenses). +- Do not use the existing PipecatVoice raw WebSocket code or its historical configuration as a fallback; it is protocol-incompatible with SmallWebRTC. + +A firmware-level project must first export/package the required WebRTC dependency set, prove an external ELF link with zero unresolved loader symbols (or move the client into firmware), set deterministic memory budgets, and then perform an `--esp32` SmallWebRTC live offer/ICE/media test. Only after that gate may the approved minimal auto-start UI be implemented. diff --git a/Apps/PipecatVoice/main/Source/main.c b/Apps/PipecatVoice/main/Source/main.c index 119b184..8351f03 100644 --- a/Apps/PipecatVoice/main/Source/main.c +++ b/Apps/PipecatVoice/main/Source/main.c @@ -1,830 +1,320 @@ #include #include #include -#include #include -#include +#include + -#include "websocket.h" #include +#include +#include +#include +#include +#include -#include +#include #include -#include -#include +#include -#include "freertos/FreeRTOS.h" -#include "freertos/task.h" -#include "esp_log.h" +#include "voice_protocol.h" +#include "websocket.h" + +/* Exported by firmware 0.8.0-dev although absent from the CDN SDK header. */ +struct Device* device_find_by_name(const char* name); +struct Device* device_find_first_by_type(const struct DeviceType* type); #define TAG "PipecatVoice" - -typedef enum { - STATE_IDLE, - STATE_CONNECTING, - STATE_LISTENING, - STATE_THINKING, - STATE_SPEAKING, - STATE_ERROR -} PipecatVoiceState; +#define DEFAULT_ENDPOINT "ws://192.168.68.102:8644/api/esp32/voice/ws" +#define DEFAULT_DEVICE_ID "tactility-14c19d1a790" typedef struct { AppHandle app; - bool visible; - PipecatVoiceState state; - char server_url[128]; + volatile bool visible; + volatile bool streaming; + volatile bool playing; + volatile bool socket_failed; + int fd; + PvState state; + unsigned retry_attempt; + size_t expected_audio_bytes; + char endpoint[128]; char device_id[64]; char api_key[128]; - - char last_transcript[256]; - char last_response[512]; - char error_message[128]; - - // UI components - lv_obj_t* lbl_status; - lv_obj_t* lbl_transcript; - lv_obj_t* lbl_response; - lv_obj_t* btn_ptt; - lv_obj_t* btn_ptt_label; - lv_obj_t* btn_grace; - lv_obj_t* btn_elias; - - bool speaker_selected; - lv_obj_t* speaker_select_cont; - lv_obj_t* app_main_cont; - - // PTT control - bool is_pressed; - bool start_session; - bool stop_session; - bool cancel_session; - - // FreeRTOS Tasks - TaskHandle_t worker_task; - TaskHandle_t rx_task; - - // Hardware - struct Device* i2s_dev; - - // WebSocket - int ws_fd; - bool ws_connected; - bool ws_done; - - bool ui_update_pending; -} PipecatVoiceCtx; + char detail[96]; + lv_obj_t* state_label; + lv_obj_t* detail_label; + struct Device* stream_dev; + AudioStreamHandle input_handle; + AudioStreamHandle output_handle; + TaskHandle_t worker; + TaskHandle_t receiver; + SemaphoreHandle_t socket_lock; + SemaphoreHandle_t audio_lock; +} VoiceContext; -/* ─── Forward Declarations ─── */ -static void pipecatvoice_task(void* arg); -static void pipecatvoice_rx_task(void* arg); -static void update_ui(PipecatVoiceCtx* ctx); -static void load_config(PipecatVoiceCtx* ctx); +static void update_ui(VoiceContext* ctx) { + if (!ctx->visible || !tt_lvgl_lock(pdMS_TO_TICKS(100))) return; + lv_label_set_text(ctx->state_label, pv_state_label(ctx->state)); + lv_obj_set_style_text_color(ctx->state_label, + ctx->state == PV_STREAMING ? lv_color_hex(0x32c86e) : + ctx->state == PV_FAILED ? lv_color_hex(0xd94a4a) : lv_color_hex(0xe0b64a), LV_PART_MAIN); + lv_label_set_text(ctx->detail_label, ctx->detail); + tt_lvgl_unlock(); +} -/* ─── Helper for URL parsing ─── */ -static bool parse_ws_url(const char* url, char* host, int* port, char* path) { - if (strncmp(url, "ws://", 5) != 0) return false; - const char* p = url + 5; - const char* colon = strchr(p, ':'); - const char* slash = strchr(p, '/'); - - if (slash == NULL) { - strcpy(path, "/"); - } else { - strcpy(path, slash); - } - - if (colon != NULL && (slash == NULL || colon < slash)) { - int host_len = colon - p; - memcpy(host, p, host_len); - host[host_len] = '\0'; - *port = atoi(colon + 1); - } else { - int host_len = slash ? (slash - p) : strlen(p); - memcpy(host, p, host_len); - host[host_len] = '\0'; - *port = 80; +static void set_state(VoiceContext* ctx, PvState state, const char* detail) { + ctx->state = state; + snprintf(ctx->detail, sizeof(ctx->detail), "%s", detail); + update_ui(ctx); +} + +static bool open_input_stream(VoiceContext* ctx) { + if (ctx->stream_dev == NULL) return false; + if (ctx->output_handle) { audio_stream_close(ctx->output_handle); ctx->output_handle = NULL; } + if (ctx->input_handle) return true; + struct AudioStreamConfig cfg = { + .sample_rate = 16000, + .bits_per_sample = 16, + .channels = 1, + }; + if (audio_stream_open_input(ctx->stream_dev, &cfg, &ctx->input_handle) != ERROR_NONE) { + ESP_LOGW(TAG, "audio_stream_open_input failed"); + return false; } + audio_stream_set_mute(ctx->stream_dev, AUDIO_CODEC_DIR_INPUT, false); + audio_stream_set_volume(ctx->stream_dev, AUDIO_CODEC_DIR_INPUT, 100.0f); return true; } -/* ─── Config Loading/Saving ─── */ -static void load_config(PipecatVoiceCtx* ctx) { - // Default fallback config - snprintf(ctx->server_url, sizeof(ctx->server_url), "ws://192.168.68.102:8642/api/esp32/voice/ws"); - snprintf(ctx->device_id, sizeof(ctx->device_id), "pipecatvoice"); - snprintf(ctx->api_key, sizeof(ctx->api_key), "mcT1YA1vOr9wXSiHpCYalweEGGZKX-PIfZv2drp8BSg"); - - char path[256]; - size_t path_size = sizeof(path); - tt_app_get_user_data_child_path(ctx->app, "config.json", path, &path_size); - - FILE* file = fopen(path, "r"); - if (file == NULL) { - // Create user data directory if it doesn't exist - char dir_path[256]; - size_t dir_size = sizeof(dir_path); - tt_app_get_user_data_path(ctx->app, dir_path, &dir_size); - mkdir(dir_path, 0755); - - // Write default config file - file = fopen(path, "w"); - if (file != NULL) { - fprintf(file, "{\n \"server_url\": \"ws://192.168.68.102:8642/api/esp32/voice/ws\",\n \"device_id\": \"pipecatvoice\",\n \"api_key\": \"mcT1YA1vOr9wXSiHpCYalweEGGZKX-PIfZv2drp8BSg\"\n}\n"); - fclose(file); - } - ESP_LOGI(TAG, "Created default config.json at %s", path); - return; +static bool open_output_stream(VoiceContext* ctx) { + if (ctx->stream_dev == NULL) return false; + if (ctx->input_handle) { audio_stream_close(ctx->input_handle); ctx->input_handle = NULL; } + if (ctx->output_handle) return true; + struct AudioStreamConfig cfg = { + .sample_rate = 16000, + .bits_per_sample = 16, + .channels = 1, + }; + if (audio_stream_open_output(ctx->stream_dev, &cfg, &ctx->output_handle) != ERROR_NONE) { + ESP_LOGW(TAG, "audio_stream_open_output failed"); + return false; } - - fseek(file, 0, SEEK_END); - long size = ftell(file); - fseek(file, 0, SEEK_SET); - - if (size > 0 && size < 4096) { - char* buf = malloc(size + 1); - if (buf != NULL) { - size_t read_bytes = fread(buf, 1, size, file); - buf[read_bytes] = '\0'; - cJSON* json = cJSON_Parse(buf); - if (json != NULL) { - cJSON* url_item = cJSON_GetObjectItem(json, "server_url"); - if (url_item != NULL && url_item->valuestring != NULL) { - strncpy(ctx->server_url, url_item->valuestring, sizeof(ctx->server_url) - 1); - } - cJSON* dev_item = cJSON_GetObjectItem(json, "device_id"); - if (dev_item != NULL && dev_item->valuestring != NULL) { - strncpy(ctx->device_id, dev_item->valuestring, sizeof(ctx->device_id) - 1); - } - cJSON* key_item = cJSON_GetObjectItem(json, "api_key"); - if (key_item != NULL && key_item->valuestring != NULL) { - strncpy(ctx->api_key, key_item->valuestring, sizeof(ctx->api_key) - 1); - } - cJSON_Delete(json); - } - free(buf); - } - } - fclose(file); - ESP_LOGI(TAG, "Loaded config: server=%s device=%s", ctx->server_url, ctx->device_id); + audio_stream_set_mute(ctx->stream_dev, AUDIO_CODEC_DIR_OUTPUT, false); + audio_stream_set_volume(ctx->stream_dev, AUDIO_CODEC_DIR_OUTPUT, 80.0f); + return true; } -/* ─── JSON Message Processing ─── */ -static void parse_json_message(PipecatVoiceCtx* ctx, const char* json_str) { - cJSON* json = cJSON_Parse(json_str); - if (json == NULL) return; - - cJSON* evt_item = cJSON_GetObjectItem(json, "event"); - if (evt_item != NULL && evt_item->valuestring != NULL) { - const char* evt = evt_item->valuestring; - ESP_LOGI(TAG, "WS Event: %s", evt); - - if (strcmp(evt, "ready") == 0) { - ESP_LOGI(TAG, "Server ready"); - } else if (strcmp(evt, "listening") == 0) { - ctx->state = STATE_LISTENING; - ctx->ui_update_pending = true; - } else if (strcmp(evt, "transcript") == 0) { - cJSON* txt_item = cJSON_GetObjectItem(json, "text"); - if (txt_item != NULL && txt_item->valuestring != NULL) { - strncpy(ctx->last_transcript, txt_item->valuestring, sizeof(ctx->last_transcript) - 1); - ctx->ui_update_pending = true; - } - } else if (strcmp(evt, "thinking") == 0) { - ctx->state = STATE_THINKING; - ctx->ui_update_pending = true; - } else if (strcmp(evt, "response_text") == 0) { - cJSON* txt_item = cJSON_GetObjectItem(json, "text"); - if (txt_item != NULL && txt_item->valuestring != NULL) { - strncpy(ctx->last_response, txt_item->valuestring, sizeof(ctx->last_response) - 1); - ctx->ui_update_pending = true; - } - } else if (strcmp(evt, "audio_start") == 0) { - ctx->state = STATE_SPEAKING; - ctx->ui_update_pending = true; - - // I2S is already configured globally at session startup - } else if (strcmp(evt, "audio_end") == 0) { - ESP_LOGI(TAG, "Audio response ended"); - } else if (strcmp(evt, "done") == 0) { - ctx->ws_done = true; - } else if (strcmp(evt, "error") == 0) { - cJSON* msg_item = cJSON_GetObjectItem(json, "message"); - if (msg_item != NULL && msg_item->valuestring != NULL) { - strncpy(ctx->error_message, msg_item->valuestring, sizeof(ctx->error_message) - 1); - } else { - snprintf(ctx->error_message, sizeof(ctx->error_message), "Unknown server error"); - } - ctx->state = STATE_ERROR; - ctx->ui_update_pending = true; - } - } - cJSON_Delete(json); +static void close_audio(VoiceContext* ctx) { + if (ctx->output_handle) { audio_stream_close(ctx->output_handle); ctx->output_handle = NULL; } + if (ctx->input_handle) { audio_stream_close(ctx->input_handle); ctx->input_handle = NULL; } } -/* ─── WebSocket RX (Receive) Task ─── */ -static void pipecatvoice_rx_task(void* arg) { - PipecatVoiceCtx* ctx = (PipecatVoiceCtx*)arg; - uint8_t* rx_buf = malloc(8192); - if (rx_buf == NULL) { - ESP_LOGE(TAG, "Failed to allocate RX buffer"); - ctx->ws_done = true; - ctx->rx_task = NULL; - vTaskDelete(NULL); +static int send_locked(VoiceContext* ctx, const uint8_t* data, size_t length, bool binary) { + if (ctx->fd < 0 || xSemaphoreTake(ctx->socket_lock, pdMS_TO_TICKS(500)) != pdTRUE) return -1; + int result = ws_send(ctx->fd, data, length, binary); + xSemaphoreGive(ctx->socket_lock); + return result; +} + +static void handle_event(VoiceContext* ctx, const char* text) { + cJSON* root = cJSON_Parse(text); + if (root == NULL) return; + cJSON* version = cJSON_GetObjectItem(root, "v"); + cJSON* event = cJSON_GetObjectItem(root, "event"); + if (!cJSON_IsNumber(version) || version->valueint != PV_PROTOCOL_VERSION || !cJSON_IsString(event)) { + cJSON_Delete(root); return; } - int opcode; - - ESP_LOGI(TAG, "WS Receive task started"); - - while (ctx->ws_fd >= 0) { - int r = ws_recv(ctx->ws_fd, &opcode, rx_buf, 8191); - if (r < 0) { - if (r == -1) { - ESP_LOGI(TAG, "WS connection closed or read error, errno=%d", errno); - } else { - ESP_LOGE(TAG, "WS rx buffer overflow"); + if (strcmp(event->valuestring, "ready") == 0) { + ctx->streaming = true; + } else if (strcmp(event->valuestring, "state") == 0) { + cJSON* state = cJSON_GetObjectItem(root, "state"); + if (cJSON_IsString(state) && strcmp(state->valuestring, "listening") == 0) { + ctx->streaming = true; + set_state(ctx, PV_STREAMING, "Continuous microphone streaming"); + } + } else if (strcmp(event->valuestring, "audio") == 0) { + cJSON* format = cJSON_GetObjectItem(root, "format"); + cJSON* rate = cJSON_GetObjectItem(root, "sample_rate"); + cJSON* channels = cJSON_GetObjectItem(root, "channels"); + cJSON* width = cJSON_GetObjectItem(root, "sample_width"); + cJSON* length = cJSON_GetObjectItem(root, "byte_length"); + if (cJSON_IsString(format) && cJSON_IsNumber(rate) && cJSON_IsNumber(channels) && cJSON_IsNumber(width) && + cJSON_IsNumber(length) && pv_valid_downstream_audio(format->valuestring, rate->valueint, channels->valueint, + width->valueint, (size_t)length->valueint)) { + ctx->expected_audio_bytes = (size_t)length->valueint; + ctx->playing = true; + set_state(ctx, PV_STREAMING, "Playing response; microphone paused"); + } else { + ctx->socket_failed = true; + } + } else if (strcmp(event->valuestring, "error") == 0) { + /* The server-provided message is intentionally not copied to display/logs. */ + ctx->socket_failed = true; + } + cJSON_Delete(root); +} + +static void receiver_task(void* argument) { + VoiceContext* ctx = argument; + uint8_t* buffer = malloc(PV_DOWNSTREAM_MAX + 1U); + if (buffer == NULL) { ctx->socket_failed = true; ctx->receiver = NULL; vTaskDelete(NULL); } + while (ctx->visible && ctx->fd >= 0) { + int opcode = 0; + bool final = false; + int received = ws_recv(ctx->fd, &opcode, &final, buffer, PV_DOWNSTREAM_MAX); + if (received < 0 || !final) { ctx->socket_failed = true; break; } + if (opcode == 0x01) { + buffer[received] = '\0'; + handle_event(ctx, (const char*)buffer); + } else if (opcode == 0x02) { + if (!ctx->playing || !pv_binary_matches_metadata(ctx->expected_audio_bytes, (size_t)received)) { ctx->socket_failed = true; break; } + xSemaphoreTake(ctx->audio_lock, portMAX_DELAY); + bool ok = open_output_stream(ctx); + size_t written = 0; + if (ok) ok = audio_stream_write(ctx->output_handle, buffer, (size_t)received, &written, pdMS_TO_TICKS(3000)) == ERROR_NONE; + open_input_stream(ctx); + xSemaphoreGive(ctx->audio_lock); + if (!ok || written != (size_t)received) { ctx->socket_failed = true; break; } + ctx->expected_audio_bytes = 0; + ctx->playing = false; + } else if (opcode == 0x09) { + if (xSemaphoreTake(ctx->socket_lock, pdMS_TO_TICKS(500)) == pdTRUE) { + ws_send_pong(ctx->fd, buffer, (size_t)received); + xSemaphoreGive(ctx->socket_lock); } - ctx->ws_done = true; + } else if (opcode == 0x08) { + ctx->socket_failed = true; break; } - - if (opcode == 0x01) { // Text frame (JSON) - rx_buf[r] = '\0'; - parse_json_message(ctx, (char*)rx_buf); - } else if (opcode == 0x02) { // Binary frame (Audio data) - if (ctx->state == STATE_SPEAKING && ctx->i2s_dev != NULL) { - const uint8_t* payload_ptr = rx_buf; - size_t payload_len = r; - - // Skip WAV header if present in the first chunk - if (payload_len > 44 && memcmp(payload_ptr, "RIFF", 4) == 0) { - payload_ptr += 44; - payload_len -= 44; - } - - size_t written = 0; - device_lock(ctx->i2s_dev); - i2s_controller_write(ctx->i2s_dev, payload_ptr, payload_len, &written, pdMS_TO_TICKS(100)); - device_unlock(ctx->i2s_dev); - } - } else if (opcode == 0x09) { // PING frame - ESP_LOGI(TAG, "WS PING received, sending PONG"); - ws_send_pong(ctx->ws_fd, rx_buf, r); - } } - - free(rx_buf); - ctx->rx_task = NULL; + free(buffer); + ctx->receiver = NULL; vTaskDelete(NULL); } -/* ─── UI Speaker Selection Event Callback ─── */ -static void speaker_event_cb(lv_event_t* e) { - PipecatVoiceCtx* ctx = (PipecatVoiceCtx*)lv_event_get_user_data(e); - lv_obj_t* target = lv_event_get_target(e); - lv_event_code_t code = lv_event_get_code(e); - - if (code == LV_EVENT_CLICKED && !ctx->speaker_selected) { - if (target == ctx->btn_grace) { - strcpy(ctx->device_id, "grace-pipecat"); - } else if (target == ctx->btn_elias) { - strcpy(ctx->device_id, "elias-pipecat"); +static void close_session(VoiceContext* ctx) { + int fd = ctx->fd; + ctx->fd = -1; + ctx->streaming = false; + ctx->playing = false; + ctx->expected_audio_bytes = 0; + if (fd >= 0) { + if (xSemaphoreTake(ctx->socket_lock, pdMS_TO_TICKS(100)) == pdTRUE) { + ws_send_close(fd); + xSemaphoreGive(ctx->socket_lock); } - - ctx->speaker_selected = true; - - // Hide selection screen, show main screen - lv_obj_add_flag(ctx->speaker_select_cont, LV_OBJ_FLAG_HIDDEN); - lv_obj_remove_flag(ctx->app_main_cont, LV_OBJ_FLAG_HIDDEN); - - // Trigger UI update to refresh status and PTT button - ctx->ui_update_pending = true; + ws_close(fd); } + close_audio(ctx); } -/* ─── UI Press-to-Talk Event Callback ─── */ -static void ptt_event_cb(lv_event_t* e) { - PipecatVoiceCtx* ctx = (PipecatVoiceCtx*)lv_event_get_user_data(e); - lv_event_code_t code = lv_event_get_code(e); - - if (code == LV_EVENT_PRESSED) { - ctx->is_pressed = true; - if (ctx->state == STATE_IDLE || ctx->state == ERROR_NONE) { - ctx->start_session = true; - } - } else if (code == LV_EVENT_RELEASED || code == LV_EVENT_PRESS_LOST) { - ctx->is_pressed = false; - if (ctx->state == STATE_LISTENING) { - ctx->stop_session = true; - } else if (ctx->state == STATE_CONNECTING) { - ctx->cancel_session = true; - } +static void worker_task(void* argument) { + VoiceContext* ctx = argument; + PvEndpoint endpoint; + if (!pv_parse_endpoint(ctx->endpoint, &endpoint)) { + set_state(ctx, PV_FAILED, "Set a private-LAN ws:// endpoint in config.json"); + ctx->worker = NULL; + vTaskDelete(NULL); } -} - -/* ─── Core UI Update Function ─── */ -static void update_ui(PipecatVoiceCtx* ctx) { - if (!ctx->visible) return; - - if (tt_lvgl_lock(pdMS_TO_TICKS(500))) { - if (!ctx->visible) { - tt_lvgl_unlock(); - return; - } - - if (!ctx->speaker_selected) { - lv_label_set_text(ctx->lbl_status, "Choose Speaker"); - lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0x9E9E9E), LV_PART_MAIN); - tt_lvgl_unlock(); - return; - } - - switch (ctx->state) { - case STATE_IDLE: - lv_label_set_text(ctx->lbl_status, "Ready"); - lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0x9E9E9E), LV_PART_MAIN); - lv_label_set_text(ctx->btn_ptt_label, LV_SYMBOL_AUDIO " Hold to Talk"); - lv_obj_set_style_bg_color(ctx->btn_ptt, lv_color_hex(0x6200EE), LV_PART_MAIN); - lv_obj_remove_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN); - break; - case STATE_CONNECTING: - lv_label_set_text(ctx->lbl_status, "Connecting..."); - lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0xFFC107), LV_PART_MAIN); - lv_label_set_text(ctx->btn_ptt_label, "Connecting..."); - lv_obj_set_style_bg_color(ctx->btn_ptt, lv_color_hex(0x757575), LV_PART_MAIN); - lv_obj_remove_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN); - break; - case STATE_LISTENING: - lv_label_set_text(ctx->lbl_status, "Listening..."); - lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0x00E676), LV_PART_MAIN); - lv_label_set_text(ctx->btn_ptt_label, "Release to Stop"); - lv_obj_set_style_bg_color(ctx->btn_ptt, lv_color_hex(0xD50000), LV_PART_MAIN); - lv_obj_remove_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN); - break; - case STATE_THINKING: - lv_label_set_text(ctx->lbl_status, "Thinking..."); - lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0x00B0FF), LV_PART_MAIN); - lv_obj_add_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN); - break; - case STATE_SPEAKING: - lv_label_set_text(ctx->lbl_status, "Speaking..."); - lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0xAA00FF), LV_PART_MAIN); - lv_obj_add_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN); - break; - case STATE_ERROR: - lv_label_set_text(ctx->lbl_status, "Error"); - lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0xFF1744), LV_PART_MAIN); - lv_label_set_text(ctx->btn_ptt_label, LV_SYMBOL_REFRESH " Try Again"); - lv_obj_set_style_bg_color(ctx->btn_ptt, lv_color_hex(0xFF1744), LV_PART_MAIN); - lv_obj_remove_flag(ctx->btn_ptt, LV_OBJ_FLAG_HIDDEN); - - // Show the full error message in the bot's response bubble - if (ctx->error_message[0] != '\0') { - snprintf(ctx->last_response, sizeof(ctx->last_response), "Error: %s", ctx->error_message); - } - break; - } - - lv_label_set_text(ctx->lbl_transcript, ctx->last_transcript[0] != '\0' ? ctx->last_transcript : "(Your question will appear here)"); - lv_label_set_text(ctx->lbl_response, ctx->last_response[0] != '\0' ? ctx->last_response : "(Answer will appear here)"); - - tt_lvgl_unlock(); - } -} - -/* ─── Streaming & Main Worker Task ─── */ -static void pipecatvoice_task(void* arg) { - PipecatVoiceCtx* ctx = (PipecatVoiceCtx*)arg; - char host[64]; - char path[128]; - int port = 80; - - ctx->ws_fd = -1; - - + uint8_t pcm[1024]; while (ctx->visible) { - if (ctx->start_session) { - ctx->start_session = false; - ctx->stop_session = false; - ctx->cancel_session = false; - - ctx->state = STATE_CONNECTING; - ctx->last_transcript[0] = '\0'; - ctx->last_response[0] = '\0'; - ctx->error_message[0] = '\0'; - ctx->ws_done = false; - update_ui(ctx); - - if (!parse_ws_url(ctx->server_url, host, &port, path)) { - ctx->state = STATE_ERROR; - snprintf(ctx->error_message, sizeof(ctx->error_message), "Invalid Server URL"); - update_ui(ctx); - continue; - } - - ESP_LOGI(TAG, "Connecting: host=%s port=%d path=%s", host, port, path); - int fd = ws_connect(host, port, path, ctx->device_id, ctx->api_key); - if (fd < 0) { - ctx->state = STATE_ERROR; - snprintf(ctx->error_message, sizeof(ctx->error_message), "Connect failed"); - update_ui(ctx); - continue; - } - - ctx->ws_fd = fd; - ctx->ws_connected = true; - - // Configure I2S on demand for this session (16 kHz, 16-bit, mono) - struct I2sConfig session_cfg = { - .communication_format = I2S_FORMAT_STAND_I2S, - .sample_rate = 16000, - .bits_per_sample = 16, - .channel_left = 0, - .channel_right = I2S_CHANNEL_NONE - }; - if (ctx->i2s_dev != NULL) { - device_lock(ctx->i2s_dev); - i2s_controller_set_config(ctx->i2s_dev, &session_cfg); - device_unlock(ctx->i2s_dev); - } - - // Spawn background RX task to read and parse events - xTaskCreate(pipecatvoice_rx_task, "pipecat_rx", 4096, ctx, 6, &ctx->rx_task); - - // Send start event handshake - char start_json[256]; - snprintf(start_json, sizeof(start_json), - "{\"event\":\"start\",\"device_id\":\"%s\",\"sample_rate\":16000,\"channels\":1,\"sample_width\":2,\"format\":\"pcm_s16le\"}", - ctx->device_id); - if (ws_send(ctx->ws_fd, (const uint8_t*)start_json, strlen(start_json), false) < 0) { - ctx->state = STATE_ERROR; - snprintf(ctx->error_message, sizeof(ctx->error_message), "Handshake send failed"); - ws_close(ctx->ws_fd); - ctx->ws_fd = -1; - ctx->ws_connected = false; - update_ui(ctx); - continue; - } - - // Wait for server to switch us to LISTENING - int timeout_ms = 4000; - while (ctx->state != STATE_LISTENING && timeout_ms > 0 && !ctx->cancel_session && !ctx->ws_done) { - vTaskDelay(pdMS_TO_TICKS(50)); - timeout_ms -= 50; - } - - if (ctx->state != STATE_LISTENING) { - if (ctx->state != STATE_ERROR) { - ctx->state = STATE_ERROR; - snprintf(ctx->error_message, sizeof(ctx->error_message), "Handshake timeout"); - } - ws_close(ctx->ws_fd); - ctx->ws_fd = -1; - ctx->ws_connected = false; - update_ui(ctx); - continue; - } - - // I2S is already configured globally at session startup - - uint8_t* buffer = malloc(1024); - if (buffer == NULL) { - ESP_LOGE(TAG, "Failed to allocate record buffer"); - ctx->state = STATE_ERROR; - snprintf(ctx->error_message, sizeof(ctx->error_message), "Out of memory"); - ws_close(ctx->ws_fd); - ctx->ws_fd = -1; - ctx->ws_connected = false; - update_ui(ctx); - continue; - } - size_t total_sent_bytes = 0; - - // Stream audio loop while PTT is held - while (ctx->is_pressed && !ctx->stop_session && !ctx->cancel_session && !ctx->ws_done && total_sent_bytes < 320000) { - size_t bytes_read = 0; - error_t r = i2s_controller_read(ctx->i2s_dev, buffer, 1024, &bytes_read, pdMS_TO_TICKS(100)); - if (r == ERROR_NONE && bytes_read > 0) { - if (ws_send(ctx->ws_fd, buffer, bytes_read, true) < 0) { - ESP_LOGE(TAG, "Audio stream send failed"); - break; - } - total_sent_bytes += bytes_read; - } - } - - free(buffer); - - if (ctx->cancel_session || total_sent_bytes < 3200) { - ESP_LOGI(TAG, "Cancelling audio session"); - const char* cancel_json = "{\"event\":\"cancel\"}"; - ws_send(ctx->ws_fd, (const uint8_t*)cancel_json, strlen(cancel_json), false); - ctx->state = STATE_IDLE; - } else { - const char* stop_json = "{\"event\":\"stop\"}"; - ws_send(ctx->ws_fd, (const uint8_t*)stop_json, strlen(stop_json), false); - ctx->state = STATE_THINKING; - update_ui(ctx); - - // Wait for response to finish (increase to 90s to match socket timeout) - int wait_timeout_ms = 90000; - while (!ctx->ws_done && ctx->state != STATE_ERROR && wait_timeout_ms > 0) { - vTaskDelay(pdMS_TO_TICKS(100)); - wait_timeout_ms -= 100; - if (ctx->ui_update_pending) { - ctx->ui_update_pending = false; - update_ui(ctx); + set_state(ctx, ctx->retry_attempt ? PV_RECONNECTING : PV_CONNECTING, + ctx->retry_attempt ? "Retrying voice gateway" : "Connecting to voice gateway"); + ctx->socket_failed = false; + ESP_LOGI(TAG, "Opening voice gateway session"); + ctx->fd = ws_connect(endpoint.host, endpoint.port, endpoint.path, ctx->device_id, ctx->api_key); + if (ctx->fd >= 0) { + char session[64]; + snprintf(session, sizeof(session), "pv-%08lx", (unsigned long)esp_random()); + char start[256]; + if (pv_make_start_json(start, sizeof(start), session, ctx->device_id) && + send_locked(ctx, (const uint8_t*)start, strlen(start), false) == 0) { + if (!open_input_stream(ctx)) { + set_state(ctx, PV_FAILED, "Audio stream unavailable"); + ctx->socket_failed = true; + } else { + xTaskCreate(receiver_task, "pv_rx", 6144, ctx, 6, &ctx->receiver); + uint32_t stable_ticks = 0; + while (ctx->visible && !ctx->socket_failed) { + if (!ctx->streaming || ctx->playing) { vTaskDelay(pdMS_TO_TICKS(20)); continue; } + xSemaphoreTake(ctx->audio_lock, portMAX_DELAY); + bool opened = open_input_stream(ctx); + size_t read = 0; + error_t read_result = opened ? audio_stream_read(ctx->input_handle, pcm, sizeof(pcm), &read, pdMS_TO_TICKS(100)) : ERROR_RESOURCE; + xSemaphoreGive(ctx->audio_lock); + if (!opened) { ctx->socket_failed = true; break; } + if (read_result == ERROR_NONE && pv_valid_pcm_chunk(read) && send_locked(ctx, pcm, read, true) < 0) ctx->socket_failed = true; + if (++stable_ticks >= 300) ctx->retry_attempt = 0; } } - - if (!ctx->ws_done && ctx->state != STATE_ERROR) { - ctx->state = STATE_ERROR; - snprintf(ctx->error_message, sizeof(ctx->error_message), "Response timeout"); - } } - - // Clean up socket - int fd_to_close = ctx->ws_fd; - ctx->ws_fd = -1; - ctx->ws_connected = false; - ws_close(fd_to_close); - - // Reset I2S controller to release DMA and stop white noise - if (ctx->i2s_dev != NULL) { - device_lock(ctx->i2s_dev); - i2s_controller_reset(ctx->i2s_dev); - device_unlock(ctx->i2s_dev); - } - - // Wait for receive task to exit - int rx_timeout = 100; - while (ctx->rx_task != NULL && rx_timeout > 0) { - vTaskDelay(pdMS_TO_TICKS(10)); - rx_timeout--; - } - - if (ctx->state != STATE_ERROR) { - ctx->state = STATE_IDLE; - } - update_ui(ctx); - } - - vTaskDelay(pdMS_TO_TICKS(100)); - if (ctx->ui_update_pending) { - ctx->ui_update_pending = false; - update_ui(ctx); } + close_session(ctx); + if (!ctx->visible) break; + uint32_t delay = pv_retry_delay_seconds(ctx->retry_attempt++); + ESP_LOGW(TAG, "Voice gateway session unavailable; retry in %lu seconds", (unsigned long)delay); + set_state(ctx, PV_RECONNECTING, "Gateway unavailable; retry scheduled"); + for (uint32_t second = 0; ctx->visible && second < delay; ++second) vTaskDelay(pdMS_TO_TICKS(1000)); } - - ctx->worker_task = NULL; + ctx->worker = NULL; vTaskDelete(NULL); } -/* ─── App Lifecycle ─── */ - -static void* create_data(void) { - PipecatVoiceCtx* ctx = calloc(1, sizeof(PipecatVoiceCtx)); - if (ctx != NULL) { - ctx->state = STATE_IDLE; - ctx->ws_fd = -1; - } - return ctx; +static void load_config(VoiceContext* ctx) { + snprintf(ctx->endpoint, sizeof(ctx->endpoint), "%s", DEFAULT_ENDPOINT); + snprintf(ctx->device_id, sizeof(ctx->device_id), "%s", DEFAULT_DEVICE_ID); + ctx->api_key[0] = '\0'; + char path[256]; size_t path_size = sizeof(path); + tt_app_get_user_data_child_path(ctx->app, "config.json", path, &path_size); + FILE* file = fopen(path, "r"); + if (file == NULL) return; + char json[512]; size_t bytes = fread(json, 1, sizeof(json) - 1, file); fclose(file); json[bytes] = '\0'; + cJSON* root = cJSON_Parse(json); + cJSON* endpoint = root ? cJSON_GetObjectItem(root, "server_url") : NULL; + cJSON* device = root ? cJSON_GetObjectItem(root, "device_id") : NULL; + cJSON* key = root ? cJSON_GetObjectItem(root, "api_key") : NULL; + if (cJSON_IsString(endpoint)) snprintf(ctx->endpoint, sizeof(ctx->endpoint), "%s", endpoint->valuestring); + if (cJSON_IsString(device)) snprintf(ctx->device_id, sizeof(ctx->device_id), "%s", device->valuestring); + if (cJSON_IsString(key)) snprintf(ctx->api_key, sizeof(ctx->api_key), "%s", key->valuestring); + cJSON_Delete(root); } -static void destroy_data(void* data) { - free(data); -} - -static void on_create(AppHandle app, void* data) { - PipecatVoiceCtx* ctx = (PipecatVoiceCtx*)data; - ctx->app = app; -} - -static void on_destroy(AppHandle app, void* data) { - // No-op -} +static void* create_data(void) { VoiceContext* ctx = calloc(1, sizeof(*ctx)); if (ctx) ctx->fd = -1; return ctx; } +static void destroy_data(void* data) { free(data); } +static void on_create(AppHandle app, void* data) { ((VoiceContext*)data)->app = app; } +static void on_destroy(AppHandle app, void* data) { (void)app; (void)data; } static void on_show(AppHandle app, void* data, lv_obj_t* parent) { - PipecatVoiceCtx* ctx = (PipecatVoiceCtx*)data; - ctx->visible = true; - - load_config(ctx); - - // Find I2S controller - ctx->i2s_dev = device_find_by_name("i2s0"); - if (ctx->i2s_dev == NULL) { - ESP_LOGE(TAG, "I2S controller 'i2s0' not found!"); - } - - // Style the parent screen - lv_obj_set_flex_flow(parent, LV_FLEX_FLOW_COLUMN); - lv_obj_set_flex_align(parent, LV_FLEX_ALIGN_START, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER); - lv_obj_set_style_pad_all(parent, 10, LV_PART_MAIN); - lv_obj_set_style_pad_row(parent, 12, LV_PART_MAIN); - lv_obj_set_style_bg_color(parent, lv_color_hex(0x0C0B12), LV_PART_MAIN); // Rich dark background - - // Add App Toolbar + VoiceContext* ctx = data; ctx->visible = true; load_config(ctx); + ctx->stream_dev = device_find_by_name("audio-stream"); + if (ctx->stream_dev == NULL) ctx->stream_dev = device_find_first_by_type(&AUDIO_STREAM_TYPE); + ctx->socket_lock = xSemaphoreCreateMutex(); + ctx->audio_lock = xSemaphoreCreateMutex(); lv_obj_t* toolbar = tt_lvgl_toolbar_create_for_app(parent, app); lv_obj_align(toolbar, LV_ALIGN_TOP_MID, 0, 0); - - // Status Label inside toolbar (replaces the static "ReynaBot Voice" label) - ctx->lbl_status = lv_label_create(toolbar); - lv_label_set_text(ctx->lbl_status, "Ready"); - lv_obj_add_flag(ctx->lbl_status, LV_OBJ_FLAG_FLOATING); - lv_obj_align(ctx->lbl_status, LV_ALIGN_RIGHT_MID, -10, 0); - lv_obj_set_style_text_color(ctx->lbl_status, lv_color_hex(0x9E9E9E), LV_PART_MAIN); + ctx->state_label = lv_label_create(parent); - // Initial state - ctx->speaker_selected = false; - - // Default to Grace if device_id is not already configured to elias-esp32 - if (strcmp(ctx->device_id, "elias-pipecat") != 0) { - strcpy(ctx->device_id, "grace-pipecat"); - } - - // 1. Speaker Selection Container (Startup Screen) - ctx->speaker_select_cont = lv_obj_create(parent); - lv_obj_set_width(ctx->speaker_select_cont, lv_pct(95)); - lv_obj_set_flex_grow(ctx->speaker_select_cont, 1); - lv_obj_set_flex_flow(ctx->speaker_select_cont, LV_FLEX_FLOW_COLUMN); - lv_obj_set_flex_align(ctx->speaker_select_cont, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER); - lv_obj_set_style_pad_all(ctx->speaker_select_cont, 0, LV_PART_MAIN); - lv_obj_set_style_bg_opa(ctx->speaker_select_cont, 0, LV_PART_MAIN); - lv_obj_set_style_border_width(ctx->speaker_select_cont, 0, LV_PART_MAIN); - lv_obj_set_style_pad_row(ctx->speaker_select_cont, 16, LV_PART_MAIN); - - // Title Prompt - lv_obj_t* prompt_lbl = lv_label_create(ctx->speaker_select_cont); - lv_label_set_text(prompt_lbl, "Who is speaking?"); - lv_obj_set_style_text_color(prompt_lbl, lv_color_hex(0xFFFFFF), LV_PART_MAIN); - - // Row of two selection buttons - lv_obj_t* speaker_row = lv_obj_create(ctx->speaker_select_cont); - lv_obj_set_size(speaker_row, lv_pct(100), 140); - lv_obj_set_flex_flow(speaker_row, LV_FLEX_FLOW_ROW); - lv_obj_set_flex_align(speaker_row, LV_FLEX_ALIGN_SPACE_BETWEEN, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER); - lv_obj_set_style_pad_all(speaker_row, 0, LV_PART_MAIN); - lv_obj_set_style_bg_opa(speaker_row, 0, LV_PART_MAIN); - lv_obj_set_style_border_width(speaker_row, 0, LV_PART_MAIN); - - // Grace Button (Girl) - ctx->btn_grace = lv_btn_create(speaker_row); - lv_obj_set_size(ctx->btn_grace, lv_pct(46), 130); - lv_obj_set_flex_flow(ctx->btn_grace, LV_FLEX_FLOW_COLUMN); - lv_obj_set_flex_align(ctx->btn_grace, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER); - lv_obj_set_style_pad_all(ctx->btn_grace, 8, LV_PART_MAIN); - lv_obj_set_style_pad_row(ctx->btn_grace, 8, LV_PART_MAIN); - lv_obj_set_style_radius(ctx->btn_grace, 12, LV_PART_MAIN); - lv_obj_set_style_bg_color(ctx->btn_grace, lv_color_hex(0x1C1B22), LV_PART_MAIN); - lv_obj_set_style_border_width(ctx->btn_grace, 1, LV_PART_MAIN); - lv_obj_set_style_border_color(ctx->btn_grace, lv_color_hex(0x2D2B36), LV_PART_MAIN); - lv_obj_add_event_cb(ctx->btn_grace, speaker_event_cb, LV_EVENT_CLICKED, ctx); - - lv_obj_t* icon_grace = lv_image_create(ctx->btn_grace); - lv_obj_set_size(icon_grace, 80, 80); - char path_grace[256] = "A:"; - size_t sz_grace = sizeof(path_grace) - 2; - tt_app_get_assets_child_path(ctx->app, "girl.png", path_grace + 2, &sz_grace); - lv_image_set_src(icon_grace, path_grace); - - lv_obj_t* lbl_grace = lv_label_create(ctx->btn_grace); - lv_label_set_text(lbl_grace, "Grace"); - lv_obj_set_style_text_color(lbl_grace, lv_color_hex(0xFFFFFF), LV_PART_MAIN); - - // Elias Button (Boy) - ctx->btn_elias = lv_btn_create(speaker_row); - lv_obj_set_size(ctx->btn_elias, lv_pct(46), 130); - lv_obj_set_flex_flow(ctx->btn_elias, LV_FLEX_FLOW_COLUMN); - lv_obj_set_flex_align(ctx->btn_elias, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER); - lv_obj_set_style_pad_all(ctx->btn_elias, 8, LV_PART_MAIN); - lv_obj_set_style_pad_row(ctx->btn_elias, 8, LV_PART_MAIN); - lv_obj_set_style_radius(ctx->btn_elias, 12, LV_PART_MAIN); - lv_obj_set_style_bg_color(ctx->btn_elias, lv_color_hex(0x1C1B22), LV_PART_MAIN); - lv_obj_set_style_border_width(ctx->btn_elias, 1, LV_PART_MAIN); - lv_obj_set_style_border_color(ctx->btn_elias, lv_color_hex(0x2D2B36), LV_PART_MAIN); - lv_obj_add_event_cb(ctx->btn_elias, speaker_event_cb, LV_EVENT_CLICKED, ctx); - - lv_obj_t* icon_elias = lv_image_create(ctx->btn_elias); - lv_obj_set_size(icon_elias, 80, 80); - char path_elias[256] = "A:"; - size_t sz_elias = sizeof(path_elias) - 2; - tt_app_get_assets_child_path(ctx->app, "boy.png", path_elias + 2, &sz_elias); - lv_image_set_src(icon_elias, path_elias); - - lv_obj_t* lbl_elias = lv_label_create(ctx->btn_elias); - lv_label_set_text(lbl_elias, "Elias"); - lv_obj_set_style_text_color(lbl_elias, lv_color_hex(0xFFFFFF), LV_PART_MAIN); - - // 2. Main App Container (Usual App Screen) - ctx->app_main_cont = lv_obj_create(parent); - lv_obj_set_width(ctx->app_main_cont, lv_pct(95)); - lv_obj_set_flex_grow(ctx->app_main_cont, 1); - lv_obj_set_flex_flow(ctx->app_main_cont, LV_FLEX_FLOW_COLUMN); - lv_obj_set_flex_align(ctx->app_main_cont, LV_FLEX_ALIGN_START, LV_FLEX_ALIGN_CENTER, LV_FLEX_ALIGN_CENTER); - lv_obj_set_style_pad_all(ctx->app_main_cont, 0, LV_PART_MAIN); - lv_obj_set_style_pad_row(ctx->app_main_cont, 12, LV_PART_MAIN); - lv_obj_set_style_bg_opa(ctx->app_main_cont, 0, LV_PART_MAIN); - lv_obj_set_style_border_width(ctx->app_main_cont, 0, LV_PART_MAIN); - lv_obj_add_flag(ctx->app_main_cont, LV_OBJ_FLAG_HIDDEN); // Hidden by default - - // Conversation bubble container - lv_obj_t* conv_card = lv_obj_create(ctx->app_main_cont); - lv_obj_set_width(conv_card, lv_pct(100)); - lv_obj_set_flex_grow(conv_card, 1); - lv_obj_set_flex_flow(conv_card, LV_FLEX_FLOW_COLUMN); - lv_obj_set_flex_align(conv_card, LV_FLEX_ALIGN_START, LV_FLEX_ALIGN_START, LV_FLEX_ALIGN_START); - lv_obj_set_style_pad_all(conv_card, 12, LV_PART_MAIN); - lv_obj_set_style_pad_row(conv_card, 10, LV_PART_MAIN); - lv_obj_set_style_bg_color(conv_card, lv_color_hex(0x13121A), LV_PART_MAIN); - lv_obj_set_style_border_color(conv_card, lv_color_hex(0x23212C), LV_PART_MAIN); - lv_obj_set_style_border_width(conv_card, 1, LV_PART_MAIN); - lv_obj_set_style_radius(conv_card, 12, LV_PART_MAIN); - - // User bubble - lv_obj_t* user_label_title = lv_label_create(conv_card); - lv_label_set_text(user_label_title, "👤 You:"); - lv_obj_set_style_text_color(user_label_title, lv_color_hex(0x00E676), LV_PART_MAIN); - - ctx->lbl_transcript = lv_label_create(conv_card); - lv_label_set_text(ctx->lbl_transcript, "(Your question will appear here)"); - lv_label_set_long_mode(ctx->lbl_transcript, LV_LABEL_LONG_WRAP); - lv_obj_set_width(ctx->lbl_transcript, lv_pct(100)); - lv_obj_set_style_text_color(ctx->lbl_transcript, lv_color_hex(0xE0E0E0), LV_PART_MAIN); - - // Separator line - lv_obj_t* sep = lv_line_create(conv_card); - static lv_point_precise_t line_points[] = { {0, 0}, {300, 0} }; - lv_line_set_points(sep, line_points, 2); - lv_obj_set_style_line_color(sep, lv_color_hex(0x2D2B36), LV_PART_MAIN); - lv_obj_set_style_line_width(sep, 1, LV_PART_MAIN); - lv_obj_set_width(sep, lv_pct(100)); - - // Assistant bubble - lv_obj_t* bot_label_title = lv_label_create(conv_card); - lv_label_set_text(bot_label_title, "🤖 Pipecat:"); - lv_obj_set_style_text_color(bot_label_title, lv_color_hex(0xAA00FF), LV_PART_MAIN); - - ctx->lbl_response = lv_label_create(conv_card); - lv_label_set_text(ctx->lbl_response, "(Answer will appear here)"); - lv_label_set_long_mode(ctx->lbl_response, LV_LABEL_LONG_WRAP); - lv_obj_set_width(ctx->lbl_response, lv_pct(100)); - lv_obj_set_style_text_color(ctx->lbl_response, lv_color_hex(0xE0E0E0), LV_PART_MAIN); - - // Push-to-Talk Button - ctx->btn_ptt = lv_btn_create(ctx->app_main_cont); - lv_obj_set_size(ctx->btn_ptt, 180, 50); - lv_obj_set_style_radius(ctx->btn_ptt, 25, LV_PART_MAIN); - lv_obj_set_style_bg_color(ctx->btn_ptt, lv_color_hex(0x6200EE), LV_PART_MAIN); - - ctx->btn_ptt_label = lv_label_create(ctx->btn_ptt); - lv_label_set_text(ctx->btn_ptt_label, LV_SYMBOL_AUDIO " Hold to Talk"); - lv_obj_center(ctx->btn_ptt_label); - - lv_obj_add_event_cb(ctx->btn_ptt, ptt_event_cb, LV_EVENT_ALL, ctx); - - // Launch main worker task - xTaskCreate(pipecatvoice_task, "pipecat_worker", 6144, ctx, 5, &ctx->worker_task); + lv_obj_align(ctx->state_label, LV_ALIGN_CENTER, 0, -30); + ctx->detail_label = lv_label_create(parent); + lv_obj_set_width(ctx->detail_label, lv_pct(88)); lv_label_set_long_mode(ctx->detail_label, LV_LABEL_LONG_WRAP); + lv_obj_set_style_text_align(ctx->detail_label, LV_TEXT_ALIGN_CENTER, LV_PART_MAIN); + lv_obj_align(ctx->detail_label, LV_ALIGN_CENTER, 0, 25); + if (ctx->stream_dev == NULL || ctx->socket_lock == NULL || ctx->audio_lock == NULL) set_state(ctx, PV_FAILED, "Audio service unavailable"); + else xTaskCreate(worker_task, "pv_worker", 7168, ctx, 5, &ctx->worker); } static void on_hide(AppHandle app, void* data) { - PipecatVoiceCtx* ctx = (PipecatVoiceCtx*)data; - if (ctx == NULL) return; - - ESP_LOGI(TAG, "on_hide: cleaning up"); - ctx->visible = false; - - if (ctx->ws_fd >= 0) { - int fd_to_close = ctx->ws_fd; - ctx->ws_fd = -1; - ws_close(fd_to_close); - } - - // Reset I2S controller to stop DMA and looping noise - if (ctx->i2s_dev != NULL) { - device_lock(ctx->i2s_dev); - i2s_controller_reset(ctx->i2s_dev); - device_unlock(ctx->i2s_dev); - } - - // Wait briefly for tasks to exit - int timeout = 100; - while ((ctx->worker_task != NULL || ctx->rx_task != NULL) && timeout > 0) { - vTaskDelay(pdMS_TO_TICKS(10)); - timeout--; - } + (void)app; VoiceContext* ctx = data; ctx->visible = false; close_session(ctx); + for (unsigned i = 0; (ctx->worker || ctx->receiver) && i < 100; ++i) vTaskDelay(pdMS_TO_TICKS(10)); + close_audio(ctx); + if (ctx->socket_lock) { vSemaphoreDelete(ctx->socket_lock); ctx->socket_lock = NULL; } + if (ctx->audio_lock) { vSemaphoreDelete(ctx->audio_lock); ctx->audio_lock = NULL; } } int main(int argc, char* argv[]) { - (void)argc; - (void)argv; - tt_app_register((AppRegistration) { - .createData = create_data, - .destroyData = destroy_data, - .onCreate = on_create, - .onDestroy = on_destroy, - .onShow = on_show, - .onHide = on_hide - }); + (void)argc; (void)argv; + tt_app_register((AppRegistration){.createData=create_data,.destroyData=destroy_data,.onCreate=on_create,.onDestroy=on_destroy,.onShow=on_show,.onHide=on_hide}); return 0; -} +} \ No newline at end of file diff --git a/Apps/PipecatVoice/main/Source/voice_protocol.c b/Apps/PipecatVoice/main/Source/voice_protocol.c new file mode 100644 index 0000000..5af40bd --- /dev/null +++ b/Apps/PipecatVoice/main/Source/voice_protocol.c @@ -0,0 +1,113 @@ +#include "voice_protocol.h" + +#include +#include + +/* NOTE: Do not use ctype.h (isalnum/isalpha/isdigit/isspace) in this app. Those + * functions read the `_ctype_` table, which is resolved from the flashed firmware + * at runtime; the firmware's table does not behave correctly for side-loaded ELF + * apps, so isalnum('a') can return false. Use explicit ASCII range checks instead. */ + +static bool is_digit(unsigned char c) { return c >= '0' && c <= '9'; } + +static bool is_space(unsigned char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\v' || c == '\f'; +} + +static bool is_alnum(unsigned char c) { + return is_digit(c) || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +static bool copy_part(char* destination, size_t destination_size, const char* start, size_t length) { + if (length == 0 || length >= destination_size) return false; + memcpy(destination, start, length); + destination[length] = '\0'; + return true; +} + +static bool is_loopback(const char* host) { + return strcmp(host, "localhost") == 0 || strcmp(host, "::1") == 0 || strncmp(host, "127.", 4) == 0; +} + +static bool valid_identifier(const char* value) { + if (value == NULL || *value == '\0') return false; + for (const unsigned char* p = (const unsigned char*)value; *p; ++p) { + if (!is_alnum(*p) && *p != '-' && *p != '_' && *p != '.') return false; + } + return true; +} + +bool pv_parse_endpoint(const char* url, PvEndpoint* endpoint) { + if (url == NULL || endpoint == NULL || strncmp(url, "ws://", 5) != 0) return false; + const char* authority = url + 5; + const char* path = strchr(authority, '/'); + const char* authority_end = path ? path : authority + strlen(authority); + const char* colon = NULL; + for (const char* p = authority; p < authority_end; ++p) { + if (*p == ':') { + if (colon != NULL) return false; + colon = p; + } + if (is_space((unsigned char)*p) || *p == '@' || *p == '?' || *p == '#') return false; + } + size_t host_length = (size_t)((colon ? colon : authority_end) - authority); + if (!copy_part(endpoint->host, sizeof(endpoint->host), authority, host_length) || is_loopback(endpoint->host)) return false; + endpoint->port = 80; + if (colon != NULL) { + unsigned long port = 0; + for (const char* p = colon + 1; p < authority_end; ++p) { + if (!is_digit((unsigned char)*p)) return false; + port = port * 10U + (unsigned long)(*p - '0'); + if (port > 65535U) return false; + } + if (port == 0) return false; + endpoint->port = (uint16_t)port; + } + return path == NULL ? copy_part(endpoint->path, sizeof(endpoint->path), "/", 1) + : copy_part(endpoint->path, sizeof(endpoint->path), path, strlen(path)); +} + +bool pv_make_start_json(char* out, size_t out_size, const char* session_id, const char* device_id) { + if (out == NULL || !valid_identifier(session_id) || !valid_identifier(device_id)) return false; + int written = snprintf(out, out_size, + "{\"v\":1,\"event\":\"start\",\"session_id\":\"%s\",\"device_id\":\"%s\",\"audio\":{\"format\":\"pcm_s16le\",\"sample_rate\":16000,\"channels\":1,\"sample_width\":2}}", + session_id, device_id); + return written > 0 && (size_t)written < out_size; +} + +bool pv_valid_pcm_chunk(size_t bytes) { + return bytes > 0 && bytes <= PV_PCM_CHUNK_MAX && (bytes % 2U) == 0; +} + +bool pv_valid_downstream_audio(const char* format, int sample_rate, int channels, int sample_width, size_t byte_length) { + return format != NULL && strcmp(format, "pcm_s16le") == 0 && sample_rate > 0 && sample_rate <= 48000 && + channels == 1 && sample_width == 2 && byte_length > 0 && byte_length <= PV_DOWNSTREAM_MAX && + (byte_length % 2U) == 0; +} + +bool pv_binary_matches_metadata(size_t expected_bytes, size_t received_bytes) { + return expected_bytes > 0 && expected_bytes == received_bytes; +} + +uint32_t pv_retry_delay_seconds(unsigned attempt) { + uint32_t delay = 1; + while (attempt > 0 && delay < 30) { + delay *= 2; + --attempt; + } + return delay > 30 ? 30 : delay; +} + +PvState pv_disconnect_state(bool endpoint_valid) { + return endpoint_valid ? PV_RECONNECTING : PV_FAILED; +} + +const char* pv_state_label(PvState state) { + switch (state) { + case PV_CONNECTING: return "CONNECTING"; + case PV_STREAMING: return "STREAMING"; + case PV_RECONNECTING: return "RECONNECTING"; + case PV_FAILED: return "FAILED"; + default: return "FAILED"; + } +} \ No newline at end of file diff --git a/Apps/PipecatVoice/main/Source/voice_protocol.h b/Apps/PipecatVoice/main/Source/voice_protocol.h new file mode 100644 index 0000000..1181e9f --- /dev/null +++ b/Apps/PipecatVoice/main/Source/voice_protocol.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include + +#define PV_PROTOCOL_VERSION 1 +#define PV_PCM_CHUNK_MAX 16384U +#define PV_DOWNSTREAM_MAX 65536U + +typedef enum { + PV_CONNECTING, + PV_STREAMING, + PV_RECONNECTING, + PV_FAILED, +} PvState; + +typedef struct { + char host[64]; + char path[96]; + uint16_t port; +} PvEndpoint; + +bool pv_parse_endpoint(const char* url, PvEndpoint* endpoint); +bool pv_make_start_json(char* out, size_t out_size, const char* session_id, const char* device_id); +bool pv_valid_pcm_chunk(size_t bytes); +bool pv_valid_downstream_audio(const char* format, int sample_rate, int channels, int sample_width, size_t byte_length); +bool pv_binary_matches_metadata(size_t expected_bytes, size_t received_bytes); +uint32_t pv_retry_delay_seconds(unsigned attempt); +PvState pv_disconnect_state(bool endpoint_valid); +const char* pv_state_label(PvState state); diff --git a/Apps/PipecatVoice/main/Source/websocket.c b/Apps/PipecatVoice/main/Source/websocket.c index caffac5..7695a8d 100644 --- a/Apps/PipecatVoice/main/Source/websocket.c +++ b/Apps/PipecatVoice/main/Source/websocket.c @@ -2,239 +2,166 @@ #include #include -#include #include -#include -#include #include +#include +#include +#include -static int recv_all(int fd, void* buf, size_t len) { - size_t total = 0; - char* p = (char*)buf; - while (total < len) { - int r = lwip_recv(fd, p + total, len - total, 0); - if (r <= 0) { - return -1; - } - total += r; +#define TAG "PipecatVoiceWs" +#define WS_HEADER_LIMIT 1024U +#define WS_CONTROL_LIMIT 125U + +static int send_all(int fd, const uint8_t* data, size_t length) { + size_t sent = 0; + while (sent < length) { + int result = lwip_send(fd, data + sent, length - sent, 0); + if (result <= 0) return -1; + sent += (size_t)result; } return 0; } -static uint16_t my_htons(uint16_t val) { - return (uint16_t)(((val & 0xff) << 8) | ((val & 0xff00) >> 8)); +static int recv_all(int fd, uint8_t* data, size_t length) { + size_t received = 0; + while (received < length) { + int result = lwip_recv(fd, data + received, length - received, 0); + if (result <= 0) return -1; + received += (size_t)result; + } + return 0; } -int ws_connect(const char* host, int port, const char* path, const char* device_id, const char* auth_key) { +static int discard(int fd, uint64_t length) { + uint8_t buffer[256]; + while (length > 0) { + size_t chunk = length > sizeof(buffer) ? sizeof(buffer) : (size_t)length; + if (recv_all(fd, buffer, chunk) < 0) return -1; + length -= chunk; + } + return 0; +} + +static int send_frame(int fd, uint8_t opcode, const uint8_t* payload, size_t length) { + if (length > 65535U || ((opcode & 0x08U) && length > WS_CONTROL_LIMIT)) return -1; + uint8_t header[8]; + size_t header_length = 2; + header[0] = 0x80U | opcode; + if (length < 126U) { + header[1] = 0x80U | (uint8_t)length; + } else { + header[1] = 0x80U | 126U; + header[2] = (uint8_t)(length >> 8U); + header[3] = (uint8_t)length; + header_length = 4; + } + uint8_t mask[4]; + uint32_t random = esp_random(); + memcpy(mask, &random, sizeof(mask)); + memcpy(header + header_length, mask, sizeof(mask)); + header_length += sizeof(mask); + if (send_all(fd, header, header_length) < 0) return -1; + + uint8_t chunk[512]; + size_t offset = 0; + while (offset < length) { + size_t count = length - offset > sizeof(chunk) ? sizeof(chunk) : length - offset; + for (size_t i = 0; i < count; ++i) chunk[i] = payload[offset + i] ^ mask[(offset + i) % sizeof(mask)]; + if (send_all(fd, chunk, count) < 0) return -1; + offset += count; + } + return 0; +} + +int ws_connect(const char* host, int port, const char* path, const char* device_id, const char* api_key) { + if (host == NULL || path == NULL || device_id == NULL || api_key == NULL || port < 1 || port > 65535) return -1; int fd = lwip_socket(AF_INET, SOCK_STREAM, 0); - if (fd < 0) return -1; - - struct sockaddr_in addr; - memset(&addr, 0, sizeof(addr)); - addr.sin_family = AF_INET; - addr.sin_port = my_htons(port); - addr.sin_addr.s_addr = ipaddr_addr(host); - - if (lwip_connect(fd, (struct sockaddr*)&addr, sizeof(addr)) < 0) { + if (fd < 0) { + ESP_LOGW(TAG, "socket create failed"); + return -1; + } + struct sockaddr_in address = {0}; + address.sin_family = AF_INET; + address.sin_port = htons((uint16_t)port); + address.sin_addr.s_addr = ipaddr_addr(host); + if (address.sin_addr.s_addr == IPADDR_NONE) { + ESP_LOGW(TAG, "endpoint address parse failed"); close(fd); return -1; } - - // Set socket receive timeout (e.g. 90 seconds) to prevent blocking indefinitely - struct timeval tv; - tv.tv_sec = 90; - tv.tv_usec = 0; - lwip_setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); - - // Send HTTP upgrade handshake request - char req[1024]; - snprintf(req, sizeof(req), - "GET %s HTTP/1.1\r\n" - "Host: %s:%d\r\n" - "Upgrade: websocket\r\n" - "Connection: Upgrade\r\n" - "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n" - "Sec-WebSocket-Version: 13\r\n" - "Authorization: Bearer %s\r\n" - "X-Device-ID: %s\r\n" - "\r\n", - path, host, port, auth_key, device_id); - - if (lwip_send(fd, req, strlen(req), 0) < 0) { + if (lwip_connect(fd, (struct sockaddr*)&address, sizeof(address)) < 0) { + ESP_LOGW(TAG, "TCP connect failed"); close(fd); return -1; } - - // Read HTTP response headers until we hit "\r\n\r\n" - char header_buf[1024]; - size_t header_len = 0; - while (header_len < sizeof(header_buf) - 1) { - char c; - int r = lwip_recv(fd, &c, 1, 0); - if (r <= 0) { + struct timeval timeout = {.tv_sec = 15, .tv_usec = 0}; + lwip_setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); + char request[WS_HEADER_LIMIT]; + int request_length = snprintf(request, sizeof(request), + "GET %s HTTP/1.1\r\nHost: %s:%d\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n" + "Sec-WebSocket-Key: MDEyMzQ1Njc4OWFiY2RlZg==\r\nSec-WebSocket-Version: 13\r\n" + "Authorization: Bearer %s\r\nX-Device-ID: %s\r\n\r\n", + path, host, port, api_key, device_id); + if (request_length < 0 || (size_t)request_length >= sizeof(request) || send_all(fd, (const uint8_t*)request, (size_t)request_length) < 0) { + ESP_LOGW(TAG, "WebSocket upgrade request failed"); + close(fd); + return -1; + } + char response[WS_HEADER_LIMIT]; + size_t length = 0; + while (length + 1 < sizeof(response)) { + if (recv_all(fd, (uint8_t*)&response[length], 1) < 0) { + ESP_LOGW(TAG, "WebSocket upgrade response failed"); close(fd); return -1; } - header_buf[header_len++] = c; - header_buf[header_len] = '\0'; - - if (header_len >= 4 && strcmp(header_buf + header_len - 4, "\r\n\r\n") == 0) { - break; - } + response[++length] = '\0'; + if (length >= 4 && memcmp(response + length - 4, "\r\n\r\n", 4) == 0) break; } - - // Verify HTTP 101 Switching Protocols response status - if (strstr(header_buf, "HTTP/1.1 101") == NULL && strstr(header_buf, "HTTP/1.0 101") == NULL) { + if (length + 1 >= sizeof(response) || strstr(response, " 101 ") == NULL) { + ESP_LOGW(TAG, "WebSocket upgrade rejected"); close(fd); return -1; } - + ESP_LOGI(TAG, "WebSocket upgrade accepted"); return fd; } -int ws_send(int fd, const uint8_t* data, size_t len, bool binary) { - uint8_t header[10]; - size_t header_len = 0; - - header[0] = binary ? 0x82 : 0x81; - - if (len < 126) { - header[1] = 0x80 | (uint8_t)len; - header_len = 2; - } else { - header[1] = 0x80 | 126; - header[2] = (uint8_t)((len >> 8) & 0xFF); - header[3] = (uint8_t)(len & 0xFF); - header_len = 4; - } - - // Use fixed client mask for performance: 0x12, 0x34, 0x56, 0x78 - uint8_t mask[4] = { 0x12, 0x34, 0x56, 0x78 }; - memcpy(header + header_len, mask, 4); - header_len += 4; - - // Send WebSocket frame header - int sent = lwip_send(fd, header, header_len, 0); - if (sent < 0) return -1; - - // Mask the payload - uint8_t* masked = malloc(len); - if (masked == NULL) return -1; - for (size_t i = 0; i < len; ++i) { - masked[i] = data[i] ^ mask[i % 4]; - } - - // Send masked payload - sent = lwip_send(fd, masked, len, 0); - free(masked); - - return sent >= 0 ? 0 : -1; +int ws_send(int fd, const uint8_t* data, size_t length, bool binary) { + if (fd < 0 || data == NULL || length == 0) return -1; + return send_frame(fd, binary ? 0x02U : 0x01U, data, length); } -int ws_recv(int fd, int* out_opcode, uint8_t* payload, size_t max_len) { +int ws_recv(int fd, int* opcode, bool* final, uint8_t* payload, size_t maximum) { uint8_t header[2]; - if (recv_all(fd, header, 2) < 0) { - return -1; + if (fd < 0 || recv_all(fd, header, sizeof(header)) < 0) return -1; + uint64_t length = header[1] & 0x7fU; + if (length == 126U) { + uint8_t extended[2]; + if (recv_all(fd, extended, sizeof(extended)) < 0) return -1; + length = ((uint64_t)extended[0] << 8U) | extended[1]; + } else if (length == 127U) { + uint8_t extended[8]; + if (recv_all(fd, extended, sizeof(extended)) < 0) return -1; + length = 0; + for (size_t i = 0; i < sizeof(extended); ++i) length = (length << 8U) | extended[i]; } - - int opcode = header[0] & 0x0F; - if (out_opcode != NULL) { - *out_opcode = opcode; + bool masked = (header[1] & 0x80U) != 0; + uint8_t mask[4] = {0}; + if (masked && recv_all(fd, mask, sizeof(mask)) < 0) return -1; + uint8_t frame_opcode = header[0] & 0x0fU; + if (((frame_opcode & 0x08U) && (length > WS_CONTROL_LIMIT || !(header[0] & 0x80U))) || length > maximum) { + if (discard(fd, length) < 0) return -1; + return -2; } - - int masked = (header[1] & 0x80) != 0; - size_t len = header[1] & 0x7F; - - if (len == 126) { - uint8_t ext_len[2]; - if (recv_all(fd, ext_len, 2) < 0) return -1; - len = ((size_t)ext_len[0] << 8) | ext_len[1]; - } else if (len == 127) { - uint8_t ext_len[8]; - if (recv_all(fd, ext_len, 8) < 0) return -1; - // Parse 64-bit length into size_t - len = ((size_t)ext_len[4] << 24) | ((size_t)ext_len[5] << 16) | ((size_t)ext_len[6] << 8) | ext_len[7]; - } - - if (masked) { - uint8_t mask[4]; - if (recv_all(fd, mask, 4) < 0) return -1; - - if (len > max_len) { - ESP_LOGE("websocket", "ws_recv overflow (masked): len=%u, max_len=%u", (unsigned)len, (unsigned)max_len); - // Buffer overflow, skip payload to align stream - size_t to_discard = len; - uint8_t discard_buf[256]; - while (to_discard > 0) { - size_t chunk = to_discard < sizeof(discard_buf) ? to_discard : sizeof(discard_buf); - if (recv_all(fd, discard_buf, chunk) < 0) return -1; - to_discard -= chunk; - } - return -2; - } - - if (recv_all(fd, payload, len) < 0) return -1; - for (size_t i = 0; i < len; ++i) { - payload[i] ^= mask[i % 4]; - } - } else { - if (len > max_len) { - ESP_LOGE("websocket", "ws_recv overflow (unmasked): len=%u, max_len=%u", (unsigned)len, (unsigned)max_len); - size_t to_discard = len; - uint8_t discard_buf[256]; - while (to_discard > 0) { - size_t chunk = to_discard < sizeof(discard_buf) ? to_discard : sizeof(discard_buf); - if (recv_all(fd, discard_buf, chunk) < 0) return -1; - to_discard -= chunk; - } - return -2; - } - - if (recv_all(fd, payload, len) < 0) return -1; - } - - return (int)len; + if (length > 0 && recv_all(fd, payload, (size_t)length) < 0) return -1; + if (masked) for (size_t i = 0; i < (size_t)length; ++i) payload[i] ^= mask[i % sizeof(mask)]; + if (opcode) *opcode = frame_opcode; + if (final) *final = (header[0] & 0x80U) != 0; + return (int)length; } -void ws_close(int fd) { - if (fd >= 0) { - close(fd); - } -} - -int ws_send_pong(int fd, const uint8_t* payload, size_t len) { - uint8_t header[10]; - size_t header_len = 0; - - header[0] = 0x8A; // FIN | PONG (0x0A) - - if (len < 126) { - header[1] = 0x80 | (uint8_t)len; - header_len = 2; - } else { - header[1] = 0x80 | 126; - header[2] = (uint8_t)((len >> 8) & 0xFF); - header[3] = (uint8_t)(len & 0xFF); - header_len = 4; - } - - uint8_t mask[4] = { 0x12, 0x34, 0x56, 0x78 }; - memcpy(header + header_len, mask, 4); - header_len += 4; - - int sent = lwip_send(fd, header, header_len, 0); - if (sent < 0) return -1; - - if (len > 0 && payload != NULL) { - uint8_t* masked = malloc(len); - if (masked == NULL) return -1; - for (size_t i = 0; i < len; ++i) { - masked[i] = payload[i] ^ mask[i % 4]; - } - sent = lwip_send(fd, masked, len, 0); - free(masked); - } - - return sent >= 0 ? 0 : -1; -} +int ws_send_pong(int fd, const uint8_t* payload, size_t length) { return send_frame(fd, 0x0aU, payload, length); } +int ws_send_close(int fd) { return send_frame(fd, 0x08U, NULL, 0); } +void ws_close(int fd) { if (fd >= 0) close(fd); } \ No newline at end of file diff --git a/Apps/PipecatVoice/main/Source/websocket.h b/Apps/PipecatVoice/main/Source/websocket.h index 396d430..d5f4f4d 100644 --- a/Apps/PipecatVoice/main/Source/websocket.h +++ b/Apps/PipecatVoice/main/Source/websocket.h @@ -14,10 +14,10 @@ extern "C" { * @param port Port number (e.g. 8642) * @param path WebSocket path (e.g. "/api/esp32/voice/ws") * @param device_id Unique device identifier - * @param auth_key Hermes Bearer API key + * @param api_key Optional profile API key; never compiled into firmware * @return Socket file descriptor on success, or -1 on failure */ -int ws_connect(const char* host, int port, const char* path, const char* device_id, const char* auth_key); +int ws_connect(const char* host, int port, const char* path, const char* device_id, const char* api_key); /** * Send a WebSocket frame. @@ -37,7 +37,7 @@ int ws_send(int fd, const uint8_t* data, size_t len, bool binary); * @param max_len Maximum length of the payload buffer * @return Received payload length on success, -1 on connection failure, or -2 on buffer overflow */ -int ws_recv(int fd, int* out_opcode, uint8_t* payload, size_t max_len); +int ws_recv(int fd, int* out_opcode, bool* out_final, uint8_t* payload, size_t max_len); /** * Close a WebSocket connection. @@ -54,6 +54,9 @@ void ws_close(int fd); */ int ws_send_pong(int fd, const uint8_t* payload, size_t len); +/** Send a clean WebSocket close control frame before closing the socket. */ +int ws_send_close(int fd); + #ifdef __cplusplus } #endif diff --git a/Apps/PipecatVoice/tests/test_voice_protocol.c b/Apps/PipecatVoice/tests/test_voice_protocol.c new file mode 100644 index 0000000..ee1a8b3 --- /dev/null +++ b/Apps/PipecatVoice/tests/test_voice_protocol.c @@ -0,0 +1,52 @@ +#include "voice_protocol.h" + +#include +#include +#include + +static void test_endpoint_validation(void) { + PvEndpoint endpoint; + assert(pv_parse_endpoint("ws://192.168.68.102:8644/api/esp32/voice/ws", &endpoint)); + assert(strcmp(endpoint.host, "192.168.68.102") == 0); + assert(endpoint.port == 8644); + assert(strcmp(endpoint.path, "/api/esp32/voice/ws") == 0); + assert(!pv_parse_endpoint("ws://127.0.0.1:8642/api", &endpoint)); + assert(!pv_parse_endpoint("ws://localhost:8642/api", &endpoint)); + assert(!pv_parse_endpoint("wss://192.168.68.102/api", &endpoint)); + assert(!pv_parse_endpoint("ws://192.168.68.102:0/api", &endpoint)); +} + +static void test_start_and_pcm_boundaries(void) { + char json[256]; + assert(pv_make_start_json(json, sizeof(json), "session-01", "tactility-14c19d1a790")); + assert(strstr(json, "\"v\":1") != NULL); + assert(strstr(json, "\"pcm_s16le\"") != NULL); + assert(!pv_make_start_json(json, sizeof(json), "bad session", "device")); + assert(pv_valid_pcm_chunk(2)); + assert(pv_valid_pcm_chunk(PV_PCM_CHUNK_MAX)); + assert(!pv_valid_pcm_chunk(0)); + assert(!pv_valid_pcm_chunk(3)); + assert(!pv_valid_pcm_chunk(PV_PCM_CHUNK_MAX + 2)); + assert(pv_valid_downstream_audio("pcm_s16le", 24000, 1, 2, 48000)); + assert(!pv_valid_downstream_audio("wav", 24000, 1, 2, 48000)); + assert(!pv_valid_downstream_audio("pcm_s16le", 24000, 2, 2, 48000)); + assert(!pv_valid_downstream_audio("pcm_s16le", 24000, 1, 2, PV_DOWNSTREAM_MAX + 2)); + assert(pv_binary_matches_metadata(48000, 48000)); + assert(!pv_binary_matches_metadata(48000, 47998)); +} + +static void test_retry_and_state(void) { + const uint32_t expected[] = {1, 2, 4, 8, 16, 30, 30}; + for (unsigned i = 0; i < sizeof(expected) / sizeof(expected[0]); ++i) assert(pv_retry_delay_seconds(i) == expected[i]); + assert(pv_disconnect_state(true) == PV_RECONNECTING); + assert(pv_disconnect_state(false) == PV_FAILED); + assert(strcmp(pv_state_label(PV_STREAMING), "STREAMING") == 0); +} + +int main(void) { + test_endpoint_validation(); + test_start_and_pcm_boundaries(); + test_retry_and_state(); + puts("voice_protocol tests passed"); + return 0; +} diff --git a/Apps/ReynaBot/main/Source/main.c b/Apps/ReynaBot/main/Source/main.c index 99cce97..99f1115 100644 --- a/Apps/ReynaBot/main/Source/main.c +++ b/Apps/ReynaBot/main/Source/main.c @@ -1,24 +1,32 @@ #include #include #include -#include #include -#include +#include #include "websocket.h" #include #include #include +#include #include #include +/* Exported by Tactility firmware 0.8.0-dev; absent from the CDN SDK header. */ +struct Device* device_find_by_name(const char* name); + #include "freertos/FreeRTOS.h" #include "freertos/task.h" #include "esp_log.h" +#include "esp_random.h" #define TAG "ReynaBot" +#define AUDIO_SAMPLE_RATE 16000U +#define AUDIO_BITS_PER_SAMPLE 16U +#define MAX_PCM_CHUNK_BYTES 16384U +#define MAX_RESPONSE_AUDIO_BYTES 65536U typedef enum { STATE_IDLE, @@ -64,10 +72,19 @@ typedef struct { TaskHandle_t worker_task; TaskHandle_t rx_task; - // Hardware - struct Device* i2s_dev; - - // WebSocket + // Hardware/audio service + struct Device* audio_stream_dev; + AudioStreamHandle input_handle; + AudioStreamHandle output_handle; + uint32_t output_sample_rate; + uint8_t output_channels; + uint8_t output_bits_per_sample; + bool output_stream_pending; + size_t expected_audio_bytes; + uint32_t expected_audio_rate; + uint8_t expected_audio_channels; + uint8_t expected_audio_bits; + bool stop_sent; int ws_fd; bool ws_connected; bool ws_done; @@ -81,6 +98,77 @@ static void reynabot_rx_task(void* arg); static void update_ui(ReynaBotCtx* ctx); static void load_config(ReynaBotCtx* ctx); +static bool find_audio_stream_device(ReynaBotCtx* ctx) { + struct Device* dev = device_find_by_name("audio-stream"); + if (dev != NULL) { + ctx->audio_stream_dev = dev; + return true; + } + // The 0.8.0-dev firmware registers this shared service by name. The + // CDN SDK headers do not expose the deprecated type-search helper, so do + // not reference it from a dynamically linked app. + return false; +} + +static void close_input_stream(ReynaBotCtx* ctx) { + if (ctx->input_handle != NULL) { + audio_stream_close(ctx->input_handle); + ctx->input_handle = NULL; + } +} + +static void close_output_stream(ReynaBotCtx* ctx) { + if (ctx->output_handle != NULL) { + audio_stream_close(ctx->output_handle); + ctx->output_handle = NULL; + } + ctx->output_stream_pending = false; +} + +static bool open_input_stream(ReynaBotCtx* ctx) { + if (ctx->audio_stream_dev == NULL) return false; + close_input_stream(ctx); + struct AudioStreamConfig config = { + .sample_rate = 16000, + .bits_per_sample = 16, + .channels = 1, + }; + error_t err = audio_stream_open_input(ctx->audio_stream_dev, &config, &ctx->input_handle); + if (err != ERROR_NONE) { + ctx->input_handle = NULL; + ESP_LOGE(TAG, "audio_stream_open_input failed: %d", err); + return false; + } + audio_stream_set_mute(ctx->audio_stream_dev, AUDIO_CODEC_DIR_INPUT, false); + audio_stream_set_volume(ctx->audio_stream_dev, AUDIO_CODEC_DIR_INPUT, 100.0f); + ESP_LOGI(TAG, "Microphone opened via audio-stream at 16000 Hz mono"); + return true; +} + +static bool open_output_stream(ReynaBotCtx* ctx, uint32_t sample_rate, uint8_t channels, uint8_t bits_per_sample) { + if (ctx->audio_stream_dev == NULL || sample_rate == 0 || channels == 0 || bits_per_sample != 16) return false; + close_output_stream(ctx); + struct AudioStreamConfig config = { + .sample_rate = sample_rate, + .bits_per_sample = bits_per_sample, + .channels = channels, + }; + error_t err = audio_stream_open_output(ctx->audio_stream_dev, &config, &ctx->output_handle); + if (err != ERROR_NONE) { + ctx->output_handle = NULL; + ESP_LOGE(TAG, "audio_stream_open_output failed: %d rate=%u ch=%u", err, (unsigned)sample_rate, channels); + return false; + } + audio_stream_set_mute(ctx->audio_stream_dev, AUDIO_CODEC_DIR_OUTPUT, false); + audio_stream_set_volume(ctx->audio_stream_dev, AUDIO_CODEC_DIR_OUTPUT, 100.0f); + ctx->output_sample_rate = sample_rate; + ctx->output_channels = channels; + ctx->output_bits_per_sample = bits_per_sample; + ctx->output_stream_pending = true; + ESP_LOGI(TAG, "Speaker opened via audio-stream at %u Hz mono", (unsigned)sample_rate); + return true; +} + /* ─── Helper for URL parsing ─── */ static bool parse_ws_url(const char* url, char* host, int* port, char* path) { if (strncmp(url, "ws://", 5) != 0) return false; @@ -111,9 +199,9 @@ static bool parse_ws_url(const char* url, char* host, int* port, char* path) { /* ─── Config Loading/Saving ─── */ static void load_config(ReynaBotCtx* ctx) { // Default fallback config - snprintf(ctx->server_url, sizeof(ctx->server_url), "ws://192.168.68.126:8643/api/esp32/voice/ws"); + snprintf(ctx->server_url, sizeof(ctx->server_url), "ws://192.168.68.112:8643/api/esp32/voice/ws"); snprintf(ctx->device_id, sizeof(ctx->device_id), "reynabot_screen"); - snprintf(ctx->api_key, sizeof(ctx->api_key), "hmek_sXB7921bZ9FXTVKARExqUZ7ttBxtEoURHRU0JCB-gNY"); + ctx->api_key[0] = '\0'; char path[256]; size_t path_size = sizeof(path); @@ -130,7 +218,7 @@ static void load_config(ReynaBotCtx* ctx) { // Write default config file file = fopen(path, "w"); if (file != NULL) { - fprintf(file, "{\n \"server_url\": \"ws://192.168.68.126:8643/api/esp32/voice/ws\",\n \"device_id\": \"reynabot_screen\",\n \"api_key\": \"mcT1YA1vOr9wXSiHpCYalweEGGZKX-PIfZv2drp8BSg\"\n}\n"); + fprintf(file, "{\n \"server_url\": \"ws://192.168.68.112:8643/api/esp32/voice/ws\",\n \"device_id\": \"reynabot_screen\",\n \"api_key\": \"\"\n}\n"); fclose(file); } ESP_LOGI(TAG, "Created default config.json at %s", path); @@ -181,6 +269,21 @@ static void parse_json_message(ReynaBotCtx* ctx, const char* json_str) { if (strcmp(evt, "ready") == 0) { ESP_LOGI(TAG, "Server ready"); + } else if (strcmp(evt, "state") == 0) { + cJSON* state_item = cJSON_GetObjectItem(json, "state"); + if (state_item != NULL && cJSON_IsString(state_item)) { + if (strcmp(state_item->valuestring, "listening") == 0) { + if (ctx->stop_sent) { + ctx->ws_done = true; + } else { + ctx->state = STATE_LISTENING; + } + ctx->ui_update_pending = true; + } else if (strcmp(state_item->valuestring, "processing") == 0) { + ctx->state = STATE_THINKING; + ctx->ui_update_pending = true; + } + } } else if (strcmp(evt, "listening") == 0) { ctx->state = STATE_LISTENING; ctx->ui_update_pending = true; @@ -199,11 +302,42 @@ static void parse_json_message(ReynaBotCtx* ctx, const char* json_str) { strncpy(ctx->last_response, txt_item->valuestring, sizeof(ctx->last_response) - 1); ctx->ui_update_pending = true; } + } else if (strcmp(evt, "audio") == 0) { + cJSON* format_item = cJSON_GetObjectItem(json, "format"); + cJSON* rate_item = cJSON_GetObjectItem(json, "sample_rate"); + cJSON* channels_item = cJSON_GetObjectItem(json, "channels"); + cJSON* width_item = cJSON_GetObjectItem(json, "sample_width"); + cJSON* length_item = cJSON_GetObjectItem(json, "byte_length"); + bool valid = format_item != NULL && cJSON_IsString(format_item) && + strcmp(format_item->valuestring, "pcm_s16le") == 0 && + rate_item != NULL && cJSON_IsNumber(rate_item) && rate_item->valueint > 0 && rate_item->valueint <= 48000 && + channels_item != NULL && cJSON_IsNumber(channels_item) && channels_item->valueint == 1 && + width_item != NULL && cJSON_IsNumber(width_item) && width_item->valueint == 2 && + length_item != NULL && cJSON_IsNumber(length_item) && length_item->valueint > 0 && + length_item->valueint <= MAX_RESPONSE_AUDIO_BYTES && (length_item->valueint % 2) == 0; + if (valid) { + ctx->expected_audio_rate = (uint32_t)rate_item->valueint; + ctx->expected_audio_channels = (uint8_t)channels_item->valueint; + ctx->expected_audio_bits = (uint8_t)width_item->valueint; + ctx->expected_audio_bytes = (size_t)length_item->valueint; + if (!open_output_stream(ctx, ctx->expected_audio_rate, ctx->expected_audio_channels, ctx->expected_audio_bits)) { + snprintf(ctx->error_message, sizeof(ctx->error_message), "Audio output unavailable"); + ctx->state = STATE_ERROR; + ctx->expected_audio_bytes = 0; + } else { + ctx->state = STATE_SPEAKING; + } + ctx->ui_update_pending = true; + } else { + snprintf(ctx->error_message, sizeof(ctx->error_message), "Invalid audio metadata"); + ctx->state = STATE_ERROR; + ctx->expected_audio_bytes = 0; + ctx->ui_update_pending = true; + } } else if (strcmp(evt, "audio_start") == 0) { + /* Legacy event: metadata must still arrive as `audio` before binary PCM. */ ctx->state = STATE_SPEAKING; ctx->ui_update_pending = true; - - // I2S is already configured globally at session startup } else if (strcmp(evt, "audio_end") == 0) { ESP_LOGI(TAG, "Audio response ended"); } else if (strcmp(evt, "done") == 0) { @@ -225,7 +359,7 @@ static void parse_json_message(ReynaBotCtx* ctx, const char* json_str) { /* ─── WebSocket RX (Receive) Task ─── */ static void reynabot_rx_task(void* arg) { ReynaBotCtx* ctx = (ReynaBotCtx*)arg; - uint8_t* rx_buf = malloc(8192); + uint8_t* rx_buf = malloc(MAX_RESPONSE_AUDIO_BYTES + 1U); if (rx_buf == NULL) { ESP_LOGE(TAG, "Failed to allocate RX buffer"); ctx->ws_done = true; @@ -238,7 +372,7 @@ static void reynabot_rx_task(void* arg) { ESP_LOGI(TAG, "WS Receive task started"); while (ctx->ws_fd >= 0) { - int r = ws_recv(ctx->ws_fd, &opcode, rx_buf, 8191); + int r = ws_recv(ctx->ws_fd, &opcode, rx_buf, MAX_RESPONSE_AUDIO_BYTES); if (r < 0) { if (r == -1) { ESP_LOGI(TAG, "WS connection closed or read error, errno=%d", errno); @@ -252,21 +386,31 @@ static void reynabot_rx_task(void* arg) { if (opcode == 0x01) { // Text frame (JSON) rx_buf[r] = '\0'; parse_json_message(ctx, (char*)rx_buf); - } else if (opcode == 0x02) { // Binary frame (Audio data) - if (ctx->state == STATE_SPEAKING && ctx->i2s_dev != NULL) { - const uint8_t* payload_ptr = rx_buf; - size_t payload_len = r; - - // Skip WAV header if present in the first chunk - if (payload_len > 44 && memcmp(payload_ptr, "RIFF", 4) == 0) { - payload_ptr += 44; - payload_len -= 44; + } else if (opcode == 0x02) { // Audio response PCM + if (ctx->output_handle == NULL || ctx->expected_audio_bytes == 0 || (size_t)r != ctx->expected_audio_bytes) { + ESP_LOGE(TAG, "Audio frame does not match metadata: got=%d expected=%u", r, (unsigned)ctx->expected_audio_bytes); + ctx->state = STATE_ERROR; + snprintf(ctx->error_message, sizeof(ctx->error_message), "Invalid audio frame"); + ctx->ui_update_pending = true; + } else { + size_t written_total = 0; + while (written_total < (size_t)r) { + size_t written = 0; + error_t err = audio_stream_write(ctx->output_handle, rx_buf + written_total, + (size_t)r - written_total, &written, pdMS_TO_TICKS(3000)); + if (err != ERROR_NONE || written == 0) { + ESP_LOGE(TAG, "audio_stream_write failed: %d written=%u", err, (unsigned)written); + ctx->state = STATE_ERROR; + snprintf(ctx->error_message, sizeof(ctx->error_message), "Audio playback failed"); + ctx->ui_update_pending = true; + break; + } + written_total += written; } - - size_t written = 0; - device_lock(ctx->i2s_dev); - i2s_controller_write(ctx->i2s_dev, payload_ptr, payload_len, &written, pdMS_TO_TICKS(100)); - device_unlock(ctx->i2s_dev); + close_output_stream(ctx); + ctx->expected_audio_bytes = 0; + ctx->state = STATE_THINKING; + ctx->ui_update_pending = true; } } else if (opcode == 0x09) { // PING frame ESP_LOGI(TAG, "WS PING received, sending PONG"); @@ -408,8 +552,11 @@ static void reynabot_task(void* arg) { ctx->start_session = false; ctx->stop_session = false; ctx->cancel_session = false; - - ctx->state = STATE_CONNECTING; + ctx->stop_sent = false; + ctx->expected_audio_bytes = 0; + close_input_stream(ctx); + close_output_stream(ctx); + ctx->last_transcript[0] = '\0'; ctx->last_response[0] = '\0'; ctx->error_message[0] = '\0'; @@ -434,29 +581,18 @@ static void reynabot_task(void* arg) { ctx->ws_fd = fd; ctx->ws_connected = true; - - // Configure I2S on demand for this session (16 kHz, 16-bit, mono) - struct I2sConfig session_cfg = { - .communication_format = I2S_FORMAT_STAND_I2S, - .sample_rate = 16000, - .bits_per_sample = 16, - .channel_left = 0, - .channel_right = I2S_CHANNEL_NONE - }; - if (ctx->i2s_dev != NULL) { - device_lock(ctx->i2s_dev); - i2s_controller_set_config(ctx->i2s_dev, &session_cfg); - device_unlock(ctx->i2s_dev); - } - - // Spawn background RX task to read and parse events + + // Spawn background RX task to read and parse events. xTaskCreate(reynabot_rx_task, "reynabot_rx", 4096, ctx, 6, &ctx->rx_task); - - // Send start event handshake - char start_json[256]; + + // The Kids LAN adapter requires protocol v1 and a fresh session id. + char session_id[80]; + snprintf(session_id, sizeof(session_id), "reynabot-%08x%08x", + (unsigned)esp_random(), (unsigned)esp_random()); + char start_json[384]; snprintf(start_json, sizeof(start_json), - "{\"event\":\"start\",\"device_id\":\"%s\",\"sample_rate\":16000,\"channels\":1,\"sample_width\":2,\"format\":\"pcm_s16le\"}", - ctx->device_id); + "{\"v\":1,\"event\":\"start\",\"session_id\":\"%s\",\"device_id\":\"%s\",\"audio\":{\"format\":\"pcm_s16le\",\"sample_rate\":16000,\"channels\":1,\"sample_width\":2}}", + session_id, ctx->device_id); if (ws_send(ctx->ws_fd, (const uint8_t*)start_json, strlen(start_json), false) < 0) { ctx->state = STATE_ERROR; snprintf(ctx->error_message, sizeof(ctx->error_message), "Handshake send failed"); @@ -486,11 +622,20 @@ static void reynabot_task(void* arg) { continue; } - // I2S is already configured globally at session startup - - uint8_t* buffer = malloc(1024); + if (!open_input_stream(ctx)) { + ctx->state = STATE_ERROR; + snprintf(ctx->error_message, sizeof(ctx->error_message), "Microphone unavailable"); + ws_close(ctx->ws_fd); + ctx->ws_fd = -1; + ctx->ws_connected = false; + update_ui(ctx); + continue; + } + + uint8_t* buffer = malloc(MAX_PCM_CHUNK_BYTES); if (buffer == NULL) { ESP_LOGE(TAG, "Failed to allocate record buffer"); + close_input_stream(ctx); ctx->state = STATE_ERROR; snprintf(ctx->error_message, sizeof(ctx->error_message), "Out of memory"); ws_close(ctx->ws_fd); @@ -500,12 +645,13 @@ static void reynabot_task(void* arg) { continue; } size_t total_sent_bytes = 0; - - // Stream audio loop while PTT is held + + // Stream 16 kHz mono PCM while PTT is held. while (ctx->is_pressed && !ctx->stop_session && !ctx->cancel_session && !ctx->ws_done && total_sent_bytes < 320000) { size_t bytes_read = 0; - error_t r = i2s_controller_read(ctx->i2s_dev, buffer, 1024, &bytes_read, pdMS_TO_TICKS(100)); - if (r == ERROR_NONE && bytes_read > 0) { + error_t r = audio_stream_read(ctx->input_handle, buffer, MAX_PCM_CHUNK_BYTES, + &bytes_read, pdMS_TO_TICKS(200)); + if (r == ERROR_NONE && bytes_read > 0 && bytes_read <= MAX_PCM_CHUNK_BYTES && (bytes_read % 2U) == 0) { if (ws_send(ctx->ws_fd, buffer, bytes_read, true) < 0) { ESP_LOGE(TAG, "Audio stream send failed"); break; @@ -513,17 +659,20 @@ static void reynabot_task(void* arg) { total_sent_bytes += bytes_read; } } - + + close_input_stream(ctx); free(buffer); - + if (ctx->cancel_session || total_sent_bytes < 3200) { ESP_LOGI(TAG, "Cancelling audio session"); const char* cancel_json = "{\"event\":\"cancel\"}"; ws_send(ctx->ws_fd, (const uint8_t*)cancel_json, strlen(cancel_json), false); + ctx->stop_sent = true; ctx->state = STATE_IDLE; } else { const char* stop_json = "{\"event\":\"stop\"}"; ws_send(ctx->ws_fd, (const uint8_t*)stop_json, strlen(stop_json), false); + ctx->stop_sent = true; ctx->state = STATE_THINKING; update_ui(ctx); @@ -550,13 +699,9 @@ static void reynabot_task(void* arg) { ctx->ws_connected = false; ws_close(fd_to_close); - // Reset I2S controller to release DMA and stop white noise - if (ctx->i2s_dev != NULL) { - device_lock(ctx->i2s_dev); - i2s_controller_reset(ctx->i2s_dev); - device_unlock(ctx->i2s_dev); - } - + close_input_stream(ctx); + close_output_stream(ctx); + // Wait for receive task to exit int rx_timeout = 100; while (ctx->rx_task != NULL && rx_timeout > 0) { @@ -611,10 +756,9 @@ static void on_show(AppHandle app, void* data, lv_obj_t* parent) { load_config(ctx); - // Find I2S controller - ctx->i2s_dev = device_find_by_name("i2s0"); - if (ctx->i2s_dev == NULL) { - ESP_LOGE(TAG, "I2S controller 'i2s0' not found!"); + // Find the firmware Audio System service; it owns codec/native-rate conversion. + if (!find_audio_stream_device(ctx)) { + ESP_LOGE(TAG, "audio-stream device not found!"); } // Style the parent screen @@ -800,13 +944,9 @@ static void on_hide(AppHandle app, void* data) { ws_close(fd_to_close); } - // Reset I2S controller to stop DMA and looping noise - if (ctx->i2s_dev != NULL) { - device_lock(ctx->i2s_dev); - i2s_controller_reset(ctx->i2s_dev); - device_unlock(ctx->i2s_dev); - } - + close_input_stream(ctx); + close_output_stream(ctx); + // Wait briefly for tasks to exit int timeout = 100; while ((ctx->worker_task != NULL || ctx->rx_task != NULL) && timeout > 0) { diff --git a/Apps/ReynaBot/main/Source/websocket.c b/Apps/ReynaBot/main/Source/websocket.c index caffac5..586f48b 100644 --- a/Apps/ReynaBot/main/Source/websocket.c +++ b/Apps/ReynaBot/main/Source/websocket.c @@ -56,8 +56,8 @@ int ws_connect(const char* host, int port, const char* path, const char* device_ "Connection: Upgrade\r\n" "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n" "Sec-WebSocket-Version: 13\r\n" - "Authorization: Bearer %s\r\n" - "X-Device-ID: %s\r\n" + "Authorization: Bearer %s\\r\\n" + "X-Device-ID: %s\\r\\n" "\r\n", path, host, port, auth_key, device_id); diff --git a/Apps/ReynaBot/main/Source/websocket.h b/Apps/ReynaBot/main/Source/websocket.h index 396d430..cdc2660 100644 --- a/Apps/ReynaBot/main/Source/websocket.h +++ b/Apps/ReynaBot/main/Source/websocket.h @@ -10,8 +10,8 @@ extern "C" { /** * Connect to a WebSocket server. - * @param host Server IP address (e.g. "192.168.68.126") - * @param port Port number (e.g. 8642) + * @param host Server IP address (for example, the Mac LAN host 192.168.68.112) + * @param port Port number (e.g. 8643 for the Kids gateway) * @param path WebSocket path (e.g. "/api/esp32/voice/ws") * @param device_id Unique device identifier * @param auth_key Hermes Bearer API key diff --git a/Libraries/SfxEngine/Source/SfxEngine.cpp b/Libraries/SfxEngine/Source/SfxEngine.cpp index 917918d..c687526 100644 --- a/Libraries/SfxEngine/Source/SfxEngine.cpp +++ b/Libraries/SfxEngine/Source/SfxEngine.cpp @@ -15,6 +15,10 @@ #include #include "esp_log.h" +// The deployed 0.8.0-dev firmware still exports this legacy lookup API, while +// the current SDK headers only expose device_get_by_name(). +extern "C" Device* device_find_by_name(const char* name); + #ifndef M_PI #define M_PI 3.14159265358979323846f #endif diff --git a/scripts/mp3_player_device_runner.py b/scripts/mp3_player_device_runner.py new file mode 100644 index 0000000..a0842bf --- /dev/null +++ b/scripts/mp3_player_device_runner.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Validate MP3 Player close-and-resume evidence captured from serial.""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + + +def evaluate_resume_trace(trace: str, fixture: str, minimum_position: int = 6) -> dict[str, int]: + """Require real playback, persisted position, and same-file relaunch evidence.""" + escaped = re.escape(fixture) + initial = re.search(rf"Starting MP3 playback: {escaped} .* resume=0\b", trace) + stream_opened = re.search(r"Audio stream opened: \d+ Hz, \d+ channels", trace) + persisted = re.search(rf"History saved on hide: {escaped} pos (\d+) total \d+", trace) + resumed = re.search(rf"Resuming last play {escaped} at (\d+) sec from history", trace) + playback_matches = list(re.finditer(rf"Starting MP3 playback: {escaped} .* resume=(\d+)\b", trace)) + resumed_playback = playback_matches[-1] if playback_matches else None + + if "Playback task stuck, force deleting" in trace: + raise RuntimeError("forced playback task termination invalidates the device test") + + if not initial or not stream_opened: + raise RuntimeError("missing verified playback evidence for the fixture") + if not persisted or not resumed or not resumed_playback: + raise RuntimeError("missing persisted or resumed playback evidence") + + persisted_position = int(persisted.group(1)) + resumed_position = int(resumed.group(1)) + resumed_playback_position = int(resumed_playback.group(1)) + if persisted_position < minimum_position: + raise RuntimeError(f"persisted position {persisted_position} is below {minimum_position}") + if resumed_position != persisted_position or resumed_playback_position != persisted_position: + raise RuntimeError("relaunch did not resume the persisted position on the same file") + return {"persisted_position": persisted_position, "resumed_position": resumed_position} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--trace", type=Path, required=True, help="Captured serial log") + parser.add_argument("--fixture", required=True, help="Absolute fixture path logged by MP3 Player") + parser.add_argument("--minimum-position", type=int, default=6) + args = parser.parse_args() + result = evaluate_resume_trace(args.trace.read_text(encoding="utf-8", errors="replace"), args.fixture, args.minimum_position) + print(f"PASS persisted_position={result['persisted_position']} resumed_position={result['resumed_position']}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_live_captions_contract.py b/tests/test_live_captions_contract.py index 2183247..9587267 100644 --- a/tests/test_live_captions_contract.py +++ b/tests/test_live_captions_contract.py @@ -30,6 +30,15 @@ class LiveCaptionsContractTests(unittest.TestCase): self.assertIn("const char* stop", source) self.assertNotIn("i2s_controller_", source) + def test_ui_autostarts_and_keeps_only_the_latest_thirty_words(self): + source = (APP / "main" / "Source" / "main.c").read_text() + self.assertIn("#define DISPLAY_WORD_LIMIT 50", source) + self.assertIn("copy_recent_words", source) + self.assertIn("tt_lvgl_toolbar_create_for_app", source) + self.assertNotIn("lv_btn_create", source) + self.assertNotIn("start_button", source) + self.assertIn("xTaskCreate(worker_task", source) + def test_connection_failure_stays_failed_until_user_starts_again(self): source = (APP / "main" / "Source" / "main.c").read_text() self.assertIn("CAPTION_FAILED", source) diff --git a/tests/test_mp3_player_contract.py b/tests/test_mp3_player_contract.py new file mode 100644 index 0000000..488e23e --- /dev/null +++ b/tests/test_mp3_player_contract.py @@ -0,0 +1,82 @@ +from pathlib import Path + + +MP3_SOURCE = Path(__file__).parents[1] / "Apps" / "Mp3Player" / "main" / "Source" / "main.c" + + +def test_mp3_player_uses_the_firmware_audio_stream_device_name(): + source = MP3_SOURCE.read_text(encoding="utf-8") + + assert 'device_find_by_name("audio-stream0")' in source + assert 'device_find_by_name("audio-stream")' not in source + assert "device_get_first_by_type" not in source + assert "device_find_first_by_type" not in source + + +def test_mp3_player_restores_a_visible_persisted_volume_control_above_the_six_button_row(): + source = MP3_SOURCE.read_text(encoding="utf-8") + + assert "lv_obj_t* card = lv_obj_create(parent);" not in source + assert "lv_label_set_text(icon, LV_SYMBOL_AUDIO);" not in source + assert "lv_obj_t* vol_box = lv_obj_create(parent);" in source + assert "g_ctx.slider_volume = lv_slider_create(vol_box);" in source + assert "lv_slider_set_value(g_ctx.slider_volume, g_ctx.volume, LV_ANIM_OFF);" in source + assert "on_volume_slider_changed" in source + assert "save_volume(ctx);" in source + assert "lv_obj_set_size(bottom_box, lv_pct(100), 52);" in source + assert "lv_obj_t* btn_close = lv_btn_create(ctrl_box);" in source + assert "lv_obj_t* btn_hist = lv_btn_create(ctrl_box);" in source + assert "lv_label_set_text(lbl_close, LV_SYMBOL_CLOSE);" in source + assert "lv_label_set_text(lbl_hist_btn, LV_SYMBOL_DIRECTORY);" in source + assert "lv_label_set_text(lbl_back, \"-15s\");" in source + assert "lv_label_set_text(lbl_fwd, \"+15s\");" in source + + +def test_mp3_player_folder_button_returns_to_the_files_list(): + source = MP3_SOURCE.read_text(encoding="utf-8") + + library_callback = source.split("static void on_library_click", 1)[1].split("/* ─── App Lifecycle", 1)[0] + assert "tt_app_stop();" not in library_callback + assert "show_history_screen(&g_ctx);" in library_callback + assert "load_first_sd_mp3" not in library_callback + + +def test_mp3_player_history_view_and_fifteen_second_seek_are_present(): + source = MP3_SOURCE.read_text(encoding="utf-8") + + assert "#define SEEK_SECONDS 15" in source + assert 'tt_app_get_user_data_child_path(app, "play_history.txt"' in source + assert "static void show_history_screen" in source + assert "static void on_history_selected" in source + assert "char label[640];" in source + assert "request_seek(ctx, -SEEK_SECONDS);" in source + assert "request_seek(ctx, SEEK_SECONDS);" in source + + +def test_mp3_player_persists_and_resumes_short_tracks_too(): + source = MP3_SOURCE.read_text(encoding="utf-8") + history_selector = source.split("static void on_history_selected", 1)[1].split("static void refresh_history_list", 1)[0] + + assert "Skip history: audio too short" not in source + assert "Resuming last 10min+ play" not in source + assert "Resuming last play %s at %d sec from history" in source + assert "if (he->total_sec >= 600)" not in history_selector + assert "resume_pos = he->pos_sec;" in history_selector + + +def test_mp3_player_close_waits_for_playback_before_stopping_the_app(): + source = MP3_SOURCE.read_text(encoding="utf-8") + + close_callback = source.split("static void on_close_click", 1)[1].split("static void on_library_click", 1)[0] + assert "wait_for_playback_task_to_exit(&g_ctx);" in close_callback + assert close_callback.index("wait_for_playback_task_to_exit(&g_ctx);") < close_callback.index("tt_app_stop();") + assert "tt_lvgl_unlock();" in source + assert "tt_lvgl_lock(portMAX_DELAY);" in source + + +def test_mp3_player_has_a_consumed_dev_autoclose_hook_for_device_tests(): + source = MP3_SOURCE.read_text(encoding="utf-8") + + assert '"mp3player_dev_autoclose_ms"' in source + assert "on_close_click(NULL);" in source + assert "unlink(marker_path);" in source diff --git a/tests/test_mp3_player_device_runner.py b/tests/test_mp3_player_device_runner.py new file mode 100644 index 0000000..302ec23 --- /dev/null +++ b/tests/test_mp3_player_device_runner.py @@ -0,0 +1,69 @@ +import importlib.util +from pathlib import Path + + +RUNNER_PATH = Path(__file__).parents[1] / "scripts" / "mp3_player_device_runner.py" + + +def load_runner(): + spec = importlib.util.spec_from_file_location("mp3_player_device_runner", RUNNER_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_resume_trace_requires_playback_persistence_and_same_path_resume(): + runner = load_runner() + fixture = "/sdcard/download/test.mp3" + trace = "\n".join( + [ + "I Mp3Player: Starting MP3 playback: /sdcard/download/test.mp3 (size: 480000 bytes) resume=0", + "I Mp3Player: Audio stream opened: 16000 Hz, 1 channels", + "I Mp3Player: History saved on hide: /sdcard/download/test.mp3 pos 9 total 30", + "I Mp3Player: Resuming last play /sdcard/download/test.mp3 at 9 sec from history", + "I Mp3Player: Starting MP3 playback: /sdcard/download/test.mp3 (size: 480000 bytes) resume=9", + ] + ) + + result = runner.evaluate_resume_trace(trace, fixture, minimum_position=6) + + assert result == {"persisted_position": 9, "resumed_position": 9} + + +def test_resume_trace_rejects_a_different_file_or_no_playback_evidence(): + runner = load_runner() + fixture = "/sdcard/download/test.mp3" + + try: + runner.evaluate_resume_trace( + "I Mp3Player: Starting MP3 playback: /sdcard/other.mp3 (size: 1 bytes) resume=0", + fixture, + minimum_position=6, + ) + except RuntimeError as exc: + assert "playback" in str(exc).lower() + else: + raise AssertionError("missing playback evidence must fail") + + +def test_resume_trace_rejects_forced_playback_task_termination(): + runner = load_runner() + fixture = "/sdcard/download/test.mp3" + trace = "\n".join( + [ + f"I Mp3Player: Starting MP3 playback: {fixture} (size: 480000 bytes) resume=0", + "I Mp3Player: Audio stream opened: 16000 Hz, 1 channels", + f"I Mp3Player: History saved on hide: {fixture} pos 9 total 180", + "W Mp3Player: Playback task stuck, force deleting", + f"I Mp3Player: Resuming last play {fixture} at 9 sec from history", + f"I Mp3Player: Starting MP3 playback: {fixture} (size: 480000 bytes) resume=9", + ] + ) + + try: + runner.evaluate_resume_trace(trace, fixture, minimum_position=6) + except RuntimeError as exc: + assert "forced" in str(exc).lower() + else: + raise AssertionError("forced task termination must fail the device test")