adolforeyna/tactility_apps
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: checkpoint app work before firmware sync

Browse Source
This commit is contained in:
Adolfo Reyna
2026-09-07 23:01:26 -04:00
parent e42036a044
commit 59612020bb
20 changed files with 2111 additions and 1218 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Apps/LiveCaptions/main/Source/main.c
+54 -55
View File
@@ -25,10 +25,11 @@ struct Device* device_find_by_name(const char* name);
struct Device* device_find_first_by_type(const struct DeviceType* type);
#define TAG "LiveCaptions"
#define DEFAULT_ENDPOINT "ws://192.168.68.102:8642/api/esp32/voice/ws"
#define DEFAULT_ENDPOINT "ws://192.168.68.102:8645/api/esp32/captions/ws"
#define DEFAULT_DEVICE_ID "tactility-14c19d1a790"
#define PCM_BUFFER_BYTES 1024U
#define EVENT_BUFFER_BYTES 4096U
#define DISPLAY_WORD_LIMIT 50
typedef enum {
CAPTION_IDLE,
@@ -59,31 +60,15 @@ typedef struct {
TaskHandle_t receiver;
SemaphoreHandle_t socket_lock;
SemaphoreHandle_t audio_lock;
lv_obj_t* state_label;
lv_obj_t* caption_label;
lv_obj_t* start_button;
lv_obj_t* stop_button;
lv_obj_t* status_label;
} CaptionContext;
static void update_ui(CaptionContext* ctx) {
if (!ctx->visible || !tt_lvgl_lock(pdMS_TO_TICKS(100))) return;
const char* state = "READY";
if (ctx->state == CAPTION_CONNECTING) state = "CONNECTING";
else if (ctx->state == CAPTION_LISTENING) state = "LISTENING";
else if (ctx->state == CAPTION_PROCESSING) state = "FINALIZING";
else if (ctx->state == CAPTION_FAILED) state = "FAILED";
lv_label_set_text(ctx->state_label, state);
lv_label_set_text(ctx->caption_label, ctx->caption[0] ? ctx->caption : ctx->detail);
if (ctx->state == CAPTION_IDLE || ctx->state == CAPTION_FAILED) {
lv_obj_clear_state(ctx->start_button, LV_STATE_DISABLED);
} else {
lv_obj_add_state(ctx->start_button, LV_STATE_DISABLED);
}
if (ctx->state == CAPTION_LISTENING || ctx->state == CAPTION_PROCESSING) {
lv_obj_clear_state(ctx->stop_button, LV_STATE_DISABLED);
} else {
lv_obj_add_state(ctx->stop_button, LV_STATE_DISABLED);
}
lv_label_set_text(ctx->caption_label, ctx->caption);
const char* status = ctx->state == CAPTION_LISTENING ? "Connected" : (ctx->state == CAPTION_CONNECTING ? "Connecting" : (ctx->state == CAPTION_FAILED ? "Failed — Reconnecting" : "Connecting"));
lv_label_set_text(ctx->status_label, status);
tt_lvgl_unlock();
}
@@ -162,14 +147,29 @@ static void append_final_caption(const char* text) {
fclose(log);
}
static bool is_space_char(char value) { return value == ' ' || value == '\n' || value == '\r' || value == ' '; }
static void copy_recent_words(char* output, size_t output_size, const char* text) {
const char* starts[DISPLAY_WORD_LIMIT];
int count = 0;
bool in_word = false;
for (const char* cursor = text; *cursor; ++cursor) {
if (is_space_char(*cursor)) { in_word = false; continue; }
if (!in_word) { starts[count % DISPLAY_WORD_LIMIT] = cursor; ++count; in_word = true; }
}
const char* first = count > DISPLAY_WORD_LIMIT ? starts[count % DISPLAY_WORD_LIMIT] : text;
snprintf(output, output_size, "%s", first);
}
static void display_caption(CaptionContext* ctx, const char* text, bool final) {
if (text == NULL || !*text) return;
snprintf(ctx->caption, sizeof(ctx->caption), "%s", text);
copy_recent_words(ctx->caption, sizeof(ctx->caption), text);
if (final && strcmp(ctx->last_final, text) != 0) {
snprintf(ctx->last_final, sizeof(ctx->last_final), "%s", text);
append_final_caption(text);
set_state(ctx, CAPTION_IDLE, "Saved final caption to SD card");
} else update_ui(ctx);
ctx->state = CAPTION_IDLE;
}
update_ui(ctx);
}
static void handle_event(CaptionContext* ctx, const char* json) {
@@ -183,7 +183,14 @@ static void handle_event(CaptionContext* ctx, const char* json) {
set_state(ctx, CAPTION_CONNECTING, "Starting caption stream");
} else if (strcmp(name, "listening") == 0) {
set_state(ctx, CAPTION_LISTENING, "Listening — press Stop when finished");
} else if (strcmp(name, "draft") == 0 || strcmp(name, "interim_transcript") == 0) {
} else if (strcmp(name, "state") == 0) {
cJSON* remote_state = cJSON_GetObjectItem(root, "state");
if (cJSON_IsString(remote_state) && strcmp(remote_state->valuestring, "listening") == 0) {
set_state(ctx, CAPTION_LISTENING, "Listening");
} else if (cJSON_IsString(remote_state) && strcmp(remote_state->valuestring, "processing") == 0) {
set_state(ctx, CAPTION_PROCESSING, "Captioning…");
}
} else if (strcmp(name, "draft") == 0 || strcmp(name, "interim_transcript") == 0 || strcmp(name, "review") == 0) {
if (cJSON_IsString(text)) display_caption(ctx, text->valuestring, false);
} else if (strcmp(name, "transcript") == 0 || strcmp(name, "final") == 0) {
cJSON* is_final = cJSON_GetObjectItem(root, "isFinal");
@@ -248,7 +255,7 @@ static void worker_task(void* argument) {
ctx->fd = ws_connect(host, port, path, ctx->device_id, ctx->api_key);
if (ctx->fd < 0) { set_state(ctx, CAPTION_FAILED, "Fail to connect"); ctx->worker = NULL; vTaskDelete(NULL); }
char start[256];
snprintf(start, sizeof(start), "{\"event\":\"start\",\"device_id\":\"%s\",\"format\":\"pcm_s16le\",\"sample_rate\":16000,\"channels\":1,\"sample_width\":2,\"session_id\":\"cap-%08lx\"}", ctx->device_id, (unsigned long)esp_random());
snprintf(start, sizeof(start), "{\"v\":1,\"event\":\"start\",\"session_id\":\"cap-%08lx\",\"device_id\":\"%s\",\"audio\":{\"format\":\"pcm_s16le\",\"sample_rate\":16000,\"channels\":1,\"sample_width\":2}}", (unsigned long)esp_random(), ctx->device_id);
if (send_locked(ctx, (const uint8_t*)start, strlen(start), false) != 0 || !open_input_stream(ctx)) {
set_state(ctx, CAPTION_FAILED, "Fail to connect"); ws_close(ctx->fd); ctx->fd = -1; ctx->worker = NULL; vTaskDelete(NULL);
}
@@ -271,26 +278,17 @@ static void worker_task(void* argument) {
for (unsigned i = 0; ctx->session_active && i < 150; ++i) vTaskDelay(pdMS_TO_TICKS(100));
}
if (ctx->fd >= 0) { ws_send_close(ctx->fd); ws_close(ctx->fd); ctx->fd = -1; }
if (ctx->socket_failed && ctx->visible) set_state(ctx, CAPTION_FAILED, "Fail to connect");
else if (ctx->state == CAPTION_PROCESSING) set_state(ctx, CAPTION_IDLE, "No final caption received");
bool reconnect = ctx->socket_failed && ctx->visible;
if (reconnect) set_state(ctx, CAPTION_FAILED, "Reconnecting");
else if (ctx->state == CAPTION_PROCESSING) set_state(ctx, CAPTION_IDLE, "");
ctx->worker = NULL;
if (reconnect) {
vTaskDelay(pdMS_TO_TICKS(3000));
if (ctx->visible) { ctx->socket_failed = false; ctx->capture_audio = true; xTaskCreate(worker_task, "caption_tx", 8192, ctx, 5, &ctx->worker); }
}
vTaskDelete(NULL);
}
static void on_start(lv_event_t* event) {
CaptionContext* ctx = lv_event_get_user_data(event);
if (ctx->worker != NULL || ctx->state == CAPTION_LISTENING || ctx->state == CAPTION_PROCESSING) return;
ctx->caption[0] = '\0'; ctx->last_final[0] = '\0'; ctx->socket_failed = false; ctx->stop_requested = false; ctx->capture_audio = true;
xTaskCreate(worker_task, "caption_tx", 8192, ctx, 5, &ctx->worker);
}
static void on_stop(lv_event_t* event) {
CaptionContext* ctx = lv_event_get_user_data(event);
if (ctx->state != CAPTION_LISTENING) return;
ctx->capture_audio = false; ctx->stop_requested = true;
set_state(ctx, CAPTION_PROCESSING, "Final captioning…");
}
static void* create_data(void) { CaptionContext* ctx = calloc(1, sizeof(*ctx)); if (ctx) ctx->fd = -1; return ctx; }
static void destroy_data(void* data) { free(data); }
static void on_create(AppHandle app, void* data) { ((CaptionContext*)data)->app = app; }
@@ -298,19 +296,20 @@ static void on_create(AppHandle app, void* data) { ((CaptionContext*)data)->app
static void on_show(AppHandle app, void* data, lv_obj_t* parent) {
CaptionContext* ctx = data; ctx->visible = true; load_config(ctx); find_audio_stream_device(ctx);
ctx->socket_lock = xSemaphoreCreateMutex(); ctx->audio_lock = xSemaphoreCreateMutex();
tt_lvgl_toolbar_create_for_app(parent, app);
ctx->state_label = lv_label_create(parent); lv_obj_align(ctx->state_label, LV_ALIGN_TOP_MID, 0, 38);
ctx->caption_label = lv_label_create(parent); lv_obj_set_width(ctx->caption_label, lv_pct(88));
lv_label_set_long_mode(ctx->caption_label, LV_LABEL_LONG_WRAP); lv_obj_set_style_text_align(ctx->caption_label, LV_TEXT_ALIGN_CENTER, 0);
lv_obj_align(ctx->caption_label, LV_ALIGN_CENTER, 0, -8);
ctx->start_button = lv_btn_create(parent); lv_obj_set_size(ctx->start_button, 100, 42); lv_obj_align(ctx->start_button, LV_ALIGN_BOTTOM_LEFT, 22, -18);
lv_obj_t* start_text = lv_label_create(ctx->start_button); lv_label_set_text(start_text, "Start"); lv_obj_center(start_text);
lv_obj_add_event_cb(ctx->start_button, on_start, LV_EVENT_CLICKED, ctx);
ctx->stop_button = lv_btn_create(parent); lv_obj_set_size(ctx->stop_button, 100, 42); lv_obj_align(ctx->stop_button, LV_ALIGN_BOTTOM_RIGHT, -22, -18);
lv_obj_t* stop_text = lv_label_create(ctx->stop_button); lv_label_set_text(stop_text, "Stop"); lv_obj_center(stop_text);
lv_obj_add_event_cb(ctx->stop_button, on_stop, LV_EVENT_CLICKED, ctx);
if (ctx->stream_dev == NULL || ctx->socket_lock == NULL || ctx->audio_lock == NULL) set_state(ctx, CAPTION_FAILED, "Audio service unavailable");
else set_state(ctx, CAPTION_IDLE, "Press Start to caption");
lv_obj_t* toolbar = tt_lvgl_toolbar_create_for_app(parent, app);
lv_obj_align(toolbar, LV_ALIGN_TOP_MID, 0, 0);
ctx->caption_label = lv_label_create(parent);
lv_obj_set_width(ctx->caption_label, lv_pct(88));
lv_label_set_long_mode(ctx->caption_label, LV_LABEL_LONG_WRAP);
lv_obj_set_style_text_align(ctx->caption_label, LV_TEXT_ALIGN_CENTER, 0);
lv_obj_align(ctx->caption_label, LV_ALIGN_CENTER, 0, 0);
ctx->status_label = lv_label_create(parent);
lv_obj_align(ctx->status_label, LV_ALIGN_BOTTOM_MID, 0, -10);
ctx->caption[0] = '\0';
if (ctx->stream_dev != NULL && ctx->socket_lock != NULL && ctx->audio_lock != NULL) {
ctx->capture_audio = true; ctx->stop_requested = false; ctx->socket_failed = false;
xTaskCreate(worker_task, "caption_tx", 8192, ctx, 5, &ctx->worker);
}
}
static void on_hide(AppHandle app, void* data) {