/* Live Captions: stream mic PCM to Mac mini and save final text on the SD card. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "websocket.h" /* These legacy names are exported by the flashed 0.8.0-dev firmware. */ struct Device* device_find_by_name(const char* name); struct Device* device_find_first_by_type(const struct DeviceType* type); #define TAG "LiveCaptions" #define DEFAULT_ENDPOINT "ws://192.168.68.102:8642/api/esp32/voice/ws" #define DEFAULT_DEVICE_ID "tactility-14c19d1a790" #define PCM_BUFFER_BYTES 1024U #define EVENT_BUFFER_BYTES 4096U typedef enum { CAPTION_IDLE, CAPTION_CONNECTING, CAPTION_LISTENING, CAPTION_PROCESSING, CAPTION_FAILED, } CaptionState; typedef struct { AppHandle app; volatile bool visible; volatile bool capture_audio; volatile bool stop_requested; volatile bool session_active; volatile bool socket_failed; int fd; CaptionState state; char endpoint[128]; char device_id[64]; char api_key[128]; char detail[96]; char caption[768]; char last_final[768]; struct Device* stream_dev; AudioStreamHandle input_handle; TaskHandle_t worker; TaskHandle_t receiver; SemaphoreHandle_t socket_lock; SemaphoreHandle_t audio_lock; lv_obj_t* state_label; lv_obj_t* caption_label; lv_obj_t* start_button; lv_obj_t* stop_button; } CaptionContext; static void update_ui(CaptionContext* ctx) { if (!ctx->visible || !tt_lvgl_lock(pdMS_TO_TICKS(100))) return; const char* state = "READY"; if (ctx->state == CAPTION_CONNECTING) state = "CONNECTING"; else if (ctx->state == CAPTION_LISTENING) state = "LISTENING"; else if (ctx->state == CAPTION_PROCESSING) state = "FINALIZING"; else if (ctx->state == CAPTION_FAILED) state = "FAILED"; lv_label_set_text(ctx->state_label, state); lv_label_set_text(ctx->caption_label, ctx->caption[0] ? ctx->caption : ctx->detail); if (ctx->state == CAPTION_IDLE || ctx->state == CAPTION_FAILED) { lv_obj_clear_state(ctx->start_button, LV_STATE_DISABLED); } else { lv_obj_add_state(ctx->start_button, LV_STATE_DISABLED); } if (ctx->state == CAPTION_LISTENING || ctx->state == CAPTION_PROCESSING) { lv_obj_clear_state(ctx->stop_button, LV_STATE_DISABLED); } else { lv_obj_add_state(ctx->stop_button, LV_STATE_DISABLED); } tt_lvgl_unlock(); } static void set_state(CaptionContext* ctx, CaptionState state, const char* detail) { ctx->state = state; snprintf(ctx->detail, sizeof(ctx->detail), "%s", detail ? detail : ""); update_ui(ctx); } static bool find_audio_stream_device(CaptionContext* ctx) { ctx->stream_dev = device_find_by_name("audio-stream"); if (ctx->stream_dev == NULL) ctx->stream_dev = device_find_first_by_type(&AUDIO_STREAM_TYPE); return ctx->stream_dev != NULL; } static bool open_input_stream(CaptionContext* ctx) { if (ctx->stream_dev == NULL) return false; if (ctx->input_handle != NULL) return true; struct AudioStreamConfig config = {.sample_rate = 16000, .bits_per_sample = 16, .channels = 1}; if (audio_stream_open_input(ctx->stream_dev, &config, &ctx->input_handle) != ERROR_NONE) return false; audio_stream_set_mute(ctx->stream_dev, AUDIO_CODEC_DIR_INPUT, false); audio_stream_set_volume(ctx->stream_dev, AUDIO_CODEC_DIR_INPUT, 100.0f); return true; } static void close_input_stream(CaptionContext* ctx) { if (ctx->input_handle != NULL) { audio_stream_close(ctx->input_handle); ctx->input_handle = NULL; } } static int send_locked(CaptionContext* ctx, const uint8_t* data, size_t length, bool binary) { if (ctx->fd < 0 || xSemaphoreTake(ctx->socket_lock, pdMS_TO_TICKS(500)) != pdTRUE) return -1; int result = ws_send(ctx->fd, data, length, binary); xSemaphoreGive(ctx->socket_lock); return result; } static bool parse_endpoint(const char* url, char* host, size_t host_size, int* port, char* path, size_t path_size) { if (url == NULL || strncmp(url, "ws://", 5) != 0) return false; const char* authority = url + 5; const char* slash = strchr(authority, '/'); const char* end = slash ? slash : authority + strlen(authority); const char* colon = NULL; for (const char* p = authority; p < end; ++p) if (*p == ':') colon = p; size_t host_len = (size_t)((colon ? colon : end) - authority); if (host_len == 0 || host_len >= host_size) return false; memcpy(host, authority, host_len); host[host_len] = '\0'; *port = 80; if (colon != NULL) { *port = atoi(colon + 1); if (*port < 1 || *port > 65535) return false; } const char* wire_path = slash ? slash : "/"; if (strlen(wire_path) >= path_size) return false; snprintf(path, path_size, "%s", wire_path); return true; } static void append_final_caption(const char* text) { if (text == NULL || !*text) return; mkdir("/sdcard/captions", 0755); time_t now = time(NULL); struct tm local; localtime_r(&now, &local); char filename[32]; strftime(filename, sizeof(filename), "%Y-%m-%d.txt", &local); char stamp[16]; strftime(stamp, sizeof(stamp), "%H:%M:%S", &local); char path[128]; snprintf(path, sizeof(path), "/sdcard/captions/%s", filename); FILE* log = fopen(path, "a"); if (log == NULL) { ESP_LOGE(TAG, "could not append %s", path); return; } fprintf(log, "[%s] %s\n", stamp, text); fclose(log); } static void display_caption(CaptionContext* ctx, const char* text, bool final) { if (text == NULL || !*text) return; snprintf(ctx->caption, sizeof(ctx->caption), "%s", text); if (final && strcmp(ctx->last_final, text) != 0) { snprintf(ctx->last_final, sizeof(ctx->last_final), "%s", text); append_final_caption(text); set_state(ctx, CAPTION_IDLE, "Saved final caption to SD card"); } else update_ui(ctx); } static void handle_event(CaptionContext* ctx, const char* json) { cJSON* root = cJSON_Parse(json); if (root == NULL) return; cJSON* event = cJSON_GetObjectItem(root, "event"); cJSON* text = cJSON_GetObjectItem(root, "text"); if (!cJSON_IsString(event)) { cJSON_Delete(root); return; } const char* name = event->valuestring; if (strcmp(name, "ready") == 0) { set_state(ctx, CAPTION_CONNECTING, "Starting caption stream"); } else if (strcmp(name, "listening") == 0) { set_state(ctx, CAPTION_LISTENING, "Listening — press Stop when finished"); } else if (strcmp(name, "draft") == 0 || strcmp(name, "interim_transcript") == 0) { if (cJSON_IsString(text)) display_caption(ctx, text->valuestring, false); } else if (strcmp(name, "transcript") == 0 || strcmp(name, "final") == 0) { cJSON* is_final = cJSON_GetObjectItem(root, "isFinal"); if (cJSON_IsString(text) && (!cJSON_IsBool(is_final) || cJSON_IsTrue(is_final) || strcmp(name, "final") == 0)) { display_caption(ctx, text->valuestring, true); } } else if (strcmp(name, "thinking") == 0) { set_state(ctx, CAPTION_PROCESSING, "Final captioning…"); } else if (strcmp(name, "error") == 0) { ctx->socket_failed = true; set_state(ctx, CAPTION_FAILED, "Fail to connect"); } cJSON_Delete(root); } static void receiver_task(void* argument) { CaptionContext* ctx = argument; uint8_t* buffer = malloc(EVENT_BUFFER_BYTES + 1U); if (buffer == NULL) { ctx->socket_failed = true; ctx->receiver = NULL; vTaskDelete(NULL); } while (ctx->visible && ctx->fd >= 0) { int opcode = 0; bool complete = false; int received = ws_recv(ctx->fd, &opcode, &complete, buffer, EVENT_BUFFER_BYTES); if (received < 0 || !complete) break; if (opcode == 0x01) { buffer[received] = '\0'; handle_event(ctx, (const char*)buffer); } else if (opcode == 0x09) { if (xSemaphoreTake(ctx->socket_lock, pdMS_TO_TICKS(500)) == pdTRUE) { ws_send_pong(ctx->fd, buffer, (size_t)received); xSemaphoreGive(ctx->socket_lock); } } else if (opcode == 0x08) break; } ctx->session_active = false; ctx->receiver = NULL; vTaskDelete(NULL); } static void load_config(CaptionContext* ctx) { snprintf(ctx->endpoint, sizeof(ctx->endpoint), "%s", DEFAULT_ENDPOINT); snprintf(ctx->device_id, sizeof(ctx->device_id), "%s", DEFAULT_DEVICE_ID); ctx->api_key[0] = '\0'; char path[256]; size_t size = sizeof(path); tt_app_get_user_data_child_path(ctx->app, "config.json", path, &size); FILE* file = fopen(path, "r"); if (file == NULL) return; char raw[512]; size_t bytes = fread(raw, 1, sizeof(raw) - 1, file); fclose(file); raw[bytes] = '\0'; cJSON* root = cJSON_Parse(raw); cJSON* endpoint = root ? cJSON_GetObjectItem(root, "server_url") : NULL; cJSON* device = root ? cJSON_GetObjectItem(root, "device_id") : NULL; cJSON* key = root ? cJSON_GetObjectItem(root, "api_key") : NULL; if (cJSON_IsString(endpoint)) snprintf(ctx->endpoint, sizeof(ctx->endpoint), "%s", endpoint->valuestring); if (cJSON_IsString(device)) snprintf(ctx->device_id, sizeof(ctx->device_id), "%s", device->valuestring); if (cJSON_IsString(key)) snprintf(ctx->api_key, sizeof(ctx->api_key), "%s", key->valuestring); cJSON_Delete(root); } static void worker_task(void* argument) { CaptionContext* ctx = argument; char host[64], path[96]; int port = 0; if (!parse_endpoint(ctx->endpoint, host, sizeof(host), &port, path, sizeof(path))) { set_state(ctx, CAPTION_FAILED, "Fail to connect"); ctx->worker = NULL; vTaskDelete(NULL); } set_state(ctx, CAPTION_CONNECTING, "Connecting to Mac mini"); ctx->fd = ws_connect(host, port, path, ctx->device_id, ctx->api_key); if (ctx->fd < 0) { set_state(ctx, CAPTION_FAILED, "Fail to connect"); ctx->worker = NULL; vTaskDelete(NULL); } char start[256]; snprintf(start, sizeof(start), "{\"event\":\"start\",\"device_id\":\"%s\",\"format\":\"pcm_s16le\",\"sample_rate\":16000,\"channels\":1,\"sample_width\":2,\"session_id\":\"cap-%08lx\"}", ctx->device_id, (unsigned long)esp_random()); if (send_locked(ctx, (const uint8_t*)start, strlen(start), false) != 0 || !open_input_stream(ctx)) { set_state(ctx, CAPTION_FAILED, "Fail to connect"); ws_close(ctx->fd); ctx->fd = -1; ctx->worker = NULL; vTaskDelete(NULL); } ctx->session_active = true; xTaskCreate(receiver_task, "caption_rx", 6144, ctx, 6, &ctx->receiver); set_state(ctx, CAPTION_LISTENING, "Listening — press Stop when finished"); uint8_t pcm[PCM_BUFFER_BYTES]; while (ctx->visible && ctx->capture_audio && !ctx->socket_failed) { size_t bytes = 0; xSemaphoreTake(ctx->audio_lock, portMAX_DELAY); error_t result = audio_stream_read(ctx->input_handle, pcm, sizeof(pcm), &bytes, pdMS_TO_TICKS(100)); xSemaphoreGive(ctx->audio_lock); if (result == ERROR_NONE && bytes > 0 && (bytes % 2U) == 0 && send_locked(ctx, pcm, bytes, true) != 0) ctx->socket_failed = true; } xSemaphoreTake(ctx->audio_lock, portMAX_DELAY); close_input_stream(ctx); xSemaphoreGive(ctx->audio_lock); if (ctx->stop_requested && !ctx->socket_failed) { const char* stop = "{\"event\":\"stop\"}"; send_locked(ctx, (const uint8_t*)stop, strlen(stop), false); set_state(ctx, CAPTION_PROCESSING, "Final captioning…"); for (unsigned i = 0; ctx->session_active && i < 150; ++i) vTaskDelay(pdMS_TO_TICKS(100)); } if (ctx->fd >= 0) { ws_send_close(ctx->fd); ws_close(ctx->fd); ctx->fd = -1; } if (ctx->socket_failed && ctx->visible) set_state(ctx, CAPTION_FAILED, "Fail to connect"); else if (ctx->state == CAPTION_PROCESSING) set_state(ctx, CAPTION_IDLE, "No final caption received"); ctx->worker = NULL; vTaskDelete(NULL); } static void on_start(lv_event_t* event) { CaptionContext* ctx = lv_event_get_user_data(event); if (ctx->worker != NULL || ctx->state == CAPTION_LISTENING || ctx->state == CAPTION_PROCESSING) return; ctx->caption[0] = '\0'; ctx->last_final[0] = '\0'; ctx->socket_failed = false; ctx->stop_requested = false; ctx->capture_audio = true; xTaskCreate(worker_task, "caption_tx", 8192, ctx, 5, &ctx->worker); } static void on_stop(lv_event_t* event) { CaptionContext* ctx = lv_event_get_user_data(event); if (ctx->state != CAPTION_LISTENING) return; ctx->capture_audio = false; ctx->stop_requested = true; set_state(ctx, CAPTION_PROCESSING, "Final captioning…"); } static void* create_data(void) { CaptionContext* ctx = calloc(1, sizeof(*ctx)); if (ctx) ctx->fd = -1; return ctx; } static void destroy_data(void* data) { free(data); } static void on_create(AppHandle app, void* data) { ((CaptionContext*)data)->app = app; } static void on_show(AppHandle app, void* data, lv_obj_t* parent) { CaptionContext* ctx = data; ctx->visible = true; load_config(ctx); find_audio_stream_device(ctx); ctx->socket_lock = xSemaphoreCreateMutex(); ctx->audio_lock = xSemaphoreCreateMutex(); tt_lvgl_toolbar_create_for_app(parent, app); ctx->state_label = lv_label_create(parent); lv_obj_align(ctx->state_label, LV_ALIGN_TOP_MID, 0, 38); ctx->caption_label = lv_label_create(parent); lv_obj_set_width(ctx->caption_label, lv_pct(88)); lv_label_set_long_mode(ctx->caption_label, LV_LABEL_LONG_WRAP); lv_obj_set_style_text_align(ctx->caption_label, LV_TEXT_ALIGN_CENTER, 0); lv_obj_align(ctx->caption_label, LV_ALIGN_CENTER, 0, -8); ctx->start_button = lv_btn_create(parent); lv_obj_set_size(ctx->start_button, 100, 42); lv_obj_align(ctx->start_button, LV_ALIGN_BOTTOM_LEFT, 22, -18); lv_obj_t* start_text = lv_label_create(ctx->start_button); lv_label_set_text(start_text, "Start"); lv_obj_center(start_text); lv_obj_add_event_cb(ctx->start_button, on_start, LV_EVENT_CLICKED, ctx); ctx->stop_button = lv_btn_create(parent); lv_obj_set_size(ctx->stop_button, 100, 42); lv_obj_align(ctx->stop_button, LV_ALIGN_BOTTOM_RIGHT, -22, -18); lv_obj_t* stop_text = lv_label_create(ctx->stop_button); lv_label_set_text(stop_text, "Stop"); lv_obj_center(stop_text); lv_obj_add_event_cb(ctx->stop_button, on_stop, LV_EVENT_CLICKED, ctx); if (ctx->stream_dev == NULL || ctx->socket_lock == NULL || ctx->audio_lock == NULL) set_state(ctx, CAPTION_FAILED, "Audio service unavailable"); else set_state(ctx, CAPTION_IDLE, "Press Start to caption"); } static void on_hide(AppHandle app, void* data) { (void)app; CaptionContext* ctx = data; ctx->visible = false; ctx->capture_audio = false; ctx->stop_requested = false; if (ctx->fd >= 0) { ws_send_close(ctx->fd); ws_close(ctx->fd); ctx->fd = -1; } for (unsigned i = 0; (ctx->worker || ctx->receiver) && i < 100; ++i) vTaskDelay(pdMS_TO_TICKS(10)); close_input_stream(ctx); if (ctx->socket_lock) { vSemaphoreDelete(ctx->socket_lock); ctx->socket_lock = NULL; } if (ctx->audio_lock) { vSemaphoreDelete(ctx->audio_lock); ctx->audio_lock = NULL; } } int main(int argc, char* argv[]) { (void)argc; (void)argv; tt_app_register((AppRegistration){.createData=create_data,.destroyData=destroy_data,.onCreate=on_create,.onShow=on_show,.onHide=on_hide}); return 0; }