feat(live-captions): add ESP32 streaming captions app

Apps/LiveCaptions/CMakeLists.txt

@@ -0,0 +1,16 @@
+cmake_minimum_required(VERSION 3.20)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+
+if (DEFINED ENV{TACTILITY_SDK_PATH})
+    set(TACTILITY_SDK_PATH $ENV{TACTILITY_SDK_PATH})
+else()
+    set(TACTILITY_SDK_PATH "../../release/TactilitySDK")
+    message(WARNING "TACTILITY_SDK_PATH is not set, defaulting to ${TACTILITY_SDK_PATH}")
+endif()
+
+include("${TACTILITY_SDK_PATH}/TactilitySDK.cmake")
+set(EXTRA_COMPONENT_DIRS ${TACTILITY_SDK_PATH})
+
+project(LiveCaptions)
+tactility_project(LiveCaptions)

Apps/LiveCaptions/README.md

@@ -0,0 +1,17 @@
+# Live Captions
+
+ESP32-S3 external app that streams 16 kHz signed-16-bit mono microphone PCM to the Mac mini Hermes voice gateway and renders its live draft/final captions. It stores **only final captions** on the device SD card in `/sdcard/captions/YYYY-MM-DD.txt`.
+
+## Configuration
+
+Before packaging, place `config.json` in the app's user-data directory (not source control):
+
+```json
+{
+  "server_url": "ws://192.168.68.102:8642/api/esp32/voice/ws",
+  "device_id": "your-registered-device-id",
+  "api_key": "device-profile-key"
+}
+```
+
+The app never retries automatically: connection failure remains `FAILED` until the user selects **Start** again. **Stop** sends the gateway `stop` event, waits for the final caption, appends it to that day’s text file, then returns to idle.

Apps/LiveCaptions/main/CMakeLists.txt

@@ -0,0 +1,8 @@
+file(GLOB_RECURSE SOURCE_FILES Source/*.c)
+set(CJSON_SOURCE "$ENV{IDF_PATH}/components/json/cJSON/cJSON.c")
+
+idf_component_register(
+    SRCS ${SOURCE_FILES} ${CJSON_SOURCE}
+    INCLUDE_DIRS Source "$ENV{IDF_PATH}/components/json/cJSON"
+    REQUIRES TactilitySDK lwip
+)

Apps/LiveCaptions/main/Source/main.c

@@ -0,0 +1,329 @@
+/* Live Captions: stream mic PCM to Mac mini and save final text on the SD card. */
+#include <tt_app.h>
+#include <tt_lvgl.h>
+#include <tt_lvgl_toolbar.h>
+#include <tactility/device.h>
+#include <tactility/drivers/audio_stream.h>
+
+#include <cJSON.h>
+#include <esp_log.h>
+#include <esp_random.h>
+#include <freertos/FreeRTOS.h>
+#include <freertos/semphr.h>
+#include <freertos/task.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <sys/stat.h>
+
+#include "websocket.h"
+
+/* These legacy names are exported by the flashed 0.8.0-dev firmware. */
+struct Device* device_find_by_name(const char* name);
+struct Device* device_find_first_by_type(const struct DeviceType* type);
+
+#define TAG "LiveCaptions"
+#define DEFAULT_ENDPOINT "ws://192.168.68.102:8642/api/esp32/voice/ws"
+#define DEFAULT_DEVICE_ID "tactility-14c19d1a790"
+#define PCM_BUFFER_BYTES 1024U
+#define EVENT_BUFFER_BYTES 4096U
+
+typedef enum {
+    CAPTION_IDLE,
+    CAPTION_CONNECTING,
+    CAPTION_LISTENING,
+    CAPTION_PROCESSING,
+    CAPTION_FAILED,
+} CaptionState;
+
+typedef struct {
+    AppHandle app;
+    volatile bool visible;
+    volatile bool capture_audio;
+    volatile bool stop_requested;
+    volatile bool session_active;
+    volatile bool socket_failed;
+    int fd;
+    CaptionState state;
+    char endpoint[128];
+    char device_id[64];
+    char api_key[128];
+    char detail[96];
+    char caption[768];
+    char last_final[768];
+    struct Device* stream_dev;
+    AudioStreamHandle input_handle;
+    TaskHandle_t worker;
+    TaskHandle_t receiver;
+    SemaphoreHandle_t socket_lock;
+    SemaphoreHandle_t audio_lock;
+    lv_obj_t* state_label;
+    lv_obj_t* caption_label;
+    lv_obj_t* start_button;
+    lv_obj_t* stop_button;
+} CaptionContext;
+
+static void update_ui(CaptionContext* ctx) {
+    if (!ctx->visible || !tt_lvgl_lock(pdMS_TO_TICKS(100))) return;
+    const char* state = "READY";
+    if (ctx->state == CAPTION_CONNECTING) state = "CONNECTING";
+    else if (ctx->state == CAPTION_LISTENING) state = "LISTENING";
+    else if (ctx->state == CAPTION_PROCESSING) state = "FINALIZING";
+    else if (ctx->state == CAPTION_FAILED) state = "FAILED";
+    lv_label_set_text(ctx->state_label, state);
+    lv_label_set_text(ctx->caption_label, ctx->caption[0] ? ctx->caption : ctx->detail);
+    if (ctx->state == CAPTION_IDLE || ctx->state == CAPTION_FAILED) {
+        lv_obj_clear_state(ctx->start_button, LV_STATE_DISABLED);
+    } else {
+        lv_obj_add_state(ctx->start_button, LV_STATE_DISABLED);
+    }
+    if (ctx->state == CAPTION_LISTENING || ctx->state == CAPTION_PROCESSING) {
+        lv_obj_clear_state(ctx->stop_button, LV_STATE_DISABLED);
+    } else {
+        lv_obj_add_state(ctx->stop_button, LV_STATE_DISABLED);
+    }
+    tt_lvgl_unlock();
+}
+
+static void set_state(CaptionContext* ctx, CaptionState state, const char* detail) {
+    ctx->state = state;
+    snprintf(ctx->detail, sizeof(ctx->detail), "%s", detail ? detail : "");
+    update_ui(ctx);
+}
+
+static bool find_audio_stream_device(CaptionContext* ctx) {
+    ctx->stream_dev = device_find_by_name("audio-stream");
+    if (ctx->stream_dev == NULL) ctx->stream_dev = device_find_first_by_type(&AUDIO_STREAM_TYPE);
+    return ctx->stream_dev != NULL;
+}
+
+static bool open_input_stream(CaptionContext* ctx) {
+    if (ctx->stream_dev == NULL) return false;
+    if (ctx->input_handle != NULL) return true;
+    struct AudioStreamConfig config = {.sample_rate = 16000, .bits_per_sample = 16, .channels = 1};
+    if (audio_stream_open_input(ctx->stream_dev, &config, &ctx->input_handle) != ERROR_NONE) return false;
+    audio_stream_set_mute(ctx->stream_dev, AUDIO_CODEC_DIR_INPUT, false);
+    audio_stream_set_volume(ctx->stream_dev, AUDIO_CODEC_DIR_INPUT, 100.0f);
+    return true;
+}
+
+static void close_input_stream(CaptionContext* ctx) {
+    if (ctx->input_handle != NULL) {
+        audio_stream_close(ctx->input_handle);
+        ctx->input_handle = NULL;
+    }
+}
+
+static int send_locked(CaptionContext* ctx, const uint8_t* data, size_t length, bool binary) {
+    if (ctx->fd < 0 || xSemaphoreTake(ctx->socket_lock, pdMS_TO_TICKS(500)) != pdTRUE) return -1;
+    int result = ws_send(ctx->fd, data, length, binary);
+    xSemaphoreGive(ctx->socket_lock);
+    return result;
+}
+
+static bool parse_endpoint(const char* url, char* host, size_t host_size, int* port, char* path, size_t path_size) {
+    if (url == NULL || strncmp(url, "ws://", 5) != 0) return false;
+    const char* authority = url + 5;
+    const char* slash = strchr(authority, '/');
+    const char* end = slash ? slash : authority + strlen(authority);
+    const char* colon = NULL;
+    for (const char* p = authority; p < end; ++p) if (*p == ':') colon = p;
+    size_t host_len = (size_t)((colon ? colon : end) - authority);
+    if (host_len == 0 || host_len >= host_size) return false;
+    memcpy(host, authority, host_len); host[host_len] = '\0';
+    *port = 80;
+    if (colon != NULL) {
+        *port = atoi(colon + 1);
+        if (*port < 1 || *port > 65535) return false;
+    }
+    const char* wire_path = slash ? slash : "/";
+    if (strlen(wire_path) >= path_size) return false;
+    snprintf(path, path_size, "%s", wire_path);
+    return true;
+}
+
+static void append_final_caption(const char* text) {
+    if (text == NULL || !*text) return;
+    mkdir("/sdcard/captions", 0755);
+    time_t now = time(NULL);
+    struct tm local;
+    localtime_r(&now, &local);
+    char filename[32];
+    strftime(filename, sizeof(filename), "%Y-%m-%d.txt", &local);
+    char stamp[16];
+    strftime(stamp, sizeof(stamp), "%H:%M:%S", &local);
+    char path[128];
+    snprintf(path, sizeof(path), "/sdcard/captions/%s", filename);
+    FILE* log = fopen(path, "a");
+    if (log == NULL) { ESP_LOGE(TAG, "could not append %s", path); return; }
+    fprintf(log, "[%s] %s\n", stamp, text);
+    fclose(log);
+}
+
+static void display_caption(CaptionContext* ctx, const char* text, bool final) {
+    if (text == NULL || !*text) return;
+    snprintf(ctx->caption, sizeof(ctx->caption), "%s", text);
+    if (final && strcmp(ctx->last_final, text) != 0) {
+        snprintf(ctx->last_final, sizeof(ctx->last_final), "%s", text);
+        append_final_caption(text);
+        set_state(ctx, CAPTION_IDLE, "Saved final caption to SD card");
+    } else update_ui(ctx);
+}
+
+static void handle_event(CaptionContext* ctx, const char* json) {
+    cJSON* root = cJSON_Parse(json);
+    if (root == NULL) return;
+    cJSON* event = cJSON_GetObjectItem(root, "event");
+    cJSON* text = cJSON_GetObjectItem(root, "text");
+    if (!cJSON_IsString(event)) { cJSON_Delete(root); return; }
+    const char* name = event->valuestring;
+    if (strcmp(name, "ready") == 0) {
+        set_state(ctx, CAPTION_CONNECTING, "Starting caption stream");
+    } else if (strcmp(name, "listening") == 0) {
+        set_state(ctx, CAPTION_LISTENING, "Listening — press Stop when finished");
+    } else if (strcmp(name, "draft") == 0 || strcmp(name, "interim_transcript") == 0) {
+        if (cJSON_IsString(text)) display_caption(ctx, text->valuestring, false);
+    } else if (strcmp(name, "transcript") == 0 || strcmp(name, "final") == 0) {
+        cJSON* is_final = cJSON_GetObjectItem(root, "isFinal");
+        if (cJSON_IsString(text) && (!cJSON_IsBool(is_final) || cJSON_IsTrue(is_final) || strcmp(name, "final") == 0)) {
+            display_caption(ctx, text->valuestring, true);
+        }
+    } else if (strcmp(name, "thinking") == 0) {
+        set_state(ctx, CAPTION_PROCESSING, "Final captioning…");
+    } else if (strcmp(name, "error") == 0) {
+        ctx->socket_failed = true;
+        set_state(ctx, CAPTION_FAILED, "Fail to connect");
+    }
+    cJSON_Delete(root);
+}
+
+static void receiver_task(void* argument) {
+    CaptionContext* ctx = argument;
+    uint8_t* buffer = malloc(EVENT_BUFFER_BYTES + 1U);
+    if (buffer == NULL) { ctx->socket_failed = true; ctx->receiver = NULL; vTaskDelete(NULL); }
+    while (ctx->visible && ctx->fd >= 0) {
+        int opcode = 0; bool complete = false;
+        int received = ws_recv(ctx->fd, &opcode, &complete, buffer, EVENT_BUFFER_BYTES);
+        if (received < 0 || !complete) break;
+        if (opcode == 0x01) { buffer[received] = '\0'; handle_event(ctx, (const char*)buffer); }
+        else if (opcode == 0x09) {
+            if (xSemaphoreTake(ctx->socket_lock, pdMS_TO_TICKS(500)) == pdTRUE) {
+                ws_send_pong(ctx->fd, buffer, (size_t)received); xSemaphoreGive(ctx->socket_lock);
+            }
+        } else if (opcode == 0x08) break;
+    }
+    ctx->session_active = false;
+    ctx->receiver = NULL;
+    vTaskDelete(NULL);
+}
+
+static void load_config(CaptionContext* ctx) {
+    snprintf(ctx->endpoint, sizeof(ctx->endpoint), "%s", DEFAULT_ENDPOINT);
+    snprintf(ctx->device_id, sizeof(ctx->device_id), "%s", DEFAULT_DEVICE_ID);
+    ctx->api_key[0] = '\0';
+    char path[256]; size_t size = sizeof(path);
+    tt_app_get_user_data_child_path(ctx->app, "config.json", path, &size);
+    FILE* file = fopen(path, "r");
+    if (file == NULL) return;
+    char raw[512]; size_t bytes = fread(raw, 1, sizeof(raw) - 1, file); fclose(file); raw[bytes] = '\0';
+    cJSON* root = cJSON_Parse(raw);
+    cJSON* endpoint = root ? cJSON_GetObjectItem(root, "server_url") : NULL;
+    cJSON* device = root ? cJSON_GetObjectItem(root, "device_id") : NULL;
+    cJSON* key = root ? cJSON_GetObjectItem(root, "api_key") : NULL;
+    if (cJSON_IsString(endpoint)) snprintf(ctx->endpoint, sizeof(ctx->endpoint), "%s", endpoint->valuestring);
+    if (cJSON_IsString(device)) snprintf(ctx->device_id, sizeof(ctx->device_id), "%s", device->valuestring);
+    if (cJSON_IsString(key)) snprintf(ctx->api_key, sizeof(ctx->api_key), "%s", key->valuestring);
+    cJSON_Delete(root);
+}
+
+static void worker_task(void* argument) {
+    CaptionContext* ctx = argument;
+    char host[64], path[96]; int port = 0;
+    if (!parse_endpoint(ctx->endpoint, host, sizeof(host), &port, path, sizeof(path))) {
+        set_state(ctx, CAPTION_FAILED, "Fail to connect"); ctx->worker = NULL; vTaskDelete(NULL);
+    }
+    set_state(ctx, CAPTION_CONNECTING, "Connecting to Mac mini");
+    ctx->fd = ws_connect(host, port, path, ctx->device_id, ctx->api_key);
+    if (ctx->fd < 0) { set_state(ctx, CAPTION_FAILED, "Fail to connect"); ctx->worker = NULL; vTaskDelete(NULL); }
+    char start[256];
+    snprintf(start, sizeof(start), "{\"event\":\"start\",\"device_id\":\"%s\",\"format\":\"pcm_s16le\",\"sample_rate\":16000,\"channels\":1,\"sample_width\":2,\"session_id\":\"cap-%08lx\"}", ctx->device_id, (unsigned long)esp_random());
+    if (send_locked(ctx, (const uint8_t*)start, strlen(start), false) != 0 || !open_input_stream(ctx)) {
+        set_state(ctx, CAPTION_FAILED, "Fail to connect"); ws_close(ctx->fd); ctx->fd = -1; ctx->worker = NULL; vTaskDelete(NULL);
+    }
+    ctx->session_active = true;
+    xTaskCreate(receiver_task, "caption_rx", 6144, ctx, 6, &ctx->receiver);
+    set_state(ctx, CAPTION_LISTENING, "Listening — press Stop when finished");
+    uint8_t pcm[PCM_BUFFER_BYTES];
+    while (ctx->visible && ctx->capture_audio && !ctx->socket_failed) {
+        size_t bytes = 0;
+        xSemaphoreTake(ctx->audio_lock, portMAX_DELAY);
+        error_t result = audio_stream_read(ctx->input_handle, pcm, sizeof(pcm), &bytes, pdMS_TO_TICKS(100));
+        xSemaphoreGive(ctx->audio_lock);
+        if (result == ERROR_NONE && bytes > 0 && (bytes % 2U) == 0 && send_locked(ctx, pcm, bytes, true) != 0) ctx->socket_failed = true;
+    }
+    xSemaphoreTake(ctx->audio_lock, portMAX_DELAY); close_input_stream(ctx); xSemaphoreGive(ctx->audio_lock);
+    if (ctx->stop_requested && !ctx->socket_failed) {
+        const char* stop = "{\"event\":\"stop\"}";
+        send_locked(ctx, (const uint8_t*)stop, strlen(stop), false);
+        set_state(ctx, CAPTION_PROCESSING, "Final captioning…");
+        for (unsigned i = 0; ctx->session_active && i < 150; ++i) vTaskDelay(pdMS_TO_TICKS(100));
+    }
+    if (ctx->fd >= 0) { ws_send_close(ctx->fd); ws_close(ctx->fd); ctx->fd = -1; }
+    if (ctx->socket_failed && ctx->visible) set_state(ctx, CAPTION_FAILED, "Fail to connect");
+    else if (ctx->state == CAPTION_PROCESSING) set_state(ctx, CAPTION_IDLE, "No final caption received");
+    ctx->worker = NULL;
+    vTaskDelete(NULL);
+}
+
+static void on_start(lv_event_t* event) {
+    CaptionContext* ctx = lv_event_get_user_data(event);
+    if (ctx->worker != NULL || ctx->state == CAPTION_LISTENING || ctx->state == CAPTION_PROCESSING) return;
+    ctx->caption[0] = '\0'; ctx->last_final[0] = '\0'; ctx->socket_failed = false; ctx->stop_requested = false; ctx->capture_audio = true;
+    xTaskCreate(worker_task, "caption_tx", 8192, ctx, 5, &ctx->worker);
+}
+
+static void on_stop(lv_event_t* event) {
+    CaptionContext* ctx = lv_event_get_user_data(event);
+    if (ctx->state != CAPTION_LISTENING) return;
+    ctx->capture_audio = false; ctx->stop_requested = true;
+    set_state(ctx, CAPTION_PROCESSING, "Final captioning…");
+}
+
+static void* create_data(void) { CaptionContext* ctx = calloc(1, sizeof(*ctx)); if (ctx) ctx->fd = -1; return ctx; }
+static void destroy_data(void* data) { free(data); }
+static void on_create(AppHandle app, void* data) { ((CaptionContext*)data)->app = app; }
+
+static void on_show(AppHandle app, void* data, lv_obj_t* parent) {
+    CaptionContext* ctx = data; ctx->visible = true; load_config(ctx); find_audio_stream_device(ctx);
+    ctx->socket_lock = xSemaphoreCreateMutex(); ctx->audio_lock = xSemaphoreCreateMutex();
+    tt_lvgl_toolbar_create_for_app(parent, app);
+    ctx->state_label = lv_label_create(parent); lv_obj_align(ctx->state_label, LV_ALIGN_TOP_MID, 0, 38);
+    ctx->caption_label = lv_label_create(parent); lv_obj_set_width(ctx->caption_label, lv_pct(88));
+    lv_label_set_long_mode(ctx->caption_label, LV_LABEL_LONG_WRAP); lv_obj_set_style_text_align(ctx->caption_label, LV_TEXT_ALIGN_CENTER, 0);
+    lv_obj_align(ctx->caption_label, LV_ALIGN_CENTER, 0, -8);
+    ctx->start_button = lv_btn_create(parent); lv_obj_set_size(ctx->start_button, 100, 42); lv_obj_align(ctx->start_button, LV_ALIGN_BOTTOM_LEFT, 22, -18);
+    lv_obj_t* start_text = lv_label_create(ctx->start_button); lv_label_set_text(start_text, "Start"); lv_obj_center(start_text);
+    lv_obj_add_event_cb(ctx->start_button, on_start, LV_EVENT_CLICKED, ctx);
+    ctx->stop_button = lv_btn_create(parent); lv_obj_set_size(ctx->stop_button, 100, 42); lv_obj_align(ctx->stop_button, LV_ALIGN_BOTTOM_RIGHT, -22, -18);
+    lv_obj_t* stop_text = lv_label_create(ctx->stop_button); lv_label_set_text(stop_text, "Stop"); lv_obj_center(stop_text);
+    lv_obj_add_event_cb(ctx->stop_button, on_stop, LV_EVENT_CLICKED, ctx);
+    if (ctx->stream_dev == NULL || ctx->socket_lock == NULL || ctx->audio_lock == NULL) set_state(ctx, CAPTION_FAILED, "Audio service unavailable");
+    else set_state(ctx, CAPTION_IDLE, "Press Start to caption");
+}
+
+static void on_hide(AppHandle app, void* data) {
+    (void)app; CaptionContext* ctx = data; ctx->visible = false; ctx->capture_audio = false; ctx->stop_requested = false;
+    if (ctx->fd >= 0) { ws_send_close(ctx->fd); ws_close(ctx->fd); ctx->fd = -1; }
+    for (unsigned i = 0; (ctx->worker || ctx->receiver) && i < 100; ++i) vTaskDelay(pdMS_TO_TICKS(10));
+    close_input_stream(ctx);
+    if (ctx->socket_lock) { vSemaphoreDelete(ctx->socket_lock); ctx->socket_lock = NULL; }
+    if (ctx->audio_lock) { vSemaphoreDelete(ctx->audio_lock); ctx->audio_lock = NULL; }
+}
+
+int main(int argc, char* argv[]) {
+    (void)argc; (void)argv;
+    tt_app_register((AppRegistration){.createData=create_data,.destroyData=destroy_data,.onCreate=on_create,.onShow=on_show,.onHide=on_hide});
+    return 0;
+}

Apps/LiveCaptions/main/Source/websocket.c

@@ -0,0 +1,167 @@
+#include "websocket.h"
+
+#include <string.h>
+#include <stdio.h>
+#include <unistd.h>
+
+#include <esp_log.h>
+#include <esp_random.h>
+#include <lwip/inet.h>
+#include <lwip/sockets.h>
+
+#define TAG "PipecatVoiceWs"
+#define WS_HEADER_LIMIT 1024U
+#define WS_CONTROL_LIMIT 125U
+
+static int send_all(int fd, const uint8_t* data, size_t length) {
+    size_t sent = 0;
+    while (sent < length) {
+        int result = lwip_send(fd, data + sent, length - sent, 0);
+        if (result <= 0) return -1;
+        sent += (size_t)result;
+    }
+    return 0;
+}
+
+static int recv_all(int fd, uint8_t* data, size_t length) {
+    size_t received = 0;
+    while (received < length) {
+        int result = lwip_recv(fd, data + received, length - received, 0);
+        if (result <= 0) return -1;
+        received += (size_t)result;
+    }
+    return 0;
+}
+
+static int discard(int fd, uint64_t length) {
+    uint8_t buffer[256];
+    while (length > 0) {
+        size_t chunk = length > sizeof(buffer) ? sizeof(buffer) : (size_t)length;
+        if (recv_all(fd, buffer, chunk) < 0) return -1;
+        length -= chunk;
+    }
+    return 0;
+}
+
+static int send_frame(int fd, uint8_t opcode, const uint8_t* payload, size_t length) {
+    if (length > 65535U || ((opcode & 0x08U) && length > WS_CONTROL_LIMIT)) return -1;
+    uint8_t header[8];
+    size_t header_length = 2;
+    header[0] = 0x80U | opcode;
+    if (length < 126U) {
+        header[1] = 0x80U | (uint8_t)length;
+    } else {
+        header[1] = 0x80U | 126U;
+        header[2] = (uint8_t)(length >> 8U);
+        header[3] = (uint8_t)length;
+        header_length = 4;
+    }
+    uint8_t mask[4];
+    uint32_t random = esp_random();
+    memcpy(mask, &random, sizeof(mask));
+    memcpy(header + header_length, mask, sizeof(mask));
+    header_length += sizeof(mask);
+    if (send_all(fd, header, header_length) < 0) return -1;
+
+    uint8_t chunk[512];
+    size_t offset = 0;
+    while (offset < length) {
+        size_t count = length - offset > sizeof(chunk) ? sizeof(chunk) : length - offset;
+        for (size_t i = 0; i < count; ++i) chunk[i] = payload[offset + i] ^ mask[(offset + i) % sizeof(mask)];
+        if (send_all(fd, chunk, count) < 0) return -1;
+        offset += count;
+    }
+    return 0;
+}
+
+int ws_connect(const char* host, int port, const char* path, const char* device_id, const char* api_key) {
+    if (host == NULL || path == NULL || device_id == NULL || api_key == NULL || port < 1 || port > 65535) return -1;
+    int fd = lwip_socket(AF_INET, SOCK_STREAM, 0);
+    if (fd < 0) {
+        ESP_LOGW(TAG, "socket create failed");
+        return -1;
+    }
+    struct sockaddr_in address = {0};
+    address.sin_family = AF_INET;
+    address.sin_port = htons((uint16_t)port);
+    address.sin_addr.s_addr = ipaddr_addr(host);
+    if (address.sin_addr.s_addr == IPADDR_NONE) {
+        ESP_LOGW(TAG, "endpoint address parse failed");
+        close(fd);
+        return -1;
+    }
+    if (lwip_connect(fd, (struct sockaddr*)&address, sizeof(address)) < 0) {
+        ESP_LOGW(TAG, "TCP connect failed");
+        close(fd);
+        return -1;
+    }
+    struct timeval timeout = {.tv_sec = 15, .tv_usec = 0};
+    lwip_setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout));
+    char request[WS_HEADER_LIMIT];
+    int request_length = snprintf(request, sizeof(request),
+        "GET %s HTTP/1.1\r\nHost: %s:%d\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n"
+        "Sec-WebSocket-Key: MDEyMzQ1Njc4OWFiY2RlZg==\r\nSec-WebSocket-Version: 13\r\n"
+        "Authorization: Bearer %s\r\nX-Device-ID: %s\r\n\r\n",
+        path, host, port, api_key, device_id);
+    if (request_length < 0 || (size_t)request_length >= sizeof(request) || send_all(fd, (const uint8_t*)request, (size_t)request_length) < 0) {
+        ESP_LOGW(TAG, "WebSocket upgrade request failed");
+        close(fd);
+        return -1;
+    }
+    char response[WS_HEADER_LIMIT];
+    size_t length = 0;
+    while (length + 1 < sizeof(response)) {
+        if (recv_all(fd, (uint8_t*)&response[length], 1) < 0) {
+            ESP_LOGW(TAG, "WebSocket upgrade response failed");
+            close(fd);
+            return -1;
+        }
+        response[++length] = '\0';
+        if (length >= 4 && memcmp(response + length - 4, "\r\n\r\n", 4) == 0) break;
+    }
+    if (length + 1 >= sizeof(response) || strstr(response, " 101 ") == NULL) {
+        ESP_LOGW(TAG, "WebSocket upgrade rejected");
+        close(fd);
+        return -1;
+    }
+    ESP_LOGI(TAG, "WebSocket upgrade accepted");
+    return fd;
+}
+
+int ws_send(int fd, const uint8_t* data, size_t length, bool binary) {
+    if (fd < 0 || data == NULL || length == 0) return -1;
+    return send_frame(fd, binary ? 0x02U : 0x01U, data, length);
+}
+
+int ws_recv(int fd, int* opcode, bool* final, uint8_t* payload, size_t maximum) {
+    uint8_t header[2];
+    if (fd < 0 || recv_all(fd, header, sizeof(header)) < 0) return -1;
+    uint64_t length = header[1] & 0x7fU;
+    if (length == 126U) {
+        uint8_t extended[2];
+        if (recv_all(fd, extended, sizeof(extended)) < 0) return -1;
+        length = ((uint64_t)extended[0] << 8U) | extended[1];
+    } else if (length == 127U) {
+        uint8_t extended[8];
+        if (recv_all(fd, extended, sizeof(extended)) < 0) return -1;
+        length = 0;
+        for (size_t i = 0; i < sizeof(extended); ++i) length = (length << 8U) | extended[i];
+    }
+    bool masked = (header[1] & 0x80U) != 0;
+    uint8_t mask[4] = {0};
+    if (masked && recv_all(fd, mask, sizeof(mask)) < 0) return -1;
+    uint8_t frame_opcode = header[0] & 0x0fU;
+    if (((frame_opcode & 0x08U) && (length > WS_CONTROL_LIMIT || !(header[0] & 0x80U))) || length > maximum) {
+        if (discard(fd, length) < 0) return -1;
+        return -2;
+    }
+    if (length > 0 && recv_all(fd, payload, (size_t)length) < 0) return -1;
+    if (masked) for (size_t i = 0; i < (size_t)length; ++i) payload[i] ^= mask[i % sizeof(mask)];
+    if (opcode) *opcode = frame_opcode;
+    if (final) *final = (header[0] & 0x80U) != 0;
+    return (int)length;
+}
+
+int ws_send_pong(int fd, const uint8_t* payload, size_t length) { return send_frame(fd, 0x0aU, payload, length); }
+int ws_send_close(int fd) { return send_frame(fd, 0x08U, NULL, 0); }
+void ws_close(int fd) { if (fd >= 0) close(fd); }

Apps/LiveCaptions/main/Source/websocket.h

@@ -0,0 +1,62 @@
+#pragma once
+
+#include <stdint.h>
+#include <stddef.h>
+#include <stdbool.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Connect to a WebSocket server.
+ * @param host Server IP address (e.g. "192.168.68.126")
+ * @param port Port number (e.g. 8642)
+ * @param path WebSocket path (e.g. "/api/esp32/voice/ws")
+ * @param device_id Unique device identifier
+ * @param api_key Optional profile API key; never compiled into firmware
+ * @return Socket file descriptor on success, or -1 on failure
+ */
+int ws_connect(const char* host, int port, const char* path, const char* device_id, const char* api_key);
+
+/**
+ * Send a WebSocket frame.
+ * @param fd Socket file descriptor
+ * @param data Data payload to send
+ * @param len Length of the data payload
+ * @param binary True for binary frame, false for text frame
+ * @return 0 on success, or -1 on failure
+ */
+int ws_send(int fd, const uint8_t* data, size_t len, bool binary);
+
+/**
+ * Receive a WebSocket frame.
+ * @param fd Socket file descriptor
+ * @param out_opcode Pointer to store the received opcode (e.g. 0x01 text, 0x02 binary)
+ * @param payload Buffer to store the received payload
+ * @param max_len Maximum length of the payload buffer
+ * @return Received payload length on success, -1 on connection failure, or -2 on buffer overflow
+ */
+int ws_recv(int fd, int* out_opcode, bool* out_final, uint8_t* payload, size_t max_len);
+
+/**
+ * Close a WebSocket connection.
+ * @param fd Socket file descriptor
+ */
+void ws_close(int fd);
+
+/**
+ * Send a WebSocket PONG frame.
+ * @param fd Socket file descriptor
+ * @param payload Payload to reflect
+ * @param len Length of payload
+ * @return 0 on success, or -1 on failure
+ */
+int ws_send_pong(int fd, const uint8_t* payload, size_t len);
+
+/** Send a clean WebSocket close control frame before closing the socket. */
+int ws_send_close(int fd);
+
+#ifdef __cplusplus
+}
+#endif

Apps/LiveCaptions/manifest.properties

@@ -0,0 +1,7 @@
+manifest.version=0.2
+target.sdk=0.8.0-dev
+target.platforms=esp32s3
+app.id=one.tactility.livecaptions
+app.version.name=1.0.0
+app.version.code=1
+app.name=Live Captions

tests/test_live_captions_contract.py

@@ -0,0 +1,42 @@
+"""Source-level contract for the Live Captions external Tactility app.
+
+The app is ELF-loaded firmware code, so this test guards the device protocol and
+persistence requirements before an ESP32-S3 build/install test is available.
+"""
+from pathlib import Path
+import unittest
+
+ROOT = Path(__file__).resolve().parents[1]
+APP = ROOT / "Apps" / "LiveCaptions"
+
+
+class LiveCaptionsContractTests(unittest.TestCase):
+    def test_manifest_identifies_the_captions_app(self):
+        manifest = (APP / "manifest.properties").read_text()
+        self.assertIn("app.id=one.tactility.livecaptions", manifest)
+        self.assertIn("app.name=Live Captions", manifest)
+        self.assertIn("target.platforms=esp32s3", manifest)
+
+    def test_app_streams_audio_and_logs_final_caption_to_daily_text_file(self):
+        source = (APP / "main" / "Source" / "main.c").read_text()
+        self.assertIn("audio_stream_open_input", source)
+        self.assertIn("/sdcard/captions", source)
+        self.assertIn('"%Y-%m-%d.txt"', source)
+        self.assertIn("append_final_caption", source)
+        self.assertIn('"draft"', source)
+        self.assertIn('"interim_transcript"', source)
+        self.assertIn('"transcript"', source)
+        self.assertIn('"final"', source)
+        self.assertIn("const char* stop", source)
+        self.assertNotIn("i2s_controller_", source)
+
+    def test_connection_failure_stays_failed_until_user_starts_again(self):
+        source = (APP / "main" / "Source" / "main.c").read_text()
+        self.assertIn("CAPTION_FAILED", source)
+        self.assertIn("Fail to connect", source)
+        self.assertNotIn("retry_attempt", source)
+        self.assertNotIn("retry scheduled", source)
+
+
+if __name__ == "__main__":
+    unittest.main()