diff --git a/Apps/LiveCaptions/CMakeLists.txt b/Apps/LiveCaptions/CMakeLists.txt new file mode 100644 index 0000000..42a05a3 --- /dev/null +++ b/Apps/LiveCaptions/CMakeLists.txt @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 3.20) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) + +if (DEFINED ENV{TACTILITY_SDK_PATH}) + set(TACTILITY_SDK_PATH $ENV{TACTILITY_SDK_PATH}) +else() + set(TACTILITY_SDK_PATH "../../release/TactilitySDK") + message(WARNING "TACTILITY_SDK_PATH is not set, defaulting to ${TACTILITY_SDK_PATH}") +endif() + +include("${TACTILITY_SDK_PATH}/TactilitySDK.cmake") +set(EXTRA_COMPONENT_DIRS ${TACTILITY_SDK_PATH}) + +project(LiveCaptions) +tactility_project(LiveCaptions) diff --git a/Apps/LiveCaptions/README.md b/Apps/LiveCaptions/README.md new file mode 100644 index 0000000..bf74a70 --- /dev/null +++ b/Apps/LiveCaptions/README.md @@ -0,0 +1,17 @@ +# Live Captions + +ESP32-S3 external app that streams 16 kHz signed-16-bit mono microphone PCM to the Mac mini Hermes voice gateway and renders its live draft/final captions. It stores **only final captions** on the device SD card in `/sdcard/captions/YYYY-MM-DD.txt`. + +## Configuration + +Before packaging, place `config.json` in the app's user-data directory (not source control): + +```json +{ + "server_url": "ws://192.168.68.102:8642/api/esp32/voice/ws", + "device_id": "your-registered-device-id", + "api_key": "device-profile-key" +} +``` + +The app never retries automatically: connection failure remains `FAILED` until the user selects **Start** again. **Stop** sends the gateway `stop` event, waits for the final caption, appends it to that day’s text file, then returns to idle. diff --git a/Apps/LiveCaptions/main/CMakeLists.txt b/Apps/LiveCaptions/main/CMakeLists.txt new file mode 100644 index 0000000..7d660cd --- /dev/null +++ b/Apps/LiveCaptions/main/CMakeLists.txt @@ -0,0 +1,8 @@ +file(GLOB_RECURSE SOURCE_FILES Source/*.c) +set(CJSON_SOURCE "$ENV{IDF_PATH}/components/json/cJSON/cJSON.c") + +idf_component_register( + SRCS ${SOURCE_FILES} ${CJSON_SOURCE} + INCLUDE_DIRS Source "$ENV{IDF_PATH}/components/json/cJSON" + REQUIRES TactilitySDK lwip +) diff --git a/Apps/LiveCaptions/main/Source/main.c b/Apps/LiveCaptions/main/Source/main.c new file mode 100644 index 0000000..05ce6c5 --- /dev/null +++ b/Apps/LiveCaptions/main/Source/main.c @@ -0,0 +1,329 @@ +/* Live Captions: stream mic PCM to Mac mini and save final text on the SD card. */ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "websocket.h" + +/* These legacy names are exported by the flashed 0.8.0-dev firmware. */ +struct Device* device_find_by_name(const char* name); +struct Device* device_find_first_by_type(const struct DeviceType* type); + +#define TAG "LiveCaptions" +#define DEFAULT_ENDPOINT "ws://192.168.68.102:8642/api/esp32/voice/ws" +#define DEFAULT_DEVICE_ID "tactility-14c19d1a790" +#define PCM_BUFFER_BYTES 1024U +#define EVENT_BUFFER_BYTES 4096U + +typedef enum { + CAPTION_IDLE, + CAPTION_CONNECTING, + CAPTION_LISTENING, + CAPTION_PROCESSING, + CAPTION_FAILED, +} CaptionState; + +typedef struct { + AppHandle app; + volatile bool visible; + volatile bool capture_audio; + volatile bool stop_requested; + volatile bool session_active; + volatile bool socket_failed; + int fd; + CaptionState state; + char endpoint[128]; + char device_id[64]; + char api_key[128]; + char detail[96]; + char caption[768]; + char last_final[768]; + struct Device* stream_dev; + AudioStreamHandle input_handle; + TaskHandle_t worker; + TaskHandle_t receiver; + SemaphoreHandle_t socket_lock; + SemaphoreHandle_t audio_lock; + lv_obj_t* state_label; + lv_obj_t* caption_label; + lv_obj_t* start_button; + lv_obj_t* stop_button; +} CaptionContext; + +static void update_ui(CaptionContext* ctx) { + if (!ctx->visible || !tt_lvgl_lock(pdMS_TO_TICKS(100))) return; + const char* state = "READY"; + if (ctx->state == CAPTION_CONNECTING) state = "CONNECTING"; + else if (ctx->state == CAPTION_LISTENING) state = "LISTENING"; + else if (ctx->state == CAPTION_PROCESSING) state = "FINALIZING"; + else if (ctx->state == CAPTION_FAILED) state = "FAILED"; + lv_label_set_text(ctx->state_label, state); + lv_label_set_text(ctx->caption_label, ctx->caption[0] ? ctx->caption : ctx->detail); + if (ctx->state == CAPTION_IDLE || ctx->state == CAPTION_FAILED) { + lv_obj_clear_state(ctx->start_button, LV_STATE_DISABLED); + } else { + lv_obj_add_state(ctx->start_button, LV_STATE_DISABLED); + } + if (ctx->state == CAPTION_LISTENING || ctx->state == CAPTION_PROCESSING) { + lv_obj_clear_state(ctx->stop_button, LV_STATE_DISABLED); + } else { + lv_obj_add_state(ctx->stop_button, LV_STATE_DISABLED); + } + tt_lvgl_unlock(); +} + +static void set_state(CaptionContext* ctx, CaptionState state, const char* detail) { + ctx->state = state; + snprintf(ctx->detail, sizeof(ctx->detail), "%s", detail ? detail : ""); + update_ui(ctx); +} + +static bool find_audio_stream_device(CaptionContext* ctx) { + ctx->stream_dev = device_find_by_name("audio-stream"); + if (ctx->stream_dev == NULL) ctx->stream_dev = device_find_first_by_type(&AUDIO_STREAM_TYPE); + return ctx->stream_dev != NULL; +} + +static bool open_input_stream(CaptionContext* ctx) { + if (ctx->stream_dev == NULL) return false; + if (ctx->input_handle != NULL) return true; + struct AudioStreamConfig config = {.sample_rate = 16000, .bits_per_sample = 16, .channels = 1}; + if (audio_stream_open_input(ctx->stream_dev, &config, &ctx->input_handle) != ERROR_NONE) return false; + audio_stream_set_mute(ctx->stream_dev, AUDIO_CODEC_DIR_INPUT, false); + audio_stream_set_volume(ctx->stream_dev, AUDIO_CODEC_DIR_INPUT, 100.0f); + return true; +} + +static void close_input_stream(CaptionContext* ctx) { + if (ctx->input_handle != NULL) { + audio_stream_close(ctx->input_handle); + ctx->input_handle = NULL; + } +} + +static int send_locked(CaptionContext* ctx, const uint8_t* data, size_t length, bool binary) { + if (ctx->fd < 0 || xSemaphoreTake(ctx->socket_lock, pdMS_TO_TICKS(500)) != pdTRUE) return -1; + int result = ws_send(ctx->fd, data, length, binary); + xSemaphoreGive(ctx->socket_lock); + return result; +} + +static bool parse_endpoint(const char* url, char* host, size_t host_size, int* port, char* path, size_t path_size) { + if (url == NULL || strncmp(url, "ws://", 5) != 0) return false; + const char* authority = url + 5; + const char* slash = strchr(authority, '/'); + const char* end = slash ? slash : authority + strlen(authority); + const char* colon = NULL; + for (const char* p = authority; p < end; ++p) if (*p == ':') colon = p; + size_t host_len = (size_t)((colon ? colon : end) - authority); + if (host_len == 0 || host_len >= host_size) return false; + memcpy(host, authority, host_len); host[host_len] = '\0'; + *port = 80; + if (colon != NULL) { + *port = atoi(colon + 1); + if (*port < 1 || *port > 65535) return false; + } + const char* wire_path = slash ? slash : "/"; + if (strlen(wire_path) >= path_size) return false; + snprintf(path, path_size, "%s", wire_path); + return true; +} + +static void append_final_caption(const char* text) { + if (text == NULL || !*text) return; + mkdir("/sdcard/captions", 0755); + time_t now = time(NULL); + struct tm local; + localtime_r(&now, &local); + char filename[32]; + strftime(filename, sizeof(filename), "%Y-%m-%d.txt", &local); + char stamp[16]; + strftime(stamp, sizeof(stamp), "%H:%M:%S", &local); + char path[128]; + snprintf(path, sizeof(path), "/sdcard/captions/%s", filename); + FILE* log = fopen(path, "a"); + if (log == NULL) { ESP_LOGE(TAG, "could not append %s", path); return; } + fprintf(log, "[%s] %s\n", stamp, text); + fclose(log); +} + +static void display_caption(CaptionContext* ctx, const char* text, bool final) { + if (text == NULL || !*text) return; + snprintf(ctx->caption, sizeof(ctx->caption), "%s", text); + if (final && strcmp(ctx->last_final, text) != 0) { + snprintf(ctx->last_final, sizeof(ctx->last_final), "%s", text); + append_final_caption(text); + set_state(ctx, CAPTION_IDLE, "Saved final caption to SD card"); + } else update_ui(ctx); +} + +static void handle_event(CaptionContext* ctx, const char* json) { + cJSON* root = cJSON_Parse(json); + if (root == NULL) return; + cJSON* event = cJSON_GetObjectItem(root, "event"); + cJSON* text = cJSON_GetObjectItem(root, "text"); + if (!cJSON_IsString(event)) { cJSON_Delete(root); return; } + const char* name = event->valuestring; + if (strcmp(name, "ready") == 0) { + set_state(ctx, CAPTION_CONNECTING, "Starting caption stream"); + } else if (strcmp(name, "listening") == 0) { + set_state(ctx, CAPTION_LISTENING, "Listening — press Stop when finished"); + } else if (strcmp(name, "draft") == 0 || strcmp(name, "interim_transcript") == 0) { + if (cJSON_IsString(text)) display_caption(ctx, text->valuestring, false); + } else if (strcmp(name, "transcript") == 0 || strcmp(name, "final") == 0) { + cJSON* is_final = cJSON_GetObjectItem(root, "isFinal"); + if (cJSON_IsString(text) && (!cJSON_IsBool(is_final) || cJSON_IsTrue(is_final) || strcmp(name, "final") == 0)) { + display_caption(ctx, text->valuestring, true); + } + } else if (strcmp(name, "thinking") == 0) { + set_state(ctx, CAPTION_PROCESSING, "Final captioning…"); + } else if (strcmp(name, "error") == 0) { + ctx->socket_failed = true; + set_state(ctx, CAPTION_FAILED, "Fail to connect"); + } + cJSON_Delete(root); +} + +static void receiver_task(void* argument) { + CaptionContext* ctx = argument; + uint8_t* buffer = malloc(EVENT_BUFFER_BYTES + 1U); + if (buffer == NULL) { ctx->socket_failed = true; ctx->receiver = NULL; vTaskDelete(NULL); } + while (ctx->visible && ctx->fd >= 0) { + int opcode = 0; bool complete = false; + int received = ws_recv(ctx->fd, &opcode, &complete, buffer, EVENT_BUFFER_BYTES); + if (received < 0 || !complete) break; + if (opcode == 0x01) { buffer[received] = '\0'; handle_event(ctx, (const char*)buffer); } + else if (opcode == 0x09) { + if (xSemaphoreTake(ctx->socket_lock, pdMS_TO_TICKS(500)) == pdTRUE) { + ws_send_pong(ctx->fd, buffer, (size_t)received); xSemaphoreGive(ctx->socket_lock); + } + } else if (opcode == 0x08) break; + } + ctx->session_active = false; + ctx->receiver = NULL; + vTaskDelete(NULL); +} + +static void load_config(CaptionContext* ctx) { + snprintf(ctx->endpoint, sizeof(ctx->endpoint), "%s", DEFAULT_ENDPOINT); + snprintf(ctx->device_id, sizeof(ctx->device_id), "%s", DEFAULT_DEVICE_ID); + ctx->api_key[0] = '\0'; + char path[256]; size_t size = sizeof(path); + tt_app_get_user_data_child_path(ctx->app, "config.json", path, &size); + FILE* file = fopen(path, "r"); + if (file == NULL) return; + char raw[512]; size_t bytes = fread(raw, 1, sizeof(raw) - 1, file); fclose(file); raw[bytes] = '\0'; + cJSON* root = cJSON_Parse(raw); + cJSON* endpoint = root ? cJSON_GetObjectItem(root, "server_url") : NULL; + cJSON* device = root ? cJSON_GetObjectItem(root, "device_id") : NULL; + cJSON* key = root ? cJSON_GetObjectItem(root, "api_key") : NULL; + if (cJSON_IsString(endpoint)) snprintf(ctx->endpoint, sizeof(ctx->endpoint), "%s", endpoint->valuestring); + if (cJSON_IsString(device)) snprintf(ctx->device_id, sizeof(ctx->device_id), "%s", device->valuestring); + if (cJSON_IsString(key)) snprintf(ctx->api_key, sizeof(ctx->api_key), "%s", key->valuestring); + cJSON_Delete(root); +} + +static void worker_task(void* argument) { + CaptionContext* ctx = argument; + char host[64], path[96]; int port = 0; + if (!parse_endpoint(ctx->endpoint, host, sizeof(host), &port, path, sizeof(path))) { + set_state(ctx, CAPTION_FAILED, "Fail to connect"); ctx->worker = NULL; vTaskDelete(NULL); + } + set_state(ctx, CAPTION_CONNECTING, "Connecting to Mac mini"); + ctx->fd = ws_connect(host, port, path, ctx->device_id, ctx->api_key); + if (ctx->fd < 0) { set_state(ctx, CAPTION_FAILED, "Fail to connect"); ctx->worker = NULL; vTaskDelete(NULL); } + char start[256]; + snprintf(start, sizeof(start), "{\"event\":\"start\",\"device_id\":\"%s\",\"format\":\"pcm_s16le\",\"sample_rate\":16000,\"channels\":1,\"sample_width\":2,\"session_id\":\"cap-%08lx\"}", ctx->device_id, (unsigned long)esp_random()); + if (send_locked(ctx, (const uint8_t*)start, strlen(start), false) != 0 || !open_input_stream(ctx)) { + set_state(ctx, CAPTION_FAILED, "Fail to connect"); ws_close(ctx->fd); ctx->fd = -1; ctx->worker = NULL; vTaskDelete(NULL); + } + ctx->session_active = true; + xTaskCreate(receiver_task, "caption_rx", 6144, ctx, 6, &ctx->receiver); + set_state(ctx, CAPTION_LISTENING, "Listening — press Stop when finished"); + uint8_t pcm[PCM_BUFFER_BYTES]; + while (ctx->visible && ctx->capture_audio && !ctx->socket_failed) { + size_t bytes = 0; + xSemaphoreTake(ctx->audio_lock, portMAX_DELAY); + error_t result = audio_stream_read(ctx->input_handle, pcm, sizeof(pcm), &bytes, pdMS_TO_TICKS(100)); + xSemaphoreGive(ctx->audio_lock); + if (result == ERROR_NONE && bytes > 0 && (bytes % 2U) == 0 && send_locked(ctx, pcm, bytes, true) != 0) ctx->socket_failed = true; + } + xSemaphoreTake(ctx->audio_lock, portMAX_DELAY); close_input_stream(ctx); xSemaphoreGive(ctx->audio_lock); + if (ctx->stop_requested && !ctx->socket_failed) { + const char* stop = "{\"event\":\"stop\"}"; + send_locked(ctx, (const uint8_t*)stop, strlen(stop), false); + set_state(ctx, CAPTION_PROCESSING, "Final captioning…"); + for (unsigned i = 0; ctx->session_active && i < 150; ++i) vTaskDelay(pdMS_TO_TICKS(100)); + } + if (ctx->fd >= 0) { ws_send_close(ctx->fd); ws_close(ctx->fd); ctx->fd = -1; } + if (ctx->socket_failed && ctx->visible) set_state(ctx, CAPTION_FAILED, "Fail to connect"); + else if (ctx->state == CAPTION_PROCESSING) set_state(ctx, CAPTION_IDLE, "No final caption received"); + ctx->worker = NULL; + vTaskDelete(NULL); +} + +static void on_start(lv_event_t* event) { + CaptionContext* ctx = lv_event_get_user_data(event); + if (ctx->worker != NULL || ctx->state == CAPTION_LISTENING || ctx->state == CAPTION_PROCESSING) return; + ctx->caption[0] = '\0'; ctx->last_final[0] = '\0'; ctx->socket_failed = false; ctx->stop_requested = false; ctx->capture_audio = true; + xTaskCreate(worker_task, "caption_tx", 8192, ctx, 5, &ctx->worker); +} + +static void on_stop(lv_event_t* event) { + CaptionContext* ctx = lv_event_get_user_data(event); + if (ctx->state != CAPTION_LISTENING) return; + ctx->capture_audio = false; ctx->stop_requested = true; + set_state(ctx, CAPTION_PROCESSING, "Final captioning…"); +} + +static void* create_data(void) { CaptionContext* ctx = calloc(1, sizeof(*ctx)); if (ctx) ctx->fd = -1; return ctx; } +static void destroy_data(void* data) { free(data); } +static void on_create(AppHandle app, void* data) { ((CaptionContext*)data)->app = app; } + +static void on_show(AppHandle app, void* data, lv_obj_t* parent) { + CaptionContext* ctx = data; ctx->visible = true; load_config(ctx); find_audio_stream_device(ctx); + ctx->socket_lock = xSemaphoreCreateMutex(); ctx->audio_lock = xSemaphoreCreateMutex(); + tt_lvgl_toolbar_create_for_app(parent, app); + ctx->state_label = lv_label_create(parent); lv_obj_align(ctx->state_label, LV_ALIGN_TOP_MID, 0, 38); + ctx->caption_label = lv_label_create(parent); lv_obj_set_width(ctx->caption_label, lv_pct(88)); + lv_label_set_long_mode(ctx->caption_label, LV_LABEL_LONG_WRAP); lv_obj_set_style_text_align(ctx->caption_label, LV_TEXT_ALIGN_CENTER, 0); + lv_obj_align(ctx->caption_label, LV_ALIGN_CENTER, 0, -8); + ctx->start_button = lv_btn_create(parent); lv_obj_set_size(ctx->start_button, 100, 42); lv_obj_align(ctx->start_button, LV_ALIGN_BOTTOM_LEFT, 22, -18); + lv_obj_t* start_text = lv_label_create(ctx->start_button); lv_label_set_text(start_text, "Start"); lv_obj_center(start_text); + lv_obj_add_event_cb(ctx->start_button, on_start, LV_EVENT_CLICKED, ctx); + ctx->stop_button = lv_btn_create(parent); lv_obj_set_size(ctx->stop_button, 100, 42); lv_obj_align(ctx->stop_button, LV_ALIGN_BOTTOM_RIGHT, -22, -18); + lv_obj_t* stop_text = lv_label_create(ctx->stop_button); lv_label_set_text(stop_text, "Stop"); lv_obj_center(stop_text); + lv_obj_add_event_cb(ctx->stop_button, on_stop, LV_EVENT_CLICKED, ctx); + if (ctx->stream_dev == NULL || ctx->socket_lock == NULL || ctx->audio_lock == NULL) set_state(ctx, CAPTION_FAILED, "Audio service unavailable"); + else set_state(ctx, CAPTION_IDLE, "Press Start to caption"); +} + +static void on_hide(AppHandle app, void* data) { + (void)app; CaptionContext* ctx = data; ctx->visible = false; ctx->capture_audio = false; ctx->stop_requested = false; + if (ctx->fd >= 0) { ws_send_close(ctx->fd); ws_close(ctx->fd); ctx->fd = -1; } + for (unsigned i = 0; (ctx->worker || ctx->receiver) && i < 100; ++i) vTaskDelay(pdMS_TO_TICKS(10)); + close_input_stream(ctx); + if (ctx->socket_lock) { vSemaphoreDelete(ctx->socket_lock); ctx->socket_lock = NULL; } + if (ctx->audio_lock) { vSemaphoreDelete(ctx->audio_lock); ctx->audio_lock = NULL; } +} + +int main(int argc, char* argv[]) { + (void)argc; (void)argv; + tt_app_register((AppRegistration){.createData=create_data,.destroyData=destroy_data,.onCreate=on_create,.onShow=on_show,.onHide=on_hide}); + return 0; +} diff --git a/Apps/LiveCaptions/main/Source/websocket.c b/Apps/LiveCaptions/main/Source/websocket.c new file mode 100644 index 0000000..7695a8d --- /dev/null +++ b/Apps/LiveCaptions/main/Source/websocket.c @@ -0,0 +1,167 @@ +#include "websocket.h" + +#include +#include +#include + +#include +#include +#include +#include + +#define TAG "PipecatVoiceWs" +#define WS_HEADER_LIMIT 1024U +#define WS_CONTROL_LIMIT 125U + +static int send_all(int fd, const uint8_t* data, size_t length) { + size_t sent = 0; + while (sent < length) { + int result = lwip_send(fd, data + sent, length - sent, 0); + if (result <= 0) return -1; + sent += (size_t)result; + } + return 0; +} + +static int recv_all(int fd, uint8_t* data, size_t length) { + size_t received = 0; + while (received < length) { + int result = lwip_recv(fd, data + received, length - received, 0); + if (result <= 0) return -1; + received += (size_t)result; + } + return 0; +} + +static int discard(int fd, uint64_t length) { + uint8_t buffer[256]; + while (length > 0) { + size_t chunk = length > sizeof(buffer) ? sizeof(buffer) : (size_t)length; + if (recv_all(fd, buffer, chunk) < 0) return -1; + length -= chunk; + } + return 0; +} + +static int send_frame(int fd, uint8_t opcode, const uint8_t* payload, size_t length) { + if (length > 65535U || ((opcode & 0x08U) && length > WS_CONTROL_LIMIT)) return -1; + uint8_t header[8]; + size_t header_length = 2; + header[0] = 0x80U | opcode; + if (length < 126U) { + header[1] = 0x80U | (uint8_t)length; + } else { + header[1] = 0x80U | 126U; + header[2] = (uint8_t)(length >> 8U); + header[3] = (uint8_t)length; + header_length = 4; + } + uint8_t mask[4]; + uint32_t random = esp_random(); + memcpy(mask, &random, sizeof(mask)); + memcpy(header + header_length, mask, sizeof(mask)); + header_length += sizeof(mask); + if (send_all(fd, header, header_length) < 0) return -1; + + uint8_t chunk[512]; + size_t offset = 0; + while (offset < length) { + size_t count = length - offset > sizeof(chunk) ? sizeof(chunk) : length - offset; + for (size_t i = 0; i < count; ++i) chunk[i] = payload[offset + i] ^ mask[(offset + i) % sizeof(mask)]; + if (send_all(fd, chunk, count) < 0) return -1; + offset += count; + } + return 0; +} + +int ws_connect(const char* host, int port, const char* path, const char* device_id, const char* api_key) { + if (host == NULL || path == NULL || device_id == NULL || api_key == NULL || port < 1 || port > 65535) return -1; + int fd = lwip_socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) { + ESP_LOGW(TAG, "socket create failed"); + return -1; + } + struct sockaddr_in address = {0}; + address.sin_family = AF_INET; + address.sin_port = htons((uint16_t)port); + address.sin_addr.s_addr = ipaddr_addr(host); + if (address.sin_addr.s_addr == IPADDR_NONE) { + ESP_LOGW(TAG, "endpoint address parse failed"); + close(fd); + return -1; + } + if (lwip_connect(fd, (struct sockaddr*)&address, sizeof(address)) < 0) { + ESP_LOGW(TAG, "TCP connect failed"); + close(fd); + return -1; + } + struct timeval timeout = {.tv_sec = 15, .tv_usec = 0}; + lwip_setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); + char request[WS_HEADER_LIMIT]; + int request_length = snprintf(request, sizeof(request), + "GET %s HTTP/1.1\r\nHost: %s:%d\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n" + "Sec-WebSocket-Key: MDEyMzQ1Njc4OWFiY2RlZg==\r\nSec-WebSocket-Version: 13\r\n" + "Authorization: Bearer %s\r\nX-Device-ID: %s\r\n\r\n", + path, host, port, api_key, device_id); + if (request_length < 0 || (size_t)request_length >= sizeof(request) || send_all(fd, (const uint8_t*)request, (size_t)request_length) < 0) { + ESP_LOGW(TAG, "WebSocket upgrade request failed"); + close(fd); + return -1; + } + char response[WS_HEADER_LIMIT]; + size_t length = 0; + while (length + 1 < sizeof(response)) { + if (recv_all(fd, (uint8_t*)&response[length], 1) < 0) { + ESP_LOGW(TAG, "WebSocket upgrade response failed"); + close(fd); + return -1; + } + response[++length] = '\0'; + if (length >= 4 && memcmp(response + length - 4, "\r\n\r\n", 4) == 0) break; + } + if (length + 1 >= sizeof(response) || strstr(response, " 101 ") == NULL) { + ESP_LOGW(TAG, "WebSocket upgrade rejected"); + close(fd); + return -1; + } + ESP_LOGI(TAG, "WebSocket upgrade accepted"); + return fd; +} + +int ws_send(int fd, const uint8_t* data, size_t length, bool binary) { + if (fd < 0 || data == NULL || length == 0) return -1; + return send_frame(fd, binary ? 0x02U : 0x01U, data, length); +} + +int ws_recv(int fd, int* opcode, bool* final, uint8_t* payload, size_t maximum) { + uint8_t header[2]; + if (fd < 0 || recv_all(fd, header, sizeof(header)) < 0) return -1; + uint64_t length = header[1] & 0x7fU; + if (length == 126U) { + uint8_t extended[2]; + if (recv_all(fd, extended, sizeof(extended)) < 0) return -1; + length = ((uint64_t)extended[0] << 8U) | extended[1]; + } else if (length == 127U) { + uint8_t extended[8]; + if (recv_all(fd, extended, sizeof(extended)) < 0) return -1; + length = 0; + for (size_t i = 0; i < sizeof(extended); ++i) length = (length << 8U) | extended[i]; + } + bool masked = (header[1] & 0x80U) != 0; + uint8_t mask[4] = {0}; + if (masked && recv_all(fd, mask, sizeof(mask)) < 0) return -1; + uint8_t frame_opcode = header[0] & 0x0fU; + if (((frame_opcode & 0x08U) && (length > WS_CONTROL_LIMIT || !(header[0] & 0x80U))) || length > maximum) { + if (discard(fd, length) < 0) return -1; + return -2; + } + if (length > 0 && recv_all(fd, payload, (size_t)length) < 0) return -1; + if (masked) for (size_t i = 0; i < (size_t)length; ++i) payload[i] ^= mask[i % sizeof(mask)]; + if (opcode) *opcode = frame_opcode; + if (final) *final = (header[0] & 0x80U) != 0; + return (int)length; +} + +int ws_send_pong(int fd, const uint8_t* payload, size_t length) { return send_frame(fd, 0x0aU, payload, length); } +int ws_send_close(int fd) { return send_frame(fd, 0x08U, NULL, 0); } +void ws_close(int fd) { if (fd >= 0) close(fd); } \ No newline at end of file diff --git a/Apps/LiveCaptions/main/Source/websocket.h b/Apps/LiveCaptions/main/Source/websocket.h new file mode 100644 index 0000000..d5f4f4d --- /dev/null +++ b/Apps/LiveCaptions/main/Source/websocket.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Connect to a WebSocket server. + * @param host Server IP address (e.g. "192.168.68.126") + * @param port Port number (e.g. 8642) + * @param path WebSocket path (e.g. "/api/esp32/voice/ws") + * @param device_id Unique device identifier + * @param api_key Optional profile API key; never compiled into firmware + * @return Socket file descriptor on success, or -1 on failure + */ +int ws_connect(const char* host, int port, const char* path, const char* device_id, const char* api_key); + +/** + * Send a WebSocket frame. + * @param fd Socket file descriptor + * @param data Data payload to send + * @param len Length of the data payload + * @param binary True for binary frame, false for text frame + * @return 0 on success, or -1 on failure + */ +int ws_send(int fd, const uint8_t* data, size_t len, bool binary); + +/** + * Receive a WebSocket frame. + * @param fd Socket file descriptor + * @param out_opcode Pointer to store the received opcode (e.g. 0x01 text, 0x02 binary) + * @param payload Buffer to store the received payload + * @param max_len Maximum length of the payload buffer + * @return Received payload length on success, -1 on connection failure, or -2 on buffer overflow + */ +int ws_recv(int fd, int* out_opcode, bool* out_final, uint8_t* payload, size_t max_len); + +/** + * Close a WebSocket connection. + * @param fd Socket file descriptor + */ +void ws_close(int fd); + +/** + * Send a WebSocket PONG frame. + * @param fd Socket file descriptor + * @param payload Payload to reflect + * @param len Length of payload + * @return 0 on success, or -1 on failure + */ +int ws_send_pong(int fd, const uint8_t* payload, size_t len); + +/** Send a clean WebSocket close control frame before closing the socket. */ +int ws_send_close(int fd); + +#ifdef __cplusplus +} +#endif diff --git a/Apps/LiveCaptions/manifest.properties b/Apps/LiveCaptions/manifest.properties new file mode 100644 index 0000000..025b5e7 --- /dev/null +++ b/Apps/LiveCaptions/manifest.properties @@ -0,0 +1,7 @@ +manifest.version=0.2 +target.sdk=0.8.0-dev +target.platforms=esp32s3 +app.id=one.tactility.livecaptions +app.version.name=1.0.0 +app.version.code=1 +app.name=Live Captions diff --git a/tests/test_live_captions_contract.py b/tests/test_live_captions_contract.py new file mode 100644 index 0000000..2183247 --- /dev/null +++ b/tests/test_live_captions_contract.py @@ -0,0 +1,42 @@ +"""Source-level contract for the Live Captions external Tactility app. + +The app is ELF-loaded firmware code, so this test guards the device protocol and +persistence requirements before an ESP32-S3 build/install test is available. +""" +from pathlib import Path +import unittest + +ROOT = Path(__file__).resolve().parents[1] +APP = ROOT / "Apps" / "LiveCaptions" + + +class LiveCaptionsContractTests(unittest.TestCase): + def test_manifest_identifies_the_captions_app(self): + manifest = (APP / "manifest.properties").read_text() + self.assertIn("app.id=one.tactility.livecaptions", manifest) + self.assertIn("app.name=Live Captions", manifest) + self.assertIn("target.platforms=esp32s3", manifest) + + def test_app_streams_audio_and_logs_final_caption_to_daily_text_file(self): + source = (APP / "main" / "Source" / "main.c").read_text() + self.assertIn("audio_stream_open_input", source) + self.assertIn("/sdcard/captions", source) + self.assertIn('"%Y-%m-%d.txt"', source) + self.assertIn("append_final_caption", source) + self.assertIn('"draft"', source) + self.assertIn('"interim_transcript"', source) + self.assertIn('"transcript"', source) + self.assertIn('"final"', source) + self.assertIn("const char* stop", source) + self.assertNotIn("i2s_controller_", source) + + def test_connection_failure_stays_failed_until_user_starts_again(self): + source = (APP / "main" / "Source" / "main.c").read_text() + self.assertIn("CAPTION_FAILED", source) + self.assertIn("Fail to connect", source) + self.assertNotIn("retry_attempt", source) + self.assertNotIn("retry scheduled", source) + + +if __name__ == "__main__": + unittest.main()