adolforeyna/tactility_apps
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: checkpoint app work before firmware sync

Browse Source
This commit is contained in:
Adolfo Reyna
2026-09-07 23:01:26 -04:00
parent e42036a044
commit 59612020bb
20 changed files with 2111 additions and 1218 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Apps/ReynaBot/main/Source/main.c
+217 -77
View File
@@ -1,24 +1,32 @@
#include <tt_app.h>
#include <tt_lvgl.h>
#include <tt_lvgl_toolbar.h>
#include <tt_wifi.h>
#include <tactility/device.h>
#include <tactility/drivers/i2s_controller.h>
#include <tactility/drivers/audio_stream.h>
#include "websocket.h"
#include <cJSON.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/stat.h>
#include <errno.h>
/* Exported by Tactility firmware 0.8.0-dev; absent from the CDN SDK header. */
struct Device* device_find_by_name(const char* name);
#include "freertos/FreeRTOS.h"
#include "freertos/task.h"
#include "esp_log.h"
#include "esp_random.h"
#define TAG "ReynaBot"
#define AUDIO_SAMPLE_RATE 16000U
#define AUDIO_BITS_PER_SAMPLE 16U
#define MAX_PCM_CHUNK_BYTES 16384U
#define MAX_RESPONSE_AUDIO_BYTES 65536U
typedef enum {
STATE_IDLE,
@@ -64,10 +72,19 @@ typedef struct {
TaskHandle_t worker_task;
TaskHandle_t rx_task;
// Hardware
struct Device* i2s_dev;
// WebSocket
// Hardware/audio service
struct Device* audio_stream_dev;
AudioStreamHandle input_handle;
AudioStreamHandle output_handle;
uint32_t output_sample_rate;
uint8_t output_channels;
uint8_t output_bits_per_sample;
bool output_stream_pending;
size_t expected_audio_bytes;
uint32_t expected_audio_rate;
uint8_t expected_audio_channels;
uint8_t expected_audio_bits;
bool stop_sent;
int ws_fd;
bool ws_connected;
bool ws_done;
@@ -81,6 +98,77 @@ static void reynabot_rx_task(void* arg);
static void update_ui(ReynaBotCtx* ctx);
static void load_config(ReynaBotCtx* ctx);
static bool find_audio_stream_device(ReynaBotCtx* ctx) {
struct Device* dev = device_find_by_name("audio-stream");
if (dev != NULL) {
ctx->audio_stream_dev = dev;
return true;
}
// The 0.8.0-dev firmware registers this shared service by name. The
// CDN SDK headers do not expose the deprecated type-search helper, so do
// not reference it from a dynamically linked app.
return false;
}
static void close_input_stream(ReynaBotCtx* ctx) {
if (ctx->input_handle != NULL) {
audio_stream_close(ctx->input_handle);
ctx->input_handle = NULL;
}
}
static void close_output_stream(ReynaBotCtx* ctx) {
if (ctx->output_handle != NULL) {
audio_stream_close(ctx->output_handle);
ctx->output_handle = NULL;
}
ctx->output_stream_pending = false;
}
static bool open_input_stream(ReynaBotCtx* ctx) {
if (ctx->audio_stream_dev == NULL) return false;
close_input_stream(ctx);
struct AudioStreamConfig config = {
.sample_rate = 16000,
.bits_per_sample = 16,
.channels = 1,
};
error_t err = audio_stream_open_input(ctx->audio_stream_dev, &config, &ctx->input_handle);
if (err != ERROR_NONE) {
ctx->input_handle = NULL;
ESP_LOGE(TAG, "audio_stream_open_input failed: %d", err);
return false;
}
audio_stream_set_mute(ctx->audio_stream_dev, AUDIO_CODEC_DIR_INPUT, false);
audio_stream_set_volume(ctx->audio_stream_dev, AUDIO_CODEC_DIR_INPUT, 100.0f);
ESP_LOGI(TAG, "Microphone opened via audio-stream at 16000 Hz mono");
return true;
}
static bool open_output_stream(ReynaBotCtx* ctx, uint32_t sample_rate, uint8_t channels, uint8_t bits_per_sample) {
if (ctx->audio_stream_dev == NULL || sample_rate == 0 || channels == 0 || bits_per_sample != 16) return false;
close_output_stream(ctx);
struct AudioStreamConfig config = {
.sample_rate = sample_rate,
.bits_per_sample = bits_per_sample,
.channels = channels,
};
error_t err = audio_stream_open_output(ctx->audio_stream_dev, &config, &ctx->output_handle);
if (err != ERROR_NONE) {
ctx->output_handle = NULL;
ESP_LOGE(TAG, "audio_stream_open_output failed: %d rate=%u ch=%u", err, (unsigned)sample_rate, channels);
return false;
}
audio_stream_set_mute(ctx->audio_stream_dev, AUDIO_CODEC_DIR_OUTPUT, false);
audio_stream_set_volume(ctx->audio_stream_dev, AUDIO_CODEC_DIR_OUTPUT, 100.0f);
ctx->output_sample_rate = sample_rate;
ctx->output_channels = channels;
ctx->output_bits_per_sample = bits_per_sample;
ctx->output_stream_pending = true;
ESP_LOGI(TAG, "Speaker opened via audio-stream at %u Hz mono", (unsigned)sample_rate);
return true;
}
/* ─── Helper for URL parsing ─── */
static bool parse_ws_url(const char* url, char* host, int* port, char* path) {
if (strncmp(url, "ws://", 5) != 0) return false;
@@ -111,9 +199,9 @@ static bool parse_ws_url(const char* url, char* host, int* port, char* path) {
/* ─── Config Loading/Saving ─── */
static void load_config(ReynaBotCtx* ctx) {
// Default fallback config
snprintf(ctx->server_url, sizeof(ctx->server_url), "ws://192.168.68.126:8643/api/esp32/voice/ws");
snprintf(ctx->server_url, sizeof(ctx->server_url), "ws://192.168.68.112:8643/api/esp32/voice/ws");
snprintf(ctx->device_id, sizeof(ctx->device_id), "reynabot_screen");
snprintf(ctx->api_key, sizeof(ctx->api_key), "hmek_sXB7921bZ9FXTVKARExqUZ7ttBxtEoURHRU0JCB-gNY");
ctx->api_key[0] = '\0';
char path[256];
size_t path_size = sizeof(path);
@@ -130,7 +218,7 @@ static void load_config(ReynaBotCtx* ctx) {
// Write default config file
file = fopen(path, "w");
if (file != NULL) {
fprintf(file, "{\n \"server_url\": \"ws://192.168.68.126:8643/api/esp32/voice/ws\",\n \"device_id\": \"reynabot_screen\",\n \"api_key\": \"mcT1YA1vOr9wXSiHpCYalweEGGZKX-PIfZv2drp8BSg\"\n}\n");
fprintf(file, "{\n \"server_url\": \"ws://192.168.68.112:8643/api/esp32/voice/ws\",\n \"device_id\": \"reynabot_screen\",\n \"api_key\": \"\"\n}\n");
fclose(file);
}
ESP_LOGI(TAG, "Created default config.json at %s", path);
@@ -181,6 +269,21 @@ static void parse_json_message(ReynaBotCtx* ctx, const char* json_str) {
if (strcmp(evt, "ready") == 0) {
ESP_LOGI(TAG, "Server ready");
} else if (strcmp(evt, "state") == 0) {
cJSON* state_item = cJSON_GetObjectItem(json, "state");
if (state_item != NULL && cJSON_IsString(state_item)) {
if (strcmp(state_item->valuestring, "listening") == 0) {
if (ctx->stop_sent) {
ctx->ws_done = true;
} else {
ctx->state = STATE_LISTENING;
}
ctx->ui_update_pending = true;
} else if (strcmp(state_item->valuestring, "processing") == 0) {
ctx->state = STATE_THINKING;
ctx->ui_update_pending = true;
}
}
} else if (strcmp(evt, "listening") == 0) {
ctx->state = STATE_LISTENING;
ctx->ui_update_pending = true;
@@ -199,11 +302,42 @@ static void parse_json_message(ReynaBotCtx* ctx, const char* json_str) {
strncpy(ctx->last_response, txt_item->valuestring, sizeof(ctx->last_response) - 1);
ctx->ui_update_pending = true;
}
} else if (strcmp(evt, "audio") == 0) {
cJSON* format_item = cJSON_GetObjectItem(json, "format");
cJSON* rate_item = cJSON_GetObjectItem(json, "sample_rate");
cJSON* channels_item = cJSON_GetObjectItem(json, "channels");
cJSON* width_item = cJSON_GetObjectItem(json, "sample_width");
cJSON* length_item = cJSON_GetObjectItem(json, "byte_length");
bool valid = format_item != NULL && cJSON_IsString(format_item) &&
strcmp(format_item->valuestring, "pcm_s16le") == 0 &&
rate_item != NULL && cJSON_IsNumber(rate_item) && rate_item->valueint > 0 && rate_item->valueint <= 48000 &&
channels_item != NULL && cJSON_IsNumber(channels_item) && channels_item->valueint == 1 &&
width_item != NULL && cJSON_IsNumber(width_item) && width_item->valueint == 2 &&
length_item != NULL && cJSON_IsNumber(length_item) && length_item->valueint > 0 &&
length_item->valueint <= MAX_RESPONSE_AUDIO_BYTES && (length_item->valueint % 2) == 0;
if (valid) {
ctx->expected_audio_rate = (uint32_t)rate_item->valueint;
ctx->expected_audio_channels = (uint8_t)channels_item->valueint;
ctx->expected_audio_bits = (uint8_t)width_item->valueint;
ctx->expected_audio_bytes = (size_t)length_item->valueint;
if (!open_output_stream(ctx, ctx->expected_audio_rate, ctx->expected_audio_channels, ctx->expected_audio_bits)) {
snprintf(ctx->error_message, sizeof(ctx->error_message), "Audio output unavailable");
ctx->state = STATE_ERROR;
ctx->expected_audio_bytes = 0;
} else {
ctx->state = STATE_SPEAKING;
}
ctx->ui_update_pending = true;
} else {
snprintf(ctx->error_message, sizeof(ctx->error_message), "Invalid audio metadata");
ctx->state = STATE_ERROR;
ctx->expected_audio_bytes = 0;
ctx->ui_update_pending = true;
}
} else if (strcmp(evt, "audio_start") == 0) {
/* Legacy event: metadata must still arrive as `audio` before binary PCM. */
ctx->state = STATE_SPEAKING;
ctx->ui_update_pending = true;
// I2S is already configured globally at session startup
} else if (strcmp(evt, "audio_end") == 0) {
ESP_LOGI(TAG, "Audio response ended");
} else if (strcmp(evt, "done") == 0) {
@@ -225,7 +359,7 @@ static void parse_json_message(ReynaBotCtx* ctx, const char* json_str) {
/* ─── WebSocket RX (Receive) Task ─── */
static void reynabot_rx_task(void* arg) {
ReynaBotCtx* ctx = (ReynaBotCtx*)arg;
uint8_t* rx_buf = malloc(8192);
uint8_t* rx_buf = malloc(MAX_RESPONSE_AUDIO_BYTES + 1U);
if (rx_buf == NULL) {
ESP_LOGE(TAG, "Failed to allocate RX buffer");
ctx->ws_done = true;
@@ -238,7 +372,7 @@ static void reynabot_rx_task(void* arg) {
ESP_LOGI(TAG, "WS Receive task started");
while (ctx->ws_fd >= 0) {
int r = ws_recv(ctx->ws_fd, &opcode, rx_buf, 8191);
int r = ws_recv(ctx->ws_fd, &opcode, rx_buf, MAX_RESPONSE_AUDIO_BYTES);
if (r < 0) {
if (r == -1) {
ESP_LOGI(TAG, "WS connection closed or read error, errno=%d", errno);
@@ -252,21 +386,31 @@ static void reynabot_rx_task(void* arg) {
if (opcode == 0x01) { // Text frame (JSON)
rx_buf[r] = '\0';
parse_json_message(ctx, (char*)rx_buf);
} else if (opcode == 0x02) { // Binary frame (Audio data)
if (ctx->state == STATE_SPEAKING && ctx->i2s_dev != NULL) {
const uint8_t* payload_ptr = rx_buf;
size_t payload_len = r;
// Skip WAV header if present in the first chunk
if (payload_len > 44 && memcmp(payload_ptr, "RIFF", 4) == 0) {
payload_ptr += 44;
payload_len -= 44;
} else if (opcode == 0x02) { // Audio response PCM
if (ctx->output_handle == NULL || ctx->expected_audio_bytes == 0 || (size_t)r != ctx->expected_audio_bytes) {
ESP_LOGE(TAG, "Audio frame does not match metadata: got=%d expected=%u", r, (unsigned)ctx->expected_audio_bytes);
ctx->state = STATE_ERROR;
snprintf(ctx->error_message, sizeof(ctx->error_message), "Invalid audio frame");
ctx->ui_update_pending = true;
} else {
size_t written_total = 0;
while (written_total < (size_t)r) {
size_t written = 0;
error_t err = audio_stream_write(ctx->output_handle, rx_buf + written_total,
(size_t)r - written_total, &written, pdMS_TO_TICKS(3000));
if (err != ERROR_NONE || written == 0) {
ESP_LOGE(TAG, "audio_stream_write failed: %d written=%u", err, (unsigned)written);
ctx->state = STATE_ERROR;
snprintf(ctx->error_message, sizeof(ctx->error_message), "Audio playback failed");
ctx->ui_update_pending = true;
break;
}
written_total += written;
}
size_t written = 0;
device_lock(ctx->i2s_dev);
i2s_controller_write(ctx->i2s_dev, payload_ptr, payload_len, &written, pdMS_TO_TICKS(100));
device_unlock(ctx->i2s_dev);
close_output_stream(ctx);
ctx->expected_audio_bytes = 0;
ctx->state = STATE_THINKING;
ctx->ui_update_pending = true;
}
} else if (opcode == 0x09) { // PING frame
ESP_LOGI(TAG, "WS PING received, sending PONG");
@@ -408,8 +552,11 @@ static void reynabot_task(void* arg) {
ctx->start_session = false;
ctx->stop_session = false;
ctx->cancel_session = false;
ctx->state = STATE_CONNECTING;
ctx->stop_sent = false;
ctx->expected_audio_bytes = 0;
close_input_stream(ctx);
close_output_stream(ctx);
ctx->last_transcript[0] = '\0';
ctx->last_response[0] = '\0';
ctx->error_message[0] = '\0';
@@ -434,29 +581,18 @@ static void reynabot_task(void* arg) {
ctx->ws_fd = fd;
ctx->ws_connected = true;
// Configure I2S on demand for this session (16 kHz, 16-bit, mono)
struct I2sConfig session_cfg = {
.communication_format = I2S_FORMAT_STAND_I2S,
.sample_rate = 16000,
.bits_per_sample = 16,
.channel_left = 0,
.channel_right = I2S_CHANNEL_NONE
};
if (ctx->i2s_dev != NULL) {
device_lock(ctx->i2s_dev);
i2s_controller_set_config(ctx->i2s_dev, &session_cfg);
device_unlock(ctx->i2s_dev);
}
// Spawn background RX task to read and parse events
// Spawn background RX task to read and parse events.
xTaskCreate(reynabot_rx_task, "reynabot_rx", 4096, ctx, 6, &ctx->rx_task);
// Send start event handshake
char start_json[256];
// The Kids LAN adapter requires protocol v1 and a fresh session id.
char session_id[80];
snprintf(session_id, sizeof(session_id), "reynabot-%08x%08x",
(unsigned)esp_random(), (unsigned)esp_random());
char start_json[384];
snprintf(start_json, sizeof(start_json),
"{\"event\":\"start\",\"device_id\":\"%s\",\"sample_rate\":16000,\"channels\":1,\"sample_width\":2,\"format\":\"pcm_s16le\"}",
ctx->device_id);
"{\"v\":1,\"event\":\"start\",\"session_id\":\"%s\",\"device_id\":\"%s\",\"audio\":{\"format\":\"pcm_s16le\",\"sample_rate\":16000,\"channels\":1,\"sample_width\":2}}",
session_id, ctx->device_id);
if (ws_send(ctx->ws_fd, (const uint8_t*)start_json, strlen(start_json), false) < 0) {
ctx->state = STATE_ERROR;
snprintf(ctx->error_message, sizeof(ctx->error_message), "Handshake send failed");
@@ -486,11 +622,20 @@ static void reynabot_task(void* arg) {
continue;
}
// I2S is already configured globally at session startup
uint8_t* buffer = malloc(1024);
if (!open_input_stream(ctx)) {
ctx->state = STATE_ERROR;
snprintf(ctx->error_message, sizeof(ctx->error_message), "Microphone unavailable");
ws_close(ctx->ws_fd);
ctx->ws_fd = -1;
ctx->ws_connected = false;
update_ui(ctx);
continue;
}
uint8_t* buffer = malloc(MAX_PCM_CHUNK_BYTES);
if (buffer == NULL) {
ESP_LOGE(TAG, "Failed to allocate record buffer");
close_input_stream(ctx);
ctx->state = STATE_ERROR;
snprintf(ctx->error_message, sizeof(ctx->error_message), "Out of memory");
ws_close(ctx->ws_fd);
@@ -500,12 +645,13 @@ static void reynabot_task(void* arg) {
continue;
}
size_t total_sent_bytes = 0;
// Stream audio loop while PTT is held
// Stream 16 kHz mono PCM while PTT is held.
while (ctx->is_pressed && !ctx->stop_session && !ctx->cancel_session && !ctx->ws_done && total_sent_bytes < 320000) {
size_t bytes_read = 0;
error_t r = i2s_controller_read(ctx->i2s_dev, buffer, 1024, &bytes_read, pdMS_TO_TICKS(100));
if (r == ERROR_NONE && bytes_read > 0) {
error_t r = audio_stream_read(ctx->input_handle, buffer, MAX_PCM_CHUNK_BYTES,
&bytes_read, pdMS_TO_TICKS(200));
if (r == ERROR_NONE && bytes_read > 0 && bytes_read <= MAX_PCM_CHUNK_BYTES && (bytes_read % 2U) == 0) {
if (ws_send(ctx->ws_fd, buffer, bytes_read, true) < 0) {
ESP_LOGE(TAG, "Audio stream send failed");
break;
@@ -513,17 +659,20 @@ static void reynabot_task(void* arg) {
total_sent_bytes += bytes_read;
}
}
close_input_stream(ctx);
free(buffer);
if (ctx->cancel_session || total_sent_bytes < 3200) {
ESP_LOGI(TAG, "Cancelling audio session");
const char* cancel_json = "{\"event\":\"cancel\"}";
ws_send(ctx->ws_fd, (const uint8_t*)cancel_json, strlen(cancel_json), false);
ctx->stop_sent = true;
ctx->state = STATE_IDLE;
} else {
const char* stop_json = "{\"event\":\"stop\"}";
ws_send(ctx->ws_fd, (const uint8_t*)stop_json, strlen(stop_json), false);
ctx->stop_sent = true;
ctx->state = STATE_THINKING;
update_ui(ctx);
@@ -550,13 +699,9 @@ static void reynabot_task(void* arg) {
ctx->ws_connected = false;
ws_close(fd_to_close);
// Reset I2S controller to release DMA and stop white noise
if (ctx->i2s_dev != NULL) {
device_lock(ctx->i2s_dev);
i2s_controller_reset(ctx->i2s_dev);
device_unlock(ctx->i2s_dev);
}
close_input_stream(ctx);
close_output_stream(ctx);
// Wait for receive task to exit
int rx_timeout = 100;
while (ctx->rx_task != NULL && rx_timeout > 0) {
@@ -611,10 +756,9 @@ static void on_show(AppHandle app, void* data, lv_obj_t* parent) {
load_config(ctx);
// Find I2S controller
ctx->i2s_dev = device_find_by_name("i2s0");
if (ctx->i2s_dev == NULL) {
ESP_LOGE(TAG, "I2S controller 'i2s0' not found!");
// Find the firmware Audio System service; it owns codec/native-rate conversion.
if (!find_audio_stream_device(ctx)) {
ESP_LOGE(TAG, "audio-stream device not found!");
}
// Style the parent screen
@@ -800,13 +944,9 @@ static void on_hide(AppHandle app, void* data) {
ws_close(fd_to_close);
}
// Reset I2S controller to stop DMA and looping noise
if (ctx->i2s_dev != NULL) {
device_lock(ctx->i2s_dev);
i2s_controller_reset(ctx->i2s_dev);
device_unlock(ctx->i2s_dev);
}
close_input_stream(ctx);
close_output_stream(ctx);
// Wait briefly for tasks to exit
int timeout = 100;
while ((ctx->worker_task != NULL || ctx->rx_task != NULL) && timeout > 0) {