chore: checkpoint app work before firmware sync
This commit is contained in:
Adolfo Reyna
@@ -0,0 +1,16 @@
|
||||
cmake_minimum_required(VERSION 3.20)
|
||||
|
||||
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
|
||||
|
||||
if (DEFINED ENV{TACTILITY_SDK_PATH})
|
||||
set(TACTILITY_SDK_PATH $ENV{TACTILITY_SDK_PATH})
|
||||
else()
|
||||
set(TACTILITY_SDK_PATH "../../release/TactilitySDK")
|
||||
message(WARNING "TACTILITY_SDK_PATH is not set, defaulting to ${TACTILITY_SDK_PATH}")
|
||||
endif()
|
||||
|
||||
include("${TACTILITY_SDK_PATH}/TactilitySDK.cmake")
|
||||
set(EXTRA_COMPONENT_DIRS ${TACTILITY_SDK_PATH})
|
||||
|
||||
project(PipecatVoice)
|
||||
tactility_project(PipecatVoice)
|
||||
+23
-12
@@ -1,19 +1,30 @@
|
||||
# PipecatVoice — Tactility native voice companion
|
||||
# Pipecat Voice
|
||||
|
||||
Native ESP32 ELF app (like VoiceRecorder/ReynaBot), not a web client.
|
||||
Minimal native Tactility client for the LAN voice-adapter protocol v1. Opening the app immediately connects to `ws://192.168.68.102:8644/api/esp32/voice/ws`, sends the versioned `start` declaration, and continuously streams 16 kHz mono signed-16-bit PCM. There is no Connect button, speaker/transport picker, PTT control, text entry, transcript display, or credential on the device.
|
||||
|
||||
- Uses `audio-stream`/`i2s_controller` APIs (see VoiceRecorder) for 16kHz PCM mic + speaker
|
||||
- Uses ReynaBot's `websocket.c/h` + `lwip` for WS transport
|
||||
- Default gateway: Hermes WS `ws://<mac>:8642/api/esp32/voice/ws` (voice profile, limited tools, imperfect STT)
|
||||
- Optional second target: Pipecat websocket transport `ws://<mac>:7861` (when server exposes `websocket` via `create_transport`)
|
||||
The Mac-hosted adapter owns VAD, STT, Pipecat/Hermes orchestration, TTS, and credentials. It returns metadata followed by a single PCM response frame; the app pauses capture, plays that response at the declared rate through Tactility `i2s0`, then resumes capture. The app only displays connection/streaming/reconnect/configuration state and uses the standard toolbar to exit.
|
||||
|
||||
Build:
|
||||
## Configuration
|
||||
|
||||
`config.json` in app user data can override the non-secret endpoint and allowed adapter device id:
|
||||
|
||||
```json
|
||||
{"server_url":"ws://192.168.68.102:8644/api/esp32/voice/ws","device_id":"tactility-14c19d1a790"}
|
||||
```
|
||||
unset PYTHONPATH; export IDF_PYTHON_ENV_PATH=~/.espressif/python_env/idf5.3_py3.9_env
|
||||
. ~/esp/esp-idf/export.sh
|
||||
export TACTILITY_SDK_PATH=~/Projects/electronics/tactility/tactility/Buildscripts/TactilitySDK # or firmware/release/TactilitySDK if built
|
||||
|
||||
Only `ws://` private-LAN endpoints are accepted; loopback endpoints are rejected. Invalid configuration is a terminal actionable state. Network failures use bounded 1, 2, 4, 8, 16, then 30-second reconnect delays. Protocol and payload violations close the session and reconnect; stale audio is never queued.
|
||||
|
||||
## Build and test
|
||||
|
||||
```sh
|
||||
cc -std=c11 -Wall -Wextra -Werror -I main/Source tests/test_voice_protocol.c main/Source/voice_protocol.c -o /tmp/pipecatvoice-protocol-test
|
||||
/tmp/pipecatvoice-protocol-test
|
||||
|
||||
unset PYTHONPATH PYTHONHOME
|
||||
export IDF_PYTHON_ENV_PATH=/Users/adolforeyna/.espressif/python_env/idf5.3_py3.9_env
|
||||
source /Users/adolforeyna/esp/esp-idf/export.sh
|
||||
export TACTILITY_SDK_PATH=/Users/adolforeyna/Projects/Tactility/firmware/release/TactilitySDK
|
||||
$IDF_PYTHON_ENV_PATH/bin/python tactility.py Apps/PipecatVoice build esp32s3 --local-sdk
|
||||
$IDF_PYTHON_ENV_PATH/bin/python tactility.py Apps/PipecatVoice install 192.168.68.112 esp32s3
|
||||
```
|
||||
|
||||
Adapted from ReynaBot (PTT + I2S + WS JSON events: ready/listening/transcript/thinking/response_text/audio_start/audio_end/done/error).
|
||||
The compatible adapter is documented at `/Users/adolforeyna/Projects/voice-assistant/hermes-esp32-voice-gateway/docs/lan-ws-pipecat-adapter.md`. It is the only supported endpoint; Pipecat `:7861` is not an optional device transport.
|
||||
@@ -0,0 +1,89 @@
|
||||
# Pipecat SmallWebRTC ESP32 feasibility spike
|
||||
|
||||
Date: 2026-08-10
|
||||
|
||||
## Decision
|
||||
|
||||
**NO-GO for a Tactility external ELF on the current SDK; GO only for a separate, full ESP-IDF firmware application.**
|
||||
|
||||
Pipecat has an official native ESP32 client, so the required transport is real and source-proven. It cannot presently be used as a Tactility runtime ELF without a firmware/SDK integration project: the client is a full ESP-IDF firmware with private, static component dependencies and system configuration that the ELF loader does not provide or export. Do not replace the rejected raw-WebSocket implementation with another transport until that integration is designed and proven.
|
||||
|
||||
This is not a proposal to flash anything. No device was deployed or flashed during this spike.
|
||||
|
||||
## Source-pinned native client
|
||||
|
||||
| Item | Evidence |
|
||||
| --- | --- |
|
||||
| Client | `https://github.com/pipecat-ai/pipecat-esp32`, commit `e70e3b1f0576e502af9e390434e1a6e8a5cd0d2e`, cloned with all recursive submodules |
|
||||
| Top-level licence | MIT (`LICENSE`, copyright Daily/OpenAI) |
|
||||
| Pipecat server | local installed `pipecat-ai 1.7.0`, Python 3.11 virtual environment |
|
||||
| Server ESP32 support | `SmallWebRTCRequestHandler(..., esp32_mode=True, host=...)` munges the SDP; `smallwebrtc_sdp_munging()` removes SHA-384/SHA-512 fingerprints and retains only the chosen host's ICE candidates |
|
||||
| ESP-IDF build used | local ESP-IDF `v5.5.2`, environment `idf5.5_py3.9_env` |
|
||||
| Native build result | upstream `esp32-s3-box-3` built successfully, including `peer`, `srtp`, `esp-libopus`, Wi-Fi, HTTP client, DTLS/SRTP, Opus, and the ESP-BOX-3 BSP |
|
||||
|
||||
The official client is designed for ESP32-S3 and uses the `libpeer` API. Its `PeerConfiguration` sets `CODEC_OPUS`, creates a peer connection, installs ICE/data/audio callbacks, and invokes `peer_connection_create_offer()`. This is the source-proven native WebRTC implementation; it owns ICE, DTLS-SRTP, RTP, and Opus rather than hand-implementing any of them.
|
||||
|
||||
## Required SmallWebRTC contract
|
||||
|
||||
The product transport is SmallWebRTC HTTP signaling plus WebRTC media, never the old raw WebSocket PCM/JSON protocol:
|
||||
|
||||
1. `POST /start` with `transport: "webrtc"`, `enableDefaultIceServers: false`, and optional `body`; retain the returned `sessionId`.
|
||||
2. `POST /sessions/{sessionId}/api/offer` with `{ "sdp": ..., "type": "offer", "pc_id": optional, "restart_pc": optional, "requestData": optional }`; Pipecat returns SDP answer, type, and `pc_id`.
|
||||
3. `PATCH /sessions/{sessionId}/api/offer` with `{ "pc_id": ..., "candidates": [{ "candidate": ..., "sdp_mid": ..., "sdp_mline_index": ... }] }` for trickle ICE. An empty candidate is the end-of-candidates marker.
|
||||
4. Use the negotiated WebRTC audio track continuously. The runner starts the bot after the offer is processed.
|
||||
|
||||
The local Pipecat source also supports the direct `/api/offer` route used by the current official ESP32 example. The session form above is the approved application contract because it supports Pipecat runner session lifecycle. The live endpoint returned HTTP 200 to `/status`, but it was not restarted with `--esp32`; therefore no live offer/candidate exchange is represented as ESP32 validation.
|
||||
|
||||
## Audio adapter boundary
|
||||
|
||||
The official client source (`media.cpp`) uses 16 kHz, mono, signed 16-bit PCM (`640` bytes = 320 samples = 20 ms) and encodes it as Opus for `peer_connection_send_audio()`. Inbound WebRTC audio reaches the `onaudiotrack` callback as Opus, is decoded to the same PCM shape, and is written to the speaker codec.
|
||||
|
||||
For a future firmware-level integration, Tactility must keep ownership at the following boundary (no hard-coded board pins):
|
||||
|
||||
- acquire the existing Tactility `audio_stream` / `i2s_controller` service;
|
||||
- pull fixed 20 ms frames, 16 kHz mono S16LE, into the native client encoder;
|
||||
- feed decoded remote S16LE frames to the existing output service;
|
||||
- serialize I/O ownership, keep bounded queues, and drop stale audio rather than accumulating latency;
|
||||
- close peer/media callbacks before releasing the audio device.
|
||||
|
||||
The current kernel exports `audio_stream_open_input`, `audio_stream_open_output`, `audio_stream_read`, `audio_stream_write`, `audio_stream_close`, and `i2s_controller_read`/`i2s_controller_write`. Those APIs are the usable boundary, not a reason to configure physical pins in the app.
|
||||
|
||||
## Full-firmware build evidence
|
||||
|
||||
The following was run in a temporary checkout; non-secret placeholder Wi-Fi values were used and no flash command was run:
|
||||
|
||||
```text
|
||||
cd /tmp/pipecat-esp32-spike
|
||||
# cloned pipecat-esp32 at e70e3b1... and initialized all recursive submodules
|
||||
cd esp32-s3-box-3
|
||||
unset PYTHONPATH PYTHONHOME
|
||||
export IDF_PYTHON_ENV_PATH=/Users/adolforeyna/.espressif/python_env/idf5.5_py3.9_env
|
||||
export WIFI_SSID=spike
|
||||
export WIFI_PASSWORD=spike
|
||||
export PIPECAT_SMALLWEBRTC_URL=http://192.168.68.112:7860/api/offer
|
||||
source /Users/adolforeyna/esp/esp-idf/export.sh
|
||||
idf.py build
|
||||
```
|
||||
|
||||
Actual result: `src.elf` and `src.bin` were produced; the IDF build ended with `Project build complete`. `src.bin` is **1,493,408 bytes** and the upstream 1.5 MiB app partition reported **79,712 bytes (5%) free**. `xtensa-esp32s3-elf-size src.elf` reported text `1,304,360`, data `201,012`, bss `2,863,205` (total `4,368,577`). The linked firmware has no undefined dynamic symbols.
|
||||
|
||||
This is important capacity evidence: even before adapting it to the target board and Tactility services, the supported client nearly fills its own dedicated application partition and has a 2.86 MiB BSS footprint.
|
||||
|
||||
## Why this does not link as a Tactility ELF
|
||||
|
||||
Tactility's `TactilitySDK.cmake` calls `project_elf()`. Its loader CMake builds a PIC shared ELF with `-nostartfiles -nostdlib -shared -e app_main`, and links only `main` plus explicitly listed `ELF_COMPONENTS` / `ELF_LIBS`. The current PipecatVoice component declares only `REQUIRES TactilitySDK lwip`.
|
||||
|
||||
The upstream client instead requires full firmware components including `peer`, `srtp`, `esp-libopus`, `esp_http_client`, `esp_wifi`, `nvs_flash`, `esp_psram`, `esp_netif`, mbedTLS, and ESP-BOX-3 BSP. The official `peer` static archive has unresolved references to the linked firmware environment such as `mbedtls_ssl_conf_dtls_srtp_protection_profiles`, `mbedtls_ssl_config_defaults`, `lwip_inet_ntop`, and socket/ICE helpers. The Tactility kernel export table contains the audio service APIs listed above but no `peer_connection`, `opus_*`, `srtp_*`, `mbedtls_*`, `esp_http_client*`, `esp_wifi*`, or `esp_netif*` exports.
|
||||
|
||||
Attempting the ordinary app build also hit a concrete local SDK packaging blocker before linking: `tactility.py Apps/PipecatVoice build esp32s3 --local-sdk` reported that `Buildscripts/TactilitySDK/0.8.0-dev-esp32s3/TactilitySDK` is missing. This must be corrected for later normal ELF builds, but it is distinct from the component/loader incompatibility.
|
||||
|
||||
Therefore copying the client sources or merely adding `REQUIRES peer` would not make a runnable ELF: it would either fail to find the private IDF component libraries during the ELF link or produce imports that the firmware loader cannot resolve. Statically embedding all dependencies is unproven and high-risk because of ELF size, duplicate runtime/library state, SDK configuration, and Wi-Fi/codec ownership conflicts.
|
||||
|
||||
## Security, licensing, and follow-up gate
|
||||
|
||||
- Do not log SDP, ICE details, credentials, or raw audio. The source-level HTTP helper currently logs offer/answer in debug mode; any reused code must remove that logging.
|
||||
- The upstream defaults deliberately disable TLS certificate verification for its demo. Production must use HTTPS with a pinned/validated trust chain; do not inherit that setting.
|
||||
- Preserve MIT notices for Pipecat ESP32 and audit each pinned submodule separately (`libpeer`, SRTP/libSRTP, Opus, and Espressif managed components have their own licenses).
|
||||
- Do not use the existing PipecatVoice raw WebSocket code or its historical configuration as a fallback; it is protocol-incompatible with SmallWebRTC.
|
||||
|
||||
A firmware-level project must first export/package the required WebRTC dependency set, prove an external ELF link with zero unresolved loader symbols (or move the client into firmware), set deterministic memory budgets, and then perform an `--esp32` SmallWebRTC live offer/ICE/media test. Only after that gate may the approved minimal auto-start UI be implemented.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,113 @@
|
||||
#include "voice_protocol.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
/* NOTE: Do not use ctype.h (isalnum/isalpha/isdigit/isspace) in this app. Those
|
||||
* functions read the `_ctype_` table, which is resolved from the flashed firmware
|
||||
* at runtime; the firmware's table does not behave correctly for side-loaded ELF
|
||||
* apps, so isalnum('a') can return false. Use explicit ASCII range checks instead. */
|
||||
|
||||
static bool is_digit(unsigned char c) { return c >= '0' && c <= '9'; }
|
||||
|
||||
static bool is_space(unsigned char c) {
|
||||
return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\v' || c == '\f';
|
||||
}
|
||||
|
||||
static bool is_alnum(unsigned char c) {
|
||||
return is_digit(c) || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
|
||||
}
|
||||
|
||||
static bool copy_part(char* destination, size_t destination_size, const char* start, size_t length) {
|
||||
if (length == 0 || length >= destination_size) return false;
|
||||
memcpy(destination, start, length);
|
||||
destination[length] = '\0';
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool is_loopback(const char* host) {
|
||||
return strcmp(host, "localhost") == 0 || strcmp(host, "::1") == 0 || strncmp(host, "127.", 4) == 0;
|
||||
}
|
||||
|
||||
static bool valid_identifier(const char* value) {
|
||||
if (value == NULL || *value == '\0') return false;
|
||||
for (const unsigned char* p = (const unsigned char*)value; *p; ++p) {
|
||||
if (!is_alnum(*p) && *p != '-' && *p != '_' && *p != '.') return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool pv_parse_endpoint(const char* url, PvEndpoint* endpoint) {
|
||||
if (url == NULL || endpoint == NULL || strncmp(url, "ws://", 5) != 0) return false;
|
||||
const char* authority = url + 5;
|
||||
const char* path = strchr(authority, '/');
|
||||
const char* authority_end = path ? path : authority + strlen(authority);
|
||||
const char* colon = NULL;
|
||||
for (const char* p = authority; p < authority_end; ++p) {
|
||||
if (*p == ':') {
|
||||
if (colon != NULL) return false;
|
||||
colon = p;
|
||||
}
|
||||
if (is_space((unsigned char)*p) || *p == '@' || *p == '?' || *p == '#') return false;
|
||||
}
|
||||
size_t host_length = (size_t)((colon ? colon : authority_end) - authority);
|
||||
if (!copy_part(endpoint->host, sizeof(endpoint->host), authority, host_length) || is_loopback(endpoint->host)) return false;
|
||||
endpoint->port = 80;
|
||||
if (colon != NULL) {
|
||||
unsigned long port = 0;
|
||||
for (const char* p = colon + 1; p < authority_end; ++p) {
|
||||
if (!is_digit((unsigned char)*p)) return false;
|
||||
port = port * 10U + (unsigned long)(*p - '0');
|
||||
if (port > 65535U) return false;
|
||||
}
|
||||
if (port == 0) return false;
|
||||
endpoint->port = (uint16_t)port;
|
||||
}
|
||||
return path == NULL ? copy_part(endpoint->path, sizeof(endpoint->path), "/", 1)
|
||||
: copy_part(endpoint->path, sizeof(endpoint->path), path, strlen(path));
|
||||
}
|
||||
|
||||
bool pv_make_start_json(char* out, size_t out_size, const char* session_id, const char* device_id) {
|
||||
if (out == NULL || !valid_identifier(session_id) || !valid_identifier(device_id)) return false;
|
||||
int written = snprintf(out, out_size,
|
||||
"{\"v\":1,\"event\":\"start\",\"session_id\":\"%s\",\"device_id\":\"%s\",\"audio\":{\"format\":\"pcm_s16le\",\"sample_rate\":16000,\"channels\":1,\"sample_width\":2}}",
|
||||
session_id, device_id);
|
||||
return written > 0 && (size_t)written < out_size;
|
||||
}
|
||||
|
||||
bool pv_valid_pcm_chunk(size_t bytes) {
|
||||
return bytes > 0 && bytes <= PV_PCM_CHUNK_MAX && (bytes % 2U) == 0;
|
||||
}
|
||||
|
||||
bool pv_valid_downstream_audio(const char* format, int sample_rate, int channels, int sample_width, size_t byte_length) {
|
||||
return format != NULL && strcmp(format, "pcm_s16le") == 0 && sample_rate > 0 && sample_rate <= 48000 &&
|
||||
channels == 1 && sample_width == 2 && byte_length > 0 && byte_length <= PV_DOWNSTREAM_MAX &&
|
||||
(byte_length % 2U) == 0;
|
||||
}
|
||||
|
||||
bool pv_binary_matches_metadata(size_t expected_bytes, size_t received_bytes) {
|
||||
return expected_bytes > 0 && expected_bytes == received_bytes;
|
||||
}
|
||||
|
||||
uint32_t pv_retry_delay_seconds(unsigned attempt) {
|
||||
uint32_t delay = 1;
|
||||
while (attempt > 0 && delay < 30) {
|
||||
delay *= 2;
|
||||
--attempt;
|
||||
}
|
||||
return delay > 30 ? 30 : delay;
|
||||
}
|
||||
|
||||
PvState pv_disconnect_state(bool endpoint_valid) {
|
||||
return endpoint_valid ? PV_RECONNECTING : PV_FAILED;
|
||||
}
|
||||
|
||||
const char* pv_state_label(PvState state) {
|
||||
switch (state) {
|
||||
case PV_CONNECTING: return "CONNECTING";
|
||||
case PV_STREAMING: return "STREAMING";
|
||||
case PV_RECONNECTING: return "RECONNECTING";
|
||||
case PV_FAILED: return "FAILED";
|
||||
default: return "FAILED";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
#pragma once
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#define PV_PROTOCOL_VERSION 1
|
||||
#define PV_PCM_CHUNK_MAX 16384U
|
||||
#define PV_DOWNSTREAM_MAX 65536U
|
||||
|
||||
typedef enum {
|
||||
PV_CONNECTING,
|
||||
PV_STREAMING,
|
||||
PV_RECONNECTING,
|
||||
PV_FAILED,
|
||||
} PvState;
|
||||
|
||||
typedef struct {
|
||||
char host[64];
|
||||
char path[96];
|
||||
uint16_t port;
|
||||
} PvEndpoint;
|
||||
|
||||
bool pv_parse_endpoint(const char* url, PvEndpoint* endpoint);
|
||||
bool pv_make_start_json(char* out, size_t out_size, const char* session_id, const char* device_id);
|
||||
bool pv_valid_pcm_chunk(size_t bytes);
|
||||
bool pv_valid_downstream_audio(const char* format, int sample_rate, int channels, int sample_width, size_t byte_length);
|
||||
bool pv_binary_matches_metadata(size_t expected_bytes, size_t received_bytes);
|
||||
uint32_t pv_retry_delay_seconds(unsigned attempt);
|
||||
PvState pv_disconnect_state(bool endpoint_valid);
|
||||
const char* pv_state_label(PvState state);
|
||||
@@ -2,239 +2,166 @@
|
||||
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <lwip/sockets.h>
|
||||
#include <lwip/inet.h>
|
||||
#include <esp_log.h>
|
||||
#include <esp_random.h>
|
||||
#include <lwip/inet.h>
|
||||
#include <lwip/sockets.h>
|
||||
|
||||
static int recv_all(int fd, void* buf, size_t len) {
|
||||
size_t total = 0;
|
||||
char* p = (char*)buf;
|
||||
while (total < len) {
|
||||
int r = lwip_recv(fd, p + total, len - total, 0);
|
||||
if (r <= 0) {
|
||||
return -1;
|
||||
}
|
||||
total += r;
|
||||
#define TAG "PipecatVoiceWs"
|
||||
#define WS_HEADER_LIMIT 1024U
|
||||
#define WS_CONTROL_LIMIT 125U
|
||||
|
||||
static int send_all(int fd, const uint8_t* data, size_t length) {
|
||||
size_t sent = 0;
|
||||
while (sent < length) {
|
||||
int result = lwip_send(fd, data + sent, length - sent, 0);
|
||||
if (result <= 0) return -1;
|
||||
sent += (size_t)result;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static uint16_t my_htons(uint16_t val) {
|
||||
return (uint16_t)(((val & 0xff) << 8) | ((val & 0xff00) >> 8));
|
||||
static int recv_all(int fd, uint8_t* data, size_t length) {
|
||||
size_t received = 0;
|
||||
while (received < length) {
|
||||
int result = lwip_recv(fd, data + received, length - received, 0);
|
||||
if (result <= 0) return -1;
|
||||
received += (size_t)result;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int ws_connect(const char* host, int port, const char* path, const char* device_id, const char* auth_key) {
|
||||
static int discard(int fd, uint64_t length) {
|
||||
uint8_t buffer[256];
|
||||
while (length > 0) {
|
||||
size_t chunk = length > sizeof(buffer) ? sizeof(buffer) : (size_t)length;
|
||||
if (recv_all(fd, buffer, chunk) < 0) return -1;
|
||||
length -= chunk;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int send_frame(int fd, uint8_t opcode, const uint8_t* payload, size_t length) {
|
||||
if (length > 65535U || ((opcode & 0x08U) && length > WS_CONTROL_LIMIT)) return -1;
|
||||
uint8_t header[8];
|
||||
size_t header_length = 2;
|
||||
header[0] = 0x80U | opcode;
|
||||
if (length < 126U) {
|
||||
header[1] = 0x80U | (uint8_t)length;
|
||||
} else {
|
||||
header[1] = 0x80U | 126U;
|
||||
header[2] = (uint8_t)(length >> 8U);
|
||||
header[3] = (uint8_t)length;
|
||||
header_length = 4;
|
||||
}
|
||||
uint8_t mask[4];
|
||||
uint32_t random = esp_random();
|
||||
memcpy(mask, &random, sizeof(mask));
|
||||
memcpy(header + header_length, mask, sizeof(mask));
|
||||
header_length += sizeof(mask);
|
||||
if (send_all(fd, header, header_length) < 0) return -1;
|
||||
|
||||
uint8_t chunk[512];
|
||||
size_t offset = 0;
|
||||
while (offset < length) {
|
||||
size_t count = length - offset > sizeof(chunk) ? sizeof(chunk) : length - offset;
|
||||
for (size_t i = 0; i < count; ++i) chunk[i] = payload[offset + i] ^ mask[(offset + i) % sizeof(mask)];
|
||||
if (send_all(fd, chunk, count) < 0) return -1;
|
||||
offset += count;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int ws_connect(const char* host, int port, const char* path, const char* device_id, const char* api_key) {
|
||||
if (host == NULL || path == NULL || device_id == NULL || api_key == NULL || port < 1 || port > 65535) return -1;
|
||||
int fd = lwip_socket(AF_INET, SOCK_STREAM, 0);
|
||||
if (fd < 0) return -1;
|
||||
|
||||
struct sockaddr_in addr;
|
||||
memset(&addr, 0, sizeof(addr));
|
||||
addr.sin_family = AF_INET;
|
||||
addr.sin_port = my_htons(port);
|
||||
addr.sin_addr.s_addr = ipaddr_addr(host);
|
||||
|
||||
if (lwip_connect(fd, (struct sockaddr*)&addr, sizeof(addr)) < 0) {
|
||||
if (fd < 0) {
|
||||
ESP_LOGW(TAG, "socket create failed");
|
||||
return -1;
|
||||
}
|
||||
struct sockaddr_in address = {0};
|
||||
address.sin_family = AF_INET;
|
||||
address.sin_port = htons((uint16_t)port);
|
||||
address.sin_addr.s_addr = ipaddr_addr(host);
|
||||
if (address.sin_addr.s_addr == IPADDR_NONE) {
|
||||
ESP_LOGW(TAG, "endpoint address parse failed");
|
||||
close(fd);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Set socket receive timeout (e.g. 90 seconds) to prevent blocking indefinitely
|
||||
struct timeval tv;
|
||||
tv.tv_sec = 90;
|
||||
tv.tv_usec = 0;
|
||||
lwip_setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv));
|
||||
|
||||
// Send HTTP upgrade handshake request
|
||||
char req[1024];
|
||||
snprintf(req, sizeof(req),
|
||||
"GET %s HTTP/1.1\r\n"
|
||||
"Host: %s:%d\r\n"
|
||||
"Upgrade: websocket\r\n"
|
||||
"Connection: Upgrade\r\n"
|
||||
"Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n"
|
||||
"Sec-WebSocket-Version: 13\r\n"
|
||||
"Authorization: Bearer %s\r\n"
|
||||
"X-Device-ID: %s\r\n"
|
||||
"\r\n",
|
||||
path, host, port, auth_key, device_id);
|
||||
|
||||
if (lwip_send(fd, req, strlen(req), 0) < 0) {
|
||||
if (lwip_connect(fd, (struct sockaddr*)&address, sizeof(address)) < 0) {
|
||||
ESP_LOGW(TAG, "TCP connect failed");
|
||||
close(fd);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Read HTTP response headers until we hit "\r\n\r\n"
|
||||
char header_buf[1024];
|
||||
size_t header_len = 0;
|
||||
while (header_len < sizeof(header_buf) - 1) {
|
||||
char c;
|
||||
int r = lwip_recv(fd, &c, 1, 0);
|
||||
if (r <= 0) {
|
||||
struct timeval timeout = {.tv_sec = 15, .tv_usec = 0};
|
||||
lwip_setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout));
|
||||
char request[WS_HEADER_LIMIT];
|
||||
int request_length = snprintf(request, sizeof(request),
|
||||
"GET %s HTTP/1.1\r\nHost: %s:%d\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n"
|
||||
"Sec-WebSocket-Key: MDEyMzQ1Njc4OWFiY2RlZg==\r\nSec-WebSocket-Version: 13\r\n"
|
||||
"Authorization: Bearer %s\r\nX-Device-ID: %s\r\n\r\n",
|
||||
path, host, port, api_key, device_id);
|
||||
if (request_length < 0 || (size_t)request_length >= sizeof(request) || send_all(fd, (const uint8_t*)request, (size_t)request_length) < 0) {
|
||||
ESP_LOGW(TAG, "WebSocket upgrade request failed");
|
||||
close(fd);
|
||||
return -1;
|
||||
}
|
||||
char response[WS_HEADER_LIMIT];
|
||||
size_t length = 0;
|
||||
while (length + 1 < sizeof(response)) {
|
||||
if (recv_all(fd, (uint8_t*)&response[length], 1) < 0) {
|
||||
ESP_LOGW(TAG, "WebSocket upgrade response failed");
|
||||
close(fd);
|
||||
return -1;
|
||||
}
|
||||
header_buf[header_len++] = c;
|
||||
header_buf[header_len] = '\0';
|
||||
|
||||
if (header_len >= 4 && strcmp(header_buf + header_len - 4, "\r\n\r\n") == 0) {
|
||||
break;
|
||||
}
|
||||
response[++length] = '\0';
|
||||
if (length >= 4 && memcmp(response + length - 4, "\r\n\r\n", 4) == 0) break;
|
||||
}
|
||||
|
||||
// Verify HTTP 101 Switching Protocols response status
|
||||
if (strstr(header_buf, "HTTP/1.1 101") == NULL && strstr(header_buf, "HTTP/1.0 101") == NULL) {
|
||||
if (length + 1 >= sizeof(response) || strstr(response, " 101 ") == NULL) {
|
||||
ESP_LOGW(TAG, "WebSocket upgrade rejected");
|
||||
close(fd);
|
||||
return -1;
|
||||
}
|
||||
|
||||
ESP_LOGI(TAG, "WebSocket upgrade accepted");
|
||||
return fd;
|
||||
}
|
||||
|
||||
int ws_send(int fd, const uint8_t* data, size_t len, bool binary) {
|
||||
uint8_t header[10];
|
||||
size_t header_len = 0;
|
||||
|
||||
header[0] = binary ? 0x82 : 0x81;
|
||||
|
||||
if (len < 126) {
|
||||
header[1] = 0x80 | (uint8_t)len;
|
||||
header_len = 2;
|
||||
} else {
|
||||
header[1] = 0x80 | 126;
|
||||
header[2] = (uint8_t)((len >> 8) & 0xFF);
|
||||
header[3] = (uint8_t)(len & 0xFF);
|
||||
header_len = 4;
|
||||
}
|
||||
|
||||
// Use fixed client mask for performance: 0x12, 0x34, 0x56, 0x78
|
||||
uint8_t mask[4] = { 0x12, 0x34, 0x56, 0x78 };
|
||||
memcpy(header + header_len, mask, 4);
|
||||
header_len += 4;
|
||||
|
||||
// Send WebSocket frame header
|
||||
int sent = lwip_send(fd, header, header_len, 0);
|
||||
if (sent < 0) return -1;
|
||||
|
||||
// Mask the payload
|
||||
uint8_t* masked = malloc(len);
|
||||
if (masked == NULL) return -1;
|
||||
for (size_t i = 0; i < len; ++i) {
|
||||
masked[i] = data[i] ^ mask[i % 4];
|
||||
}
|
||||
|
||||
// Send masked payload
|
||||
sent = lwip_send(fd, masked, len, 0);
|
||||
free(masked);
|
||||
|
||||
return sent >= 0 ? 0 : -1;
|
||||
int ws_send(int fd, const uint8_t* data, size_t length, bool binary) {
|
||||
if (fd < 0 || data == NULL || length == 0) return -1;
|
||||
return send_frame(fd, binary ? 0x02U : 0x01U, data, length);
|
||||
}
|
||||
|
||||
int ws_recv(int fd, int* out_opcode, uint8_t* payload, size_t max_len) {
|
||||
int ws_recv(int fd, int* opcode, bool* final, uint8_t* payload, size_t maximum) {
|
||||
uint8_t header[2];
|
||||
if (recv_all(fd, header, 2) < 0) {
|
||||
return -1;
|
||||
if (fd < 0 || recv_all(fd, header, sizeof(header)) < 0) return -1;
|
||||
uint64_t length = header[1] & 0x7fU;
|
||||
if (length == 126U) {
|
||||
uint8_t extended[2];
|
||||
if (recv_all(fd, extended, sizeof(extended)) < 0) return -1;
|
||||
length = ((uint64_t)extended[0] << 8U) | extended[1];
|
||||
} else if (length == 127U) {
|
||||
uint8_t extended[8];
|
||||
if (recv_all(fd, extended, sizeof(extended)) < 0) return -1;
|
||||
length = 0;
|
||||
for (size_t i = 0; i < sizeof(extended); ++i) length = (length << 8U) | extended[i];
|
||||
}
|
||||
|
||||
int opcode = header[0] & 0x0F;
|
||||
if (out_opcode != NULL) {
|
||||
*out_opcode = opcode;
|
||||
bool masked = (header[1] & 0x80U) != 0;
|
||||
uint8_t mask[4] = {0};
|
||||
if (masked && recv_all(fd, mask, sizeof(mask)) < 0) return -1;
|
||||
uint8_t frame_opcode = header[0] & 0x0fU;
|
||||
if (((frame_opcode & 0x08U) && (length > WS_CONTROL_LIMIT || !(header[0] & 0x80U))) || length > maximum) {
|
||||
if (discard(fd, length) < 0) return -1;
|
||||
return -2;
|
||||
}
|
||||
|
||||
int masked = (header[1] & 0x80) != 0;
|
||||
size_t len = header[1] & 0x7F;
|
||||
|
||||
if (len == 126) {
|
||||
uint8_t ext_len[2];
|
||||
if (recv_all(fd, ext_len, 2) < 0) return -1;
|
||||
len = ((size_t)ext_len[0] << 8) | ext_len[1];
|
||||
} else if (len == 127) {
|
||||
uint8_t ext_len[8];
|
||||
if (recv_all(fd, ext_len, 8) < 0) return -1;
|
||||
// Parse 64-bit length into size_t
|
||||
len = ((size_t)ext_len[4] << 24) | ((size_t)ext_len[5] << 16) | ((size_t)ext_len[6] << 8) | ext_len[7];
|
||||
}
|
||||
|
||||
if (masked) {
|
||||
uint8_t mask[4];
|
||||
if (recv_all(fd, mask, 4) < 0) return -1;
|
||||
|
||||
if (len > max_len) {
|
||||
ESP_LOGE("websocket", "ws_recv overflow (masked): len=%u, max_len=%u", (unsigned)len, (unsigned)max_len);
|
||||
// Buffer overflow, skip payload to align stream
|
||||
size_t to_discard = len;
|
||||
uint8_t discard_buf[256];
|
||||
while (to_discard > 0) {
|
||||
size_t chunk = to_discard < sizeof(discard_buf) ? to_discard : sizeof(discard_buf);
|
||||
if (recv_all(fd, discard_buf, chunk) < 0) return -1;
|
||||
to_discard -= chunk;
|
||||
}
|
||||
return -2;
|
||||
}
|
||||
|
||||
if (recv_all(fd, payload, len) < 0) return -1;
|
||||
for (size_t i = 0; i < len; ++i) {
|
||||
payload[i] ^= mask[i % 4];
|
||||
}
|
||||
} else {
|
||||
if (len > max_len) {
|
||||
ESP_LOGE("websocket", "ws_recv overflow (unmasked): len=%u, max_len=%u", (unsigned)len, (unsigned)max_len);
|
||||
size_t to_discard = len;
|
||||
uint8_t discard_buf[256];
|
||||
while (to_discard > 0) {
|
||||
size_t chunk = to_discard < sizeof(discard_buf) ? to_discard : sizeof(discard_buf);
|
||||
if (recv_all(fd, discard_buf, chunk) < 0) return -1;
|
||||
to_discard -= chunk;
|
||||
}
|
||||
return -2;
|
||||
}
|
||||
|
||||
if (recv_all(fd, payload, len) < 0) return -1;
|
||||
}
|
||||
|
||||
return (int)len;
|
||||
if (length > 0 && recv_all(fd, payload, (size_t)length) < 0) return -1;
|
||||
if (masked) for (size_t i = 0; i < (size_t)length; ++i) payload[i] ^= mask[i % sizeof(mask)];
|
||||
if (opcode) *opcode = frame_opcode;
|
||||
if (final) *final = (header[0] & 0x80U) != 0;
|
||||
return (int)length;
|
||||
}
|
||||
|
||||
void ws_close(int fd) {
|
||||
if (fd >= 0) {
|
||||
close(fd);
|
||||
}
|
||||
}
|
||||
|
||||
int ws_send_pong(int fd, const uint8_t* payload, size_t len) {
|
||||
uint8_t header[10];
|
||||
size_t header_len = 0;
|
||||
|
||||
header[0] = 0x8A; // FIN | PONG (0x0A)
|
||||
|
||||
if (len < 126) {
|
||||
header[1] = 0x80 | (uint8_t)len;
|
||||
header_len = 2;
|
||||
} else {
|
||||
header[1] = 0x80 | 126;
|
||||
header[2] = (uint8_t)((len >> 8) & 0xFF);
|
||||
header[3] = (uint8_t)(len & 0xFF);
|
||||
header_len = 4;
|
||||
}
|
||||
|
||||
uint8_t mask[4] = { 0x12, 0x34, 0x56, 0x78 };
|
||||
memcpy(header + header_len, mask, 4);
|
||||
header_len += 4;
|
||||
|
||||
int sent = lwip_send(fd, header, header_len, 0);
|
||||
if (sent < 0) return -1;
|
||||
|
||||
if (len > 0 && payload != NULL) {
|
||||
uint8_t* masked = malloc(len);
|
||||
if (masked == NULL) return -1;
|
||||
for (size_t i = 0; i < len; ++i) {
|
||||
masked[i] = payload[i] ^ mask[i % 4];
|
||||
}
|
||||
sent = lwip_send(fd, masked, len, 0);
|
||||
free(masked);
|
||||
}
|
||||
|
||||
return sent >= 0 ? 0 : -1;
|
||||
}
|
||||
int ws_send_pong(int fd, const uint8_t* payload, size_t length) { return send_frame(fd, 0x0aU, payload, length); }
|
||||
int ws_send_close(int fd) { return send_frame(fd, 0x08U, NULL, 0); }
|
||||
void ws_close(int fd) { if (fd >= 0) close(fd); }
|
||||
@@ -14,10 +14,10 @@ extern "C" {
|
||||
* @param port Port number (e.g. 8642)
|
||||
* @param path WebSocket path (e.g. "/api/esp32/voice/ws")
|
||||
* @param device_id Unique device identifier
|
||||
* @param auth_key Hermes Bearer API key
|
||||
* @param api_key Optional profile API key; never compiled into firmware
|
||||
* @return Socket file descriptor on success, or -1 on failure
|
||||
*/
|
||||
int ws_connect(const char* host, int port, const char* path, const char* device_id, const char* auth_key);
|
||||
int ws_connect(const char* host, int port, const char* path, const char* device_id, const char* api_key);
|
||||
|
||||
/**
|
||||
* Send a WebSocket frame.
|
||||
@@ -37,7 +37,7 @@ int ws_send(int fd, const uint8_t* data, size_t len, bool binary);
|
||||
* @param max_len Maximum length of the payload buffer
|
||||
* @return Received payload length on success, -1 on connection failure, or -2 on buffer overflow
|
||||
*/
|
||||
int ws_recv(int fd, int* out_opcode, uint8_t* payload, size_t max_len);
|
||||
int ws_recv(int fd, int* out_opcode, bool* out_final, uint8_t* payload, size_t max_len);
|
||||
|
||||
/**
|
||||
* Close a WebSocket connection.
|
||||
@@ -54,6 +54,9 @@ void ws_close(int fd);
|
||||
*/
|
||||
int ws_send_pong(int fd, const uint8_t* payload, size_t len);
|
||||
|
||||
/** Send a clean WebSocket close control frame before closing the socket. */
|
||||
int ws_send_close(int fd);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
#include "voice_protocol.h"
|
||||
|
||||
#include <assert.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
static void test_endpoint_validation(void) {
|
||||
PvEndpoint endpoint;
|
||||
assert(pv_parse_endpoint("ws://192.168.68.102:8644/api/esp32/voice/ws", &endpoint));
|
||||
assert(strcmp(endpoint.host, "192.168.68.102") == 0);
|
||||
assert(endpoint.port == 8644);
|
||||
assert(strcmp(endpoint.path, "/api/esp32/voice/ws") == 0);
|
||||
assert(!pv_parse_endpoint("ws://127.0.0.1:8642/api", &endpoint));
|
||||
assert(!pv_parse_endpoint("ws://localhost:8642/api", &endpoint));
|
||||
assert(!pv_parse_endpoint("wss://192.168.68.102/api", &endpoint));
|
||||
assert(!pv_parse_endpoint("ws://192.168.68.102:0/api", &endpoint));
|
||||
}
|
||||
|
||||
static void test_start_and_pcm_boundaries(void) {
|
||||
char json[256];
|
||||
assert(pv_make_start_json(json, sizeof(json), "session-01", "tactility-14c19d1a790"));
|
||||
assert(strstr(json, "\"v\":1") != NULL);
|
||||
assert(strstr(json, "\"pcm_s16le\"") != NULL);
|
||||
assert(!pv_make_start_json(json, sizeof(json), "bad session", "device"));
|
||||
assert(pv_valid_pcm_chunk(2));
|
||||
assert(pv_valid_pcm_chunk(PV_PCM_CHUNK_MAX));
|
||||
assert(!pv_valid_pcm_chunk(0));
|
||||
assert(!pv_valid_pcm_chunk(3));
|
||||
assert(!pv_valid_pcm_chunk(PV_PCM_CHUNK_MAX + 2));
|
||||
assert(pv_valid_downstream_audio("pcm_s16le", 24000, 1, 2, 48000));
|
||||
assert(!pv_valid_downstream_audio("wav", 24000, 1, 2, 48000));
|
||||
assert(!pv_valid_downstream_audio("pcm_s16le", 24000, 2, 2, 48000));
|
||||
assert(!pv_valid_downstream_audio("pcm_s16le", 24000, 1, 2, PV_DOWNSTREAM_MAX + 2));
|
||||
assert(pv_binary_matches_metadata(48000, 48000));
|
||||
assert(!pv_binary_matches_metadata(48000, 47998));
|
||||
}
|
||||
|
||||
static void test_retry_and_state(void) {
|
||||
const uint32_t expected[] = {1, 2, 4, 8, 16, 30, 30};
|
||||
for (unsigned i = 0; i < sizeof(expected) / sizeof(expected[0]); ++i) assert(pv_retry_delay_seconds(i) == expected[i]);
|
||||
assert(pv_disconnect_state(true) == PV_RECONNECTING);
|
||||
assert(pv_disconnect_state(false) == PV_FAILED);
|
||||
assert(strcmp(pv_state_label(PV_STREAMING), "STREAMING") == 0);
|
||||
}
|
||||
|
||||
int main(void) {
|
||||
test_endpoint_validation();
|
||||
test_start_and_pcm_boundaries();
|
||||
test_retry_and_state();
|
||||
puts("voice_protocol tests passed");
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user