Add speech transcription tools: speech_list_locales, speech_transcribe_file, speech_quick_test
Implements Apple SpeechAnalyzer + SpeechTranscriber (macOS 26+) live transcription per official docs, with Inscribe bugfix (finalizeAndFinishThroughEndOfInput) - speech_list_locales: isAvailable, supported/installed locales, reserved counts - speech_transcribe_file: transcribe any AVFoundation-readable file, returns transcript + segments + elapsed + realtimeFactor + engine metadata - speech_quick_test: say + transcribe loopback to verify pipeline Swift binary compiled on-demand via swiftc -O with Speech + AVFoundation Follows docs: AssetInventory downloadAndInstall, AssetInputSequenceProvider Mac mini is 26.5.2 Mac16,10 M4 so engine verified available
This commit is contained in:
aeroreyna
@@ -0,0 +1,370 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { writeFile, unlink, mkdir, readFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import os from "node:os";
|
||||
import fs from "node:fs";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const OUTPUT_BASE = path.join(os.homedir(), "Projects", "MacMiniMCP", "generated-transcriptions");
|
||||
|
||||
// ---------- SWIFT SOURCE BUILDER ----------
|
||||
// Follows Apple docs https://docs.developer.apple.com/tutorials/data/documentation/speech/speechanalyzer.md
|
||||
// Key bug from Inscribe benchmark: you MUST call finalizeAndFinishThroughEndOfInput() after feeding input,
|
||||
// not just finish the AsyncStream builder.
|
||||
|
||||
function swiftSourceForFileTranscription() {
|
||||
return String.raw`
|
||||
import Speech
|
||||
import AVFoundation
|
||||
import Foundation
|
||||
|
||||
@main
|
||||
struct TranscribeCLI {
|
||||
static func main() async {
|
||||
let args = CommandLine.arguments
|
||||
let audioPath = args.count > 1 ? args[1] : ""
|
||||
let localeId = args.count > 2 ? args[2] : "en-US"
|
||||
let jsonOut = args.count > 3 ? args[3] : ""
|
||||
|
||||
if audioPath.isEmpty {
|
||||
fputs("Usage: transcriber <audioPath> [locale] [jsonOutPath]\n", stderr)
|
||||
exit(1)
|
||||
}
|
||||
|
||||
let startTime = CFAbsoluteTimeGetCurrent()
|
||||
|
||||
guard FileManager.default.fileExists(atPath: audioPath) else {
|
||||
fputs("ERROR: Audio file not found: \(audioPath)\n", stderr)
|
||||
exit(3)
|
||||
}
|
||||
|
||||
let audioURL = URL(fileURLWithPath: audioPath)
|
||||
let requestedLocale = Locale(identifier: localeId)
|
||||
guard let locale = SpeechTranscriber.supportedLocale(equivalentTo: requestedLocale) ?? SpeechTranscriber.supportedLocale(equivalentTo: Locale(identifier: "en-US")) else {
|
||||
fputs("ERROR: No supported locale for \(localeId)\n", stderr)
|
||||
exit(2)
|
||||
}
|
||||
|
||||
fputs("Locale: \(locale.identifier) (requested \(localeId))\n", stderr)
|
||||
fputs("isAvailable: \(SpeechTranscriber.isAvailable)\n", stderr)
|
||||
fputs("installedLocales: \(SpeechTranscriber.installedLocales.map{$0.identifier})\n", stderr)
|
||||
fputs("supportedLocales count: \(SpeechTranscriber.supportedLocales.count)\n", stderr)
|
||||
|
||||
let transcriber = SpeechTranscriber(locale: locale, preset: .transcription)
|
||||
|
||||
// Asset install
|
||||
do {
|
||||
if let req = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
|
||||
fputs("Downloading assets for \(locale.identifier)...\n", stderr)
|
||||
try await req.downloadAndInstall()
|
||||
fputs("Assets ready\n", stderr)
|
||||
} else {
|
||||
fputs("No asset download needed\n", stderr)
|
||||
}
|
||||
} catch {
|
||||
fputs("Asset install note (continuing): \(error)\n", stderr)
|
||||
}
|
||||
|
||||
// Open with AVAudioFile for info
|
||||
let avFile: AVAudioFile
|
||||
do {
|
||||
avFile = try AVAudioFile(forReading: audioURL)
|
||||
} catch {
|
||||
fputs("ERROR opening audio: \(error)\n", stderr)
|
||||
exit(4)
|
||||
}
|
||||
fputs("Audio format: \(avFile.fileFormat) \(avFile.length) frames sr=\(avFile.processingFormat.sampleRate)\n", stderr)
|
||||
|
||||
do {
|
||||
// AssetInputSequenceProvider is the recommended file path
|
||||
let provider: AssetInputSequenceProvider
|
||||
do {
|
||||
provider = try AssetInputSequenceProvider(fileURL: audioURL)
|
||||
} catch {
|
||||
fputs("AssetInputSequenceProvider init failed: \(error), trying Capture path fallback via AVAudioFile direct read\n", stderr)
|
||||
throw error
|
||||
}
|
||||
|
||||
let analyzer = SpeechAnalyzer(modules: [transcriber])
|
||||
|
||||
var allSegments: [String] = []
|
||||
var allAttributed: [String] = []
|
||||
|
||||
// Collect results concurrently - must start BEFORE analyze
|
||||
let collector = Task {
|
||||
do {
|
||||
for try await r in transcriber.results {
|
||||
let plain = String(r.text.characters)
|
||||
if !plain.isEmpty {
|
||||
allSegments.append(plain)
|
||||
// For debugging, also log alternatives count
|
||||
fputs("[result] \(plain)\n", stderr)
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
fputs("Results error: \(error)\n", stderr)
|
||||
}
|
||||
}
|
||||
|
||||
fputs("Starting analysis...\n", stderr)
|
||||
// Start and finalize pattern - per Inscribe bug report MUST call finalizeAndFinishThroughEndOfInput()
|
||||
try await analyzer.start(inputSequence: provider.analyzerInputs)
|
||||
try await analyzer.finalizeAndFinishThroughEndOfInput()
|
||||
|
||||
await collector.value
|
||||
|
||||
let elapsed = CFAbsoluteTimeGetCurrent() - startTime
|
||||
let full = allSegments.joined(separator: " ")
|
||||
let durationSec = Double(avFile.length) / avFile.processingFormat.sampleRate
|
||||
|
||||
let payload: [String: Any] = [
|
||||
"ok": true,
|
||||
"engine": "SpeechAnalyzer+SpeechTranscriber",
|
||||
"locale": locale.identifier,
|
||||
"requestedLocale": localeId,
|
||||
"transcript": full,
|
||||
"segments": allSegments,
|
||||
"elapsedSeconds": elapsed,
|
||||
"audioPath": audioPath,
|
||||
"durationSeconds": durationSec,
|
||||
"realtimeFactor": durationSec > 0 ? elapsed / durationSec : 0,
|
||||
"frames": Int(avFile.length),
|
||||
"sampleRate": avFile.processingFormat.sampleRate,
|
||||
"macOS": ProcessInfo.processInfo.operatingSystemVersionString,
|
||||
"isAvailable": SpeechTranscriber.isAvailable,
|
||||
"installedLocales": SpeechTranscriber.installedLocales.map{$0.identifier}
|
||||
]
|
||||
|
||||
let dataOut = try JSONSerialization.data(withJSONObject: payload, options: [.prettyPrinted, .sortedKeys])
|
||||
if !jsonOut.isEmpty {
|
||||
try dataOut.write(to: URL(fileURLWithPath: jsonOut))
|
||||
}
|
||||
FileHandle.standardOutput.write(dataOut)
|
||||
|
||||
} catch {
|
||||
fputs("Analysis failed: \(error)\n", stderr)
|
||||
// Dump full error chain
|
||||
var err: Error? = error
|
||||
while let e = err {
|
||||
fputs(" -> \(e)\n", stderr)
|
||||
if let le = e as? LocalizedError {
|
||||
if let desc = le.errorDescription { fputs(" desc: \(desc)\n", stderr) }
|
||||
if let fail = le.failureReason { fputs(" reason: \(fail)\n", stderr) }
|
||||
}
|
||||
err = (e as NSError).underlyingErrors.first ?? (e as NSError).userInfo[NSUnderlyingErrorKey] as? Error
|
||||
}
|
||||
exit(6)
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
}
|
||||
|
||||
// ---------- MAIN EXPORTS ----------
|
||||
|
||||
async function ensureOutputDir() {
|
||||
await mkdir(OUTPUT_BASE, { recursive: true });
|
||||
}
|
||||
|
||||
async function buildAndRunTranscriber({ audioPath, locale = "en-US" }) {
|
||||
const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "speech-transcribe-"));
|
||||
const swiftFile = path.join(tmpDir, "TranscriberMain.swift");
|
||||
const binFile = path.join(tmpDir, "transcriber_bin");
|
||||
const jsonOut = path.join(tmpDir, "result.json");
|
||||
|
||||
await writeFile(swiftFile, swiftSourceForFileTranscription(), "utf8");
|
||||
|
||||
// Compile with swiftc
|
||||
// Need AVFoundation + Speech, so link frameworks automatically via swiftc
|
||||
try {
|
||||
fputsStderr(`Compiling transcriber for ${audioPath}...\n`);
|
||||
const compileCmd = await execFileAsync("/usr/bin/swiftc", [
|
||||
"-O",
|
||||
swiftFile,
|
||||
"-o", binFile,
|
||||
"-framework", "AVFoundation",
|
||||
"-framework", "Speech",
|
||||
], { timeout: 60_000, maxBuffer: 10 * 1024 * 1024 });
|
||||
if (compileCmd.stderr) fputsStderr(compileCmd.stderr);
|
||||
} catch (e) {
|
||||
await cleanup(tmpDir);
|
||||
throw new Error(`swiftc compile failed: ${e.stderr || e.message}\n${e.stdout || ""}`);
|
||||
}
|
||||
|
||||
try {
|
||||
await stat(binFile);
|
||||
} catch {
|
||||
await cleanup(tmpDir);
|
||||
throw new Error("Compiled binary not found after swiftc");
|
||||
}
|
||||
|
||||
// Run binary
|
||||
try {
|
||||
fputsStderr(`Running transcriber binary...\n`);
|
||||
const { stdout, stderr } = await execFileAsync(binFile, [audioPath, locale, jsonOut], {
|
||||
timeout: 120_000,
|
||||
maxBuffer: 20 * 1024 * 1024,
|
||||
});
|
||||
if (stderr) fputsStderr(`transcriber stderr: ${stderr.slice(0, 4000)}\n`);
|
||||
// Try jsonOut file first, then stdout
|
||||
let resultJson;
|
||||
try {
|
||||
const raw = await readFile(jsonOut, "utf8");
|
||||
resultJson = JSON.parse(raw);
|
||||
} catch {
|
||||
resultJson = JSON.parse(stdout);
|
||||
}
|
||||
await ensureOutputDir();
|
||||
// Copy to persistent location
|
||||
const persistName = `transcription-${Date.now()}.json`;
|
||||
const persistPath = path.join(OUTPUT_BASE, persistName);
|
||||
try {
|
||||
await writeFile(persistPath, JSON.stringify(resultJson, null, 2), "utf8");
|
||||
resultJson.persistedTo = persistPath;
|
||||
} catch {}
|
||||
await cleanup(tmpDir);
|
||||
return resultJson;
|
||||
} catch (e) {
|
||||
const errOut = e.stdout || "";
|
||||
const errErr = e.stderr || e.message || "";
|
||||
// Try to parse partial stdout as json
|
||||
try {
|
||||
const maybe = JSON.parse(e.stdout || "");
|
||||
if (maybe && maybe.ok) {
|
||||
await cleanup(tmpDir);
|
||||
return maybe;
|
||||
}
|
||||
} catch {}
|
||||
await cleanup(tmpDir);
|
||||
throw new Error(`Transcriber run failed\nSTDOUT: ${errOut.slice(0, 3000)}\nSTDERR: ${errErr.slice(0, 5000)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function fputsStderr(s) {
|
||||
try { process.stderr.write(s); } catch {}
|
||||
}
|
||||
|
||||
async function cleanup(dir) {
|
||||
try {
|
||||
const { rm } = await import("node:fs/promises");
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
} catch {}
|
||||
}
|
||||
|
||||
// ---- Tool functions exposed to MCP ----
|
||||
|
||||
export async function speechListLocales() {
|
||||
const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "speech-locale-"));
|
||||
const swiftFile = path.join(tmpDir, "ListLocales.swift");
|
||||
const binFile = path.join(tmpDir, "list_bin");
|
||||
const swiftSrc = String.raw`
|
||||
import Speech
|
||||
import Foundation
|
||||
|
||||
@main
|
||||
struct L {
|
||||
static func main() async {
|
||||
// Ensure we trigger asset inventory awareness
|
||||
let payload: [String: Any] = [
|
||||
"isAvailable": SpeechTranscriber.isAvailable,
|
||||
"supported": SpeechTranscriber.supportedLocales.map{$0.identifier}.sorted(),
|
||||
"installed": SpeechTranscriber.installedLocales.map{$0.identifier}.sorted(),
|
||||
"macOS": ProcessInfo.processInfo.operatingSystemVersionString,
|
||||
"maxReserved": AssetInventory.maximumReservedLocales,
|
||||
"reserved": AssetInventory.reservedLocales.map{$0.identifier}
|
||||
]
|
||||
let d = try! JSONSerialization.data(withJSONObject: payload, options: [.prettyPrinted, .sortedKeys])
|
||||
FileHandle.standardOutput.write(d)
|
||||
}
|
||||
}
|
||||
`;
|
||||
await writeFile(swiftFile, swiftSrc, "utf8");
|
||||
try {
|
||||
await execFileAsync("/usr/bin/swiftc", ["-O", swiftFile, "-o", binFile, "-framework", "Speech"], { timeout: 30_000 });
|
||||
const { stdout } = await execFileAsync(binFile, [], { timeout: 15_000 });
|
||||
await cleanup(tmpDir);
|
||||
return JSON.parse(stdout);
|
||||
} catch (e) {
|
||||
await cleanup(tmpDir);
|
||||
throw new Error(`List locales failed: ${e.stderr || e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function speechTranscribeFile({ filePath, locale }) {
|
||||
if (!filePath) throw new Error("filePath required");
|
||||
// Security: must be inside allowed dirs or /tmp
|
||||
const allowedPrefixes = [
|
||||
os.homedir(),
|
||||
"/tmp/",
|
||||
"/var/tmp/",
|
||||
os.tmpdir() + "/",
|
||||
];
|
||||
const resolved = path.resolve(filePath);
|
||||
const isAllowed = allowedPrefixes.some(p => resolved.startsWith(path.resolve(p)));
|
||||
if (!isAllowed) {
|
||||
// Still allow if file exists and is under /Users/adolforeyna/ (we're on Mac)
|
||||
if (!resolved.startsWith("/Users/adolforeyna/") && !resolved.startsWith("/var/folders/")) {
|
||||
throw new Error(`filePath not in allowed area: ${resolved}`);
|
||||
}
|
||||
}
|
||||
try {
|
||||
await stat(resolved);
|
||||
} catch {
|
||||
throw new Error(`File not found: ${resolved}`);
|
||||
}
|
||||
return await buildAndRunTranscriber({ audioPath: resolved, locale: locale || "en-US" });
|
||||
}
|
||||
|
||||
export async function speechQuickTest({ text, voice } = {}) {
|
||||
// Generate a test audio using say command, then transcribe it - end-to-end loopback test
|
||||
// This tests the engine without needing a real file from user
|
||||
const testText = text || "Hello world this is a test of Apple SpeechAnalyzer on the Mac mini M four";
|
||||
const testVoice = voice || "Alex";
|
||||
const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "speech-qtest-"));
|
||||
const wavPath = path.join(tmpDir, "test.wav"); // say can do aiff, we convert?
|
||||
const aiffPath = path.join(tmpDir, "test.aiff");
|
||||
|
||||
try {
|
||||
// say -> aiff (macOS say produces aiff/wav)
|
||||
await execFileAsync("/usr/bin/say", ["-v", testVoice, "-o", aiffPath, testText], { timeout: 15_000 });
|
||||
} catch (e) {
|
||||
// Try default voice
|
||||
try {
|
||||
await execFileAsync("/usr/bin/say", ["-o", aiffPath, testText], { timeout: 15_000 });
|
||||
} catch (e2) {
|
||||
await cleanup(tmpDir);
|
||||
throw new Error(`say command failed: ${e2.stderr || e2.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await stat(aiffPath);
|
||||
} catch {
|
||||
await cleanup(tmpDir);
|
||||
throw new Error("Generated audio file not found");
|
||||
}
|
||||
|
||||
// Now transcribe the generated aiff
|
||||
// AVFoundation can read aiff directly
|
||||
let result;
|
||||
try {
|
||||
result = await buildAndRunTranscriber({ audioPath: aiffPath, locale: "en-US" });
|
||||
} catch (e) {
|
||||
await cleanup(tmpDir);
|
||||
throw e;
|
||||
}
|
||||
result.testInputText = testText;
|
||||
result.testVoice = testVoice;
|
||||
result.generatedAudioPath = aiffPath;
|
||||
// Don't cleanup yet? cleanup our temp but keep result with attempted path
|
||||
// Keep aiff for debugging if needed - move to output
|
||||
try {
|
||||
await ensureOutputDir();
|
||||
const destAiff = path.join(OUTPUT_BASE, `qtest-${Date.now()}.aiff`);
|
||||
await execFileAsync("/bin/cp", [aiffPath, destAiff], { timeout: 5_000 });
|
||||
result.persistedAudio = destAiff;
|
||||
} catch {}
|
||||
await cleanup(tmpDir);
|
||||
return result;
|
||||
}
|
||||
@@ -31,6 +31,11 @@ import {
|
||||
listReminders,
|
||||
} from "./integrations/reminders.js";
|
||||
import { getSystemInfo, getSpeechApiStatus } from "./integrations/system.js";
|
||||
import {
|
||||
speechTranscribeFile,
|
||||
speechListLocales,
|
||||
speechQuickTest,
|
||||
} from "./integrations/speech-transcribe.js";
|
||||
|
||||
const jsonResult = (value) => ({
|
||||
content: [{ type: "text", text: JSON.stringify(value, null, 2) }],
|
||||
@@ -253,6 +258,33 @@ export function createMacMiniMcpServer() {
|
||||
handled(() => getSpeechApiStatus()),
|
||||
);
|
||||
|
||||
server.tool(
|
||||
"speech_list_locales",
|
||||
"List SpeechTranscriber locales: isAvailable, supported, installed, reserved, maxReserved. Use to check if your language model is ready before transcribing.",
|
||||
{},
|
||||
handled(() => speechListLocales()),
|
||||
);
|
||||
|
||||
server.tool(
|
||||
"speech_transcribe_file",
|
||||
"Transcribe an audio file using Apple's new SpeechAnalyzer + SpeechTranscriber (macOS 26+). Fastest and most accurate English engine on-device per Inscribe benchmark (2.12% WER). Supports m4a, wav, aiff, mp3, etc via AVFoundation. Returns transcript, segments, timing, realtime factor.",
|
||||
{
|
||||
filePath: z.string().min(1).describe("Absolute path to audio file on the Mac. Can be ~/path or /tmp/ etc."),
|
||||
locale: z.string().optional().describe("Locale like en-US, es-ES, etc. Defaults to en-US. Use speech_list_locales to see options."),
|
||||
},
|
||||
handled(({ filePath, locale }) => speechTranscribeFile({ filePath, locale })),
|
||||
);
|
||||
|
||||
server.tool(
|
||||
"speech_quick_test",
|
||||
"End-to-end loopback test: generates speech with macOS say command, then transcribes it with SpeechAnalyzer to verify the whole pipeline works. Returns both input text and transcript for comparison.",
|
||||
{
|
||||
text: z.string().optional().describe("Text to synthesize and transcribe. Default: hello world test."),
|
||||
voice: z.string().optional().describe("macOS say voice, e.g. Alex, Samantha. Default: Alex."),
|
||||
},
|
||||
handled(({ text, voice }) => speechQuickTest({ text, voice })),
|
||||
);
|
||||
|
||||
server.tool(
|
||||
"codex_image_get_config_status",
|
||||
"Show local Codex CLI image generation configuration.",
|
||||
|
||||
Reference in New Issue
Block a user