From 6645ef3664253e4c85f08c5c9b2b471a4d736e25 Mon Sep 17 00:00:00 2001 From: aeroreyna Date: Mon, 13 Jul 2026 17:07:45 -0400 Subject: [PATCH] Add speech transcription tools: speech_list_locales, speech_transcribe_file, speech_quick_test Implements Apple SpeechAnalyzer + SpeechTranscriber (macOS 26+) live transcription per official docs, with Inscribe bugfix (finalizeAndFinishThroughEndOfInput) - speech_list_locales: isAvailable, supported/installed locales, reserved counts - speech_transcribe_file: transcribe any AVFoundation-readable file, returns transcript + segments + elapsed + realtimeFactor + engine metadata - speech_quick_test: say + transcribe loopback to verify pipeline Swift binary compiled on-demand via swiftc -O with Speech + AVFoundation Follows docs: AssetInventory downloadAndInstall, AssetInputSequenceProvider Mac mini is 26.5.2 Mac16,10 M4 so engine verified available --- src/integrations/speech-transcribe.js | 370 ++++++++++++++++++++++++++ src/server.js | 32 +++ 2 files changed, 402 insertions(+) create mode 100644 src/integrations/speech-transcribe.js diff --git a/src/integrations/speech-transcribe.js b/src/integrations/speech-transcribe.js new file mode 100644 index 0000000..ba09a0d --- /dev/null +++ b/src/integrations/speech-transcribe.js @@ -0,0 +1,370 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { writeFile, unlink, mkdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import fs from "node:fs"; + +const execFileAsync = promisify(execFile); +const OUTPUT_BASE = path.join(os.homedir(), "Projects", "MacMiniMCP", "generated-transcriptions"); + +// ---------- SWIFT SOURCE BUILDER ---------- +// Follows Apple docs https://docs.developer.apple.com/tutorials/data/documentation/speech/speechanalyzer.md +// Key bug from Inscribe benchmark: you MUST call finalizeAndFinishThroughEndOfInput() after feeding input, +// not just finish the AsyncStream builder. + +function swiftSourceForFileTranscription() { + return String.raw` +import Speech +import AVFoundation +import Foundation + +@main +struct TranscribeCLI { + static func main() async { + let args = CommandLine.arguments + let audioPath = args.count > 1 ? args[1] : "" + let localeId = args.count > 2 ? args[2] : "en-US" + let jsonOut = args.count > 3 ? args[3] : "" + + if audioPath.isEmpty { + fputs("Usage: transcriber [locale] [jsonOutPath]\n", stderr) + exit(1) + } + + let startTime = CFAbsoluteTimeGetCurrent() + + guard FileManager.default.fileExists(atPath: audioPath) else { + fputs("ERROR: Audio file not found: \(audioPath)\n", stderr) + exit(3) + } + + let audioURL = URL(fileURLWithPath: audioPath) + let requestedLocale = Locale(identifier: localeId) + guard let locale = SpeechTranscriber.supportedLocale(equivalentTo: requestedLocale) ?? SpeechTranscriber.supportedLocale(equivalentTo: Locale(identifier: "en-US")) else { + fputs("ERROR: No supported locale for \(localeId)\n", stderr) + exit(2) + } + + fputs("Locale: \(locale.identifier) (requested \(localeId))\n", stderr) + fputs("isAvailable: \(SpeechTranscriber.isAvailable)\n", stderr) + fputs("installedLocales: \(SpeechTranscriber.installedLocales.map{$0.identifier})\n", stderr) + fputs("supportedLocales count: \(SpeechTranscriber.supportedLocales.count)\n", stderr) + + let transcriber = SpeechTranscriber(locale: locale, preset: .transcription) + + // Asset install + do { + if let req = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { + fputs("Downloading assets for \(locale.identifier)...\n", stderr) + try await req.downloadAndInstall() + fputs("Assets ready\n", stderr) + } else { + fputs("No asset download needed\n", stderr) + } + } catch { + fputs("Asset install note (continuing): \(error)\n", stderr) + } + + // Open with AVAudioFile for info + let avFile: AVAudioFile + do { + avFile = try AVAudioFile(forReading: audioURL) + } catch { + fputs("ERROR opening audio: \(error)\n", stderr) + exit(4) + } + fputs("Audio format: \(avFile.fileFormat) \(avFile.length) frames sr=\(avFile.processingFormat.sampleRate)\n", stderr) + + do { + // AssetInputSequenceProvider is the recommended file path + let provider: AssetInputSequenceProvider + do { + provider = try AssetInputSequenceProvider(fileURL: audioURL) + } catch { + fputs("AssetInputSequenceProvider init failed: \(error), trying Capture path fallback via AVAudioFile direct read\n", stderr) + throw error + } + + let analyzer = SpeechAnalyzer(modules: [transcriber]) + + var allSegments: [String] = [] + var allAttributed: [String] = [] + + // Collect results concurrently - must start BEFORE analyze + let collector = Task { + do { + for try await r in transcriber.results { + let plain = String(r.text.characters) + if !plain.isEmpty { + allSegments.append(plain) + // For debugging, also log alternatives count + fputs("[result] \(plain)\n", stderr) + } + } + } catch { + fputs("Results error: \(error)\n", stderr) + } + } + + fputs("Starting analysis...\n", stderr) + // Start and finalize pattern - per Inscribe bug report MUST call finalizeAndFinishThroughEndOfInput() + try await analyzer.start(inputSequence: provider.analyzerInputs) + try await analyzer.finalizeAndFinishThroughEndOfInput() + + await collector.value + + let elapsed = CFAbsoluteTimeGetCurrent() - startTime + let full = allSegments.joined(separator: " ") + let durationSec = Double(avFile.length) / avFile.processingFormat.sampleRate + + let payload: [String: Any] = [ + "ok": true, + "engine": "SpeechAnalyzer+SpeechTranscriber", + "locale": locale.identifier, + "requestedLocale": localeId, + "transcript": full, + "segments": allSegments, + "elapsedSeconds": elapsed, + "audioPath": audioPath, + "durationSeconds": durationSec, + "realtimeFactor": durationSec > 0 ? elapsed / durationSec : 0, + "frames": Int(avFile.length), + "sampleRate": avFile.processingFormat.sampleRate, + "macOS": ProcessInfo.processInfo.operatingSystemVersionString, + "isAvailable": SpeechTranscriber.isAvailable, + "installedLocales": SpeechTranscriber.installedLocales.map{$0.identifier} + ] + + let dataOut = try JSONSerialization.data(withJSONObject: payload, options: [.prettyPrinted, .sortedKeys]) + if !jsonOut.isEmpty { + try dataOut.write(to: URL(fileURLWithPath: jsonOut)) + } + FileHandle.standardOutput.write(dataOut) + + } catch { + fputs("Analysis failed: \(error)\n", stderr) + // Dump full error chain + var err: Error? = error + while let e = err { + fputs(" -> \(e)\n", stderr) + if let le = e as? LocalizedError { + if let desc = le.errorDescription { fputs(" desc: \(desc)\n", stderr) } + if let fail = le.failureReason { fputs(" reason: \(fail)\n", stderr) } + } + err = (e as NSError).underlyingErrors.first ?? (e as NSError).userInfo[NSUnderlyingErrorKey] as? Error + } + exit(6) + } + } +} +`; +} + +// ---------- MAIN EXPORTS ---------- + +async function ensureOutputDir() { + await mkdir(OUTPUT_BASE, { recursive: true }); +} + +async function buildAndRunTranscriber({ audioPath, locale = "en-US" }) { + const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "speech-transcribe-")); + const swiftFile = path.join(tmpDir, "TranscriberMain.swift"); + const binFile = path.join(tmpDir, "transcriber_bin"); + const jsonOut = path.join(tmpDir, "result.json"); + + await writeFile(swiftFile, swiftSourceForFileTranscription(), "utf8"); + + // Compile with swiftc + // Need AVFoundation + Speech, so link frameworks automatically via swiftc + try { + fputsStderr(`Compiling transcriber for ${audioPath}...\n`); + const compileCmd = await execFileAsync("/usr/bin/swiftc", [ + "-O", + swiftFile, + "-o", binFile, + "-framework", "AVFoundation", + "-framework", "Speech", + ], { timeout: 60_000, maxBuffer: 10 * 1024 * 1024 }); + if (compileCmd.stderr) fputsStderr(compileCmd.stderr); + } catch (e) { + await cleanup(tmpDir); + throw new Error(`swiftc compile failed: ${e.stderr || e.message}\n${e.stdout || ""}`); + } + + try { + await stat(binFile); + } catch { + await cleanup(tmpDir); + throw new Error("Compiled binary not found after swiftc"); + } + + // Run binary + try { + fputsStderr(`Running transcriber binary...\n`); + const { stdout, stderr } = await execFileAsync(binFile, [audioPath, locale, jsonOut], { + timeout: 120_000, + maxBuffer: 20 * 1024 * 1024, + }); + if (stderr) fputsStderr(`transcriber stderr: ${stderr.slice(0, 4000)}\n`); + // Try jsonOut file first, then stdout + let resultJson; + try { + const raw = await readFile(jsonOut, "utf8"); + resultJson = JSON.parse(raw); + } catch { + resultJson = JSON.parse(stdout); + } + await ensureOutputDir(); + // Copy to persistent location + const persistName = `transcription-${Date.now()}.json`; + const persistPath = path.join(OUTPUT_BASE, persistName); + try { + await writeFile(persistPath, JSON.stringify(resultJson, null, 2), "utf8"); + resultJson.persistedTo = persistPath; + } catch {} + await cleanup(tmpDir); + return resultJson; + } catch (e) { + const errOut = e.stdout || ""; + const errErr = e.stderr || e.message || ""; + // Try to parse partial stdout as json + try { + const maybe = JSON.parse(e.stdout || ""); + if (maybe && maybe.ok) { + await cleanup(tmpDir); + return maybe; + } + } catch {} + await cleanup(tmpDir); + throw new Error(`Transcriber run failed\nSTDOUT: ${errOut.slice(0, 3000)}\nSTDERR: ${errErr.slice(0, 5000)}`); + } +} + +function fputsStderr(s) { + try { process.stderr.write(s); } catch {} +} + +async function cleanup(dir) { + try { + const { rm } = await import("node:fs/promises"); + await rm(dir, { recursive: true, force: true }); + } catch {} +} + +// ---- Tool functions exposed to MCP ---- + +export async function speechListLocales() { + const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "speech-locale-")); + const swiftFile = path.join(tmpDir, "ListLocales.swift"); + const binFile = path.join(tmpDir, "list_bin"); + const swiftSrc = String.raw` +import Speech +import Foundation + +@main +struct L { + static func main() async { + // Ensure we trigger asset inventory awareness + let payload: [String: Any] = [ + "isAvailable": SpeechTranscriber.isAvailable, + "supported": SpeechTranscriber.supportedLocales.map{$0.identifier}.sorted(), + "installed": SpeechTranscriber.installedLocales.map{$0.identifier}.sorted(), + "macOS": ProcessInfo.processInfo.operatingSystemVersionString, + "maxReserved": AssetInventory.maximumReservedLocales, + "reserved": AssetInventory.reservedLocales.map{$0.identifier} + ] + let d = try! JSONSerialization.data(withJSONObject: payload, options: [.prettyPrinted, .sortedKeys]) + FileHandle.standardOutput.write(d) + } +} +`; + await writeFile(swiftFile, swiftSrc, "utf8"); + try { + await execFileAsync("/usr/bin/swiftc", ["-O", swiftFile, "-o", binFile, "-framework", "Speech"], { timeout: 30_000 }); + const { stdout } = await execFileAsync(binFile, [], { timeout: 15_000 }); + await cleanup(tmpDir); + return JSON.parse(stdout); + } catch (e) { + await cleanup(tmpDir); + throw new Error(`List locales failed: ${e.stderr || e.message}`); + } +} + +export async function speechTranscribeFile({ filePath, locale }) { + if (!filePath) throw new Error("filePath required"); + // Security: must be inside allowed dirs or /tmp + const allowedPrefixes = [ + os.homedir(), + "/tmp/", + "/var/tmp/", + os.tmpdir() + "/", + ]; + const resolved = path.resolve(filePath); + const isAllowed = allowedPrefixes.some(p => resolved.startsWith(path.resolve(p))); + if (!isAllowed) { + // Still allow if file exists and is under /Users/adolforeyna/ (we're on Mac) + if (!resolved.startsWith("/Users/adolforeyna/") && !resolved.startsWith("/var/folders/")) { + throw new Error(`filePath not in allowed area: ${resolved}`); + } + } + try { + await stat(resolved); + } catch { + throw new Error(`File not found: ${resolved}`); + } + return await buildAndRunTranscriber({ audioPath: resolved, locale: locale || "en-US" }); +} + +export async function speechQuickTest({ text, voice } = {}) { + // Generate a test audio using say command, then transcribe it - end-to-end loopback test + // This tests the engine without needing a real file from user + const testText = text || "Hello world this is a test of Apple SpeechAnalyzer on the Mac mini M four"; + const testVoice = voice || "Alex"; + const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "speech-qtest-")); + const wavPath = path.join(tmpDir, "test.wav"); // say can do aiff, we convert? + const aiffPath = path.join(tmpDir, "test.aiff"); + + try { + // say -> aiff (macOS say produces aiff/wav) + await execFileAsync("/usr/bin/say", ["-v", testVoice, "-o", aiffPath, testText], { timeout: 15_000 }); + } catch (e) { + // Try default voice + try { + await execFileAsync("/usr/bin/say", ["-o", aiffPath, testText], { timeout: 15_000 }); + } catch (e2) { + await cleanup(tmpDir); + throw new Error(`say command failed: ${e2.stderr || e2.message}`); + } + } + + try { + await stat(aiffPath); + } catch { + await cleanup(tmpDir); + throw new Error("Generated audio file not found"); + } + + // Now transcribe the generated aiff + // AVFoundation can read aiff directly + let result; + try { + result = await buildAndRunTranscriber({ audioPath: aiffPath, locale: "en-US" }); + } catch (e) { + await cleanup(tmpDir); + throw e; + } + result.testInputText = testText; + result.testVoice = testVoice; + result.generatedAudioPath = aiffPath; + // Don't cleanup yet? cleanup our temp but keep result with attempted path + // Keep aiff for debugging if needed - move to output + try { + await ensureOutputDir(); + const destAiff = path.join(OUTPUT_BASE, `qtest-${Date.now()}.aiff`); + await execFileAsync("/bin/cp", [aiffPath, destAiff], { timeout: 5_000 }); + result.persistedAudio = destAiff; + } catch {} + await cleanup(tmpDir); + return result; +} diff --git a/src/server.js b/src/server.js index 2556e31..2591d7e 100644 --- a/src/server.js +++ b/src/server.js @@ -31,6 +31,11 @@ import { listReminders, } from "./integrations/reminders.js"; import { getSystemInfo, getSpeechApiStatus } from "./integrations/system.js"; +import { + speechTranscribeFile, + speechListLocales, + speechQuickTest, +} from "./integrations/speech-transcribe.js"; const jsonResult = (value) => ({ content: [{ type: "text", text: JSON.stringify(value, null, 2) }], @@ -253,6 +258,33 @@ export function createMacMiniMcpServer() { handled(() => getSpeechApiStatus()), ); + server.tool( + "speech_list_locales", + "List SpeechTranscriber locales: isAvailable, supported, installed, reserved, maxReserved. Use to check if your language model is ready before transcribing.", + {}, + handled(() => speechListLocales()), + ); + + server.tool( + "speech_transcribe_file", + "Transcribe an audio file using Apple's new SpeechAnalyzer + SpeechTranscriber (macOS 26+). Fastest and most accurate English engine on-device per Inscribe benchmark (2.12% WER). Supports m4a, wav, aiff, mp3, etc via AVFoundation. Returns transcript, segments, timing, realtime factor.", + { + filePath: z.string().min(1).describe("Absolute path to audio file on the Mac. Can be ~/path or /tmp/ etc."), + locale: z.string().optional().describe("Locale like en-US, es-ES, etc. Defaults to en-US. Use speech_list_locales to see options."), + }, + handled(({ filePath, locale }) => speechTranscribeFile({ filePath, locale })), + ); + + server.tool( + "speech_quick_test", + "End-to-end loopback test: generates speech with macOS say command, then transcribes it with SpeechAnalyzer to verify the whole pipeline works. Returns both input text and transcript for comparison.", + { + text: z.string().optional().describe("Text to synthesize and transcribe. Default: hello world test."), + voice: z.string().optional().describe("macOS say voice, e.g. Alex, Samantha. Default: Alex."), + }, + handled(({ text, voice }) => speechQuickTest({ text, voice })), + ); + server.tool( "codex_image_get_config_status", "Show local Codex CLI image generation configuration.",