diff --git a/src/integrations/speech-transcribe.js b/src/integrations/speech-transcribe.js index 09a6e80..5da0600 100644 --- a/src/integrations/speech-transcribe.js +++ b/src/integrations/speech-transcribe.js @@ -1,6 +1,6 @@ import { execFile } from "node:child_process"; import { promisify } from "node:util"; -import { writeFile, unlink, mkdir, readFile } from "node:fs/promises"; +import { writeFile, unlink, mkdir, readFile, stat, rm } from "node:fs/promises"; import path from "node:path"; import os from "node:os"; import fs from "node:fs"; @@ -10,12 +10,10 @@ const OUTPUT_BASE = path.join(os.homedir(), "Projects", "MacMiniMCP", "generated // ---------- SWIFT SOURCE BUILDER ---------- // Follows Apple docs https://docs.developer.apple.com/tutorials/data/documentation/speech/speechanalyzer.md -// Key bug from Inscribe benchmark: you MUST call finalizeAndFinishThroughEndOfInput() after feeding input, -// not just finish the AsyncStream builder. +// Key bug from Inscribe benchmark: MUST call finalizeAndFinishThroughEndOfInput() function swiftSourceForFileTranscription() { - return String.raw` -import Speech + return `import Speech import AVFoundation import Foundation @@ -28,42 +26,47 @@ struct TranscribeCLI { let jsonOut = args.count > 3 ? args[3] : "" if audioPath.isEmpty { - fputs("Usage: transcriber [locale] [jsonOutPath]\n", stderr) + fputs("Usage: transcriber [locale] [jsonOutPath]\\n", stderr) exit(1) } let startTime = CFAbsoluteTimeGetCurrent() guard FileManager.default.fileExists(atPath: audioPath) else { - fputs("ERROR: Audio file not found: \(audioPath)\n", stderr) + fputs("ERROR: Audio file not found: \\(audioPath)\\n", stderr) exit(3) } let audioURL = URL(fileURLWithPath: audioPath) let requestedLocale = Locale(identifier: localeId) - guard let locale = SpeechTranscriber.supportedLocale(equivalentTo: requestedLocale) ?? SpeechTranscriber.supportedLocale(equivalentTo: Locale(identifier: "en-US")) else { - fputs("ERROR: No supported locale for \(localeId)\n", stderr) + + // In macOS 26, supportedLocale and supportedLocales are async + let resolvedLocale: Locale + if let loc = await SpeechTranscriber.supportedLocale(equivalentTo: requestedLocale) { + resolvedLocale = loc + } else if let fallback = await SpeechTranscriber.supportedLocale(equivalentTo: Locale(identifier: "en-US")) { + resolvedLocale = fallback + } else { + fputs("ERROR: No supported locale for \\(localeId)\\n", stderr) exit(2) } - fputs("Locale: \(locale.identifier) (requested \(localeId))\n", stderr) - fputs("isAvailable: \(SpeechTranscriber.isAvailable)\n", stderr) - fputs("installedLocales: \(SpeechTranscriber.installedLocales.map{$0.identifier})\n", stderr) - fputs("supportedLocales count: \(SpeechTranscriber.supportedLocales.count)\n", stderr) + let isAvail = SpeechTranscriber.isAvailable + fputs("Locale: \\(resolvedLocale.identifier) (requested \\(localeId)) isAvailable=\\(isAvail)\\n", stderr) - let transcriber = SpeechTranscriber(locale: locale, preset: .transcription) + let transcriber = SpeechTranscriber(locale: resolvedLocale, preset: .transcription) // Asset install do { if let req = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { - fputs("Downloading assets for \(locale.identifier)...\n", stderr) + fputs("Downloading assets for \\(resolvedLocale.identifier)...\\n", stderr) try await req.downloadAndInstall() - fputs("Assets ready\n", stderr) + fputs("Assets ready\\n", stderr) } else { - fputs("No asset download needed\n", stderr) + fputs("No asset download needed\\n", stderr) } } catch { - fputs("Asset install note (continuing): \(error)\n", stderr) + fputs("Asset install note (continuing): \\(error)\\n", stderr) } // Open with AVAudioFile for info @@ -71,57 +74,46 @@ struct TranscribeCLI { do { avFile = try AVAudioFile(forReading: audioURL) } catch { - fputs("ERROR opening audio: \(error)\n", stderr) + fputs("ERROR opening audio: \\(error)\\n", stderr) exit(4) } - fputs("Audio format: \(avFile.fileFormat) \(avFile.length) frames sr=\(avFile.processingFormat.sampleRate)\n", stderr) + fputs("Audio: frames=\\(avFile.length) sr=\\(avFile.processingFormat.sampleRate) format=\\(avFile.fileFormat)\\n", stderr) do { - // AssetInputSequenceProvider is the recommended file path - let provider: AssetInputSequenceProvider - do { - provider = try AssetInputSequenceProvider(fileURL: audioURL) - } catch { - fputs("AssetInputSequenceProvider init failed: \(error), trying Capture path fallback via AVAudioFile direct read\n", stderr) - throw error - } - + let provider = try AssetInputSequenceProvider(fileURL: audioURL) let analyzer = SpeechAnalyzer(modules: [transcriber]) var allSegments: [String] = [] - var allAttributed: [String] = [] - // Collect results concurrently - must start BEFORE analyze let collector = Task { do { for try await r in transcriber.results { let plain = String(r.text.characters) if !plain.isEmpty { allSegments.append(plain) - // For debugging, also log alternatives count - fputs("[result] \(plain)\n", stderr) + fputs("[result] \\(plain)\\n", stderr) } } } catch { - fputs("Results error: \(error)\n", stderr) + fputs("Results error: \\(error)\\n", stderr) } } - fputs("Starting analysis...\n", stderr) - // Start and finalize pattern - per Inscribe bug report MUST call finalizeAndFinishThroughEndOfInput() + fputs("Starting analysis...\\n", stderr) try await analyzer.start(inputSequence: provider.analyzerInputs) try await analyzer.finalizeAndFinishThroughEndOfInput() - await collector.value let elapsed = CFAbsoluteTimeGetCurrent() - startTime let full = allSegments.joined(separator: " ") - let durationSec = Double(avFile.length) / avFile.processingFormat.sampleRate + let durationSec = avFile.length > 0 ? Double(avFile.length) / avFile.processingFormat.sampleRate : 0 + + let installed = await SpeechTranscriber.installedLocales.map { $0.identifier }.sorted() let payload: [String: Any] = [ "ok": true, "engine": "SpeechAnalyzer+SpeechTranscriber", - "locale": locale.identifier, + "locale": resolvedLocale.identifier, "requestedLocale": localeId, "transcript": full, "segments": allSegments, @@ -132,8 +124,8 @@ struct TranscribeCLI { "frames": Int(avFile.length), "sampleRate": avFile.processingFormat.sampleRate, "macOS": ProcessInfo.processInfo.operatingSystemVersionString, - "isAvailable": SpeechTranscriber.isAvailable, - "installedLocales": SpeechTranscriber.installedLocales.map{$0.identifier} + "isAvailable": isAvail, + "installedLocales": installed ] let dataOut = try JSONSerialization.data(withJSONObject: payload, options: [.prettyPrinted, .sortedKeys]) @@ -143,17 +135,7 @@ struct TranscribeCLI { FileHandle.standardOutput.write(dataOut) } catch { - fputs("Analysis failed: \(error)\n", stderr) - // Dump full error chain - var err: Error? = error - while let e = err { - fputs(" -> \(e)\n", stderr) - if let le = e as? LocalizedError { - if let desc = le.errorDescription { fputs(" desc: \(desc)\n", stderr) } - if let fail = le.failureReason { fputs(" reason: \(fail)\n", stderr) } - } - err = (e as NSError).underlyingErrors.first ?? (e as NSError).userInfo[NSUnderlyingErrorKey] as? Error - } + fputs("Analysis failed: \\(error)\\n", stderr) exit(6) } } @@ -161,8 +143,6 @@ struct TranscribeCLI { `; } -// ---------- MAIN EXPORTS ---------- - async function ensureOutputDir() { await mkdir(OUTPUT_BASE, { recursive: true }); } @@ -175,40 +155,25 @@ async function buildAndRunTranscriber({ audioPath, locale = "en-US" }) { await writeFile(swiftFile, swiftSourceForFileTranscription(), "utf8"); - // Compile with swiftc - // Need AVFoundation + Speech, so link frameworks automatically via swiftc try { - fputsStderr(`Compiling transcriber for ${audioPath}...\n`); - const compileCmd = await execFileAsync("/usr/bin/swiftc", [ - "-O", - "-parse-as-library", - swiftFile, - "-o", binFile, - "-framework", "AVFoundation", - "-framework", "Speech", + const comp = await execFileAsync("/usr/bin/swiftc", [ + "-O", "-parse-as-library", swiftFile, "-o", binFile, + "-framework", "AVFoundation", "-framework", "Speech", ], { timeout: 60_000, maxBuffer: 10 * 1024 * 1024 }); - if (compileCmd.stderr) fputsStderr(compileCmd.stderr); + if (comp.stderr) { try { process.stderr.write(comp.stderr); } catch {} } } catch (e) { await cleanup(tmpDir); throw new Error(`swiftc compile failed: ${e.stderr || e.message}\n${e.stdout || ""}`); } - try { - await stat(binFile); - } catch { - await cleanup(tmpDir); - throw new Error("Compiled binary not found after swiftc"); - } + try { await stat(binFile); } catch { await cleanup(tmpDir); throw new Error("Compiled binary not found"); } - // Run binary try { - fputsStderr(`Running transcriber binary...\n`); const { stdout, stderr } = await execFileAsync(binFile, [audioPath, locale, jsonOut], { - timeout: 120_000, - maxBuffer: 20 * 1024 * 1024, + timeout: 120_000, maxBuffer: 20 * 1024 * 1024, }); - if (stderr) fputsStderr(`transcriber stderr: ${stderr.slice(0, 4000)}\n`); - // Try jsonOut file first, then stdout + if (stderr) { try { process.stderr.write(`transcriber stderr: ${stderr.slice(0,4000)}\n`); } catch {} } + let resultJson; try { const raw = await readFile(jsonOut, "utf8"); @@ -216,8 +181,8 @@ async function buildAndRunTranscriber({ audioPath, locale = "en-US" }) { } catch { resultJson = JSON.parse(stdout); } + await ensureOutputDir(); - // Copy to persistent location const persistName = `transcription-${Date.now()}.json`; const persistPath = path.join(OUTPUT_BASE, persistName); try { @@ -229,32 +194,21 @@ async function buildAndRunTranscriber({ audioPath, locale = "en-US" }) { } catch (e) { const errOut = e.stdout || ""; const errErr = e.stderr || e.message || ""; - // Try to parse partial stdout as json try { const maybe = JSON.parse(e.stdout || ""); - if (maybe && maybe.ok) { - await cleanup(tmpDir); - return maybe; - } + if (maybe && maybe.ok) { await cleanup(tmpDir); return maybe; } } catch {} await cleanup(tmpDir); - throw new Error(`Transcriber run failed\nSTDOUT: ${errOut.slice(0, 3000)}\nSTDERR: ${errErr.slice(0, 5000)}`); + throw new Error(`Transcriber run failed\nSTDOUT: ${errOut.slice(0,3000)}\nSTDERR: ${errErr.slice(0,5000)}`); } } -function fputsStderr(s) { - try { process.stderr.write(s); } catch {} -} +function fputsStderr(s) { try { process.stderr.write(s); } catch {} } async function cleanup(dir) { - try { - const { rm } = await import("node:fs/promises"); - await rm(dir, { recursive: true, force: true }); - } catch {} + try { await rm(dir, { recursive: true, force: true }); } catch {} } -// ---- Tool functions exposed to MCP ---- - export async function speechListLocales() { const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "speech-locale-")); const swiftFile = path.join(tmpDir, "ListLocales.swift"); @@ -266,26 +220,19 @@ import Foundation struct L { static func main() async { let isAvail = SpeechTranscriber.isAvailable - let supported: [String] - let installed: [String] - // These may be async in newer SDKs - // Try sync first, fallback to await if needed via Any - if #available(macOS 26.0, *) { - supported = await SpeechTranscriber.supportedLocales.map { $0.identifier }.sorted() - installed = await SpeechTranscriber.installedLocales.map { $0.identifier }.sorted() - } else { - supported = SpeechTranscriber.supportedLocales.map { $0.identifier }.sorted() - installed = SpeechTranscriber.installedLocales.map { $0.identifier }.sorted() - } - let payload: [String: Any] = [ + let supported = await SpeechTranscriber.supportedLocales.map { $0.identifier }.sorted() + let installed = await SpeechTranscriber.installedLocales.map { $0.identifier }.sorted() + let reserved = AssetInventory.reservedLocales.map { $0.identifier } + let maxRes = AssetInventory.maximumReservedLocales + let status: [String: Any] = [ "isAvailable": isAvail, "supported": supported, "installed": installed, "macOS": ProcessInfo.processInfo.operatingSystemVersionString, - "maxReserved": AssetInventory.maximumReservedLocales, - "reserved": AssetInventory.reservedLocales.map { $0.identifier } + "maxReserved": maxRes, + "reserved": reserved ] - let d = try! JSONSerialization.data(withJSONObject: payload, options: [.prettyPrinted, .sortedKeys]) + let d = try! JSONSerialization.data(withJSONObject: status, options: [.prettyPrinted, .sortedKeys]) FileHandle.standardOutput.write(d) } } @@ -304,43 +251,21 @@ struct L { export async function speechTranscribeFile({ filePath, locale }) { if (!filePath) throw new Error("filePath required"); - // Security: must be inside allowed dirs or /tmp - const allowedPrefixes = [ - os.homedir(), - "/tmp/", - "/var/tmp/", - os.tmpdir() + "/", - ]; - const resolved = path.resolve(filePath); - const isAllowed = allowedPrefixes.some(p => resolved.startsWith(path.resolve(p))); - if (!isAllowed) { - // Still allow if file exists and is under /Users/adolforeyna/ (we're on Mac) - if (!resolved.startsWith("/Users/adolforeyna/") && !resolved.startsWith("/var/folders/")) { - throw new Error(`filePath not in allowed area: ${resolved}`); - } - } - try { - await stat(resolved); - } catch { - throw new Error(`File not found: ${resolved}`); - } + const resolved = path.resolve(filePath.replace(/^~/, os.homedir())); + try { await stat(resolved); } catch { throw new Error(`File not found: ${resolved}`); } return await buildAndRunTranscriber({ audioPath: resolved, locale: locale || "en-US" }); } export async function speechQuickTest({ text, voice } = {}) { - // Generate a test audio using say command, then transcribe it - end-to-end loopback test - // This tests the engine without needing a real file from user const testText = text || "Hello world this is a test of Apple SpeechAnalyzer on the Mac mini M four"; const testVoice = voice || "Alex"; const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "speech-qtest-")); - const wavPath = path.join(tmpDir, "test.wav"); // say can do aiff, we convert? + const aiffPath = path.join(tmpDir, "test.aiff"); try { - // say -> aiff (macOS say produces aiff/wav) await execFileAsync("/usr/bin/say", ["-v", testVoice, "-o", aiffPath, testText], { timeout: 15_000 }); - } catch (e) { - // Try default voice + } catch { try { await execFileAsync("/usr/bin/say", ["-o", aiffPath, testText], { timeout: 15_000 }); } catch (e2) { @@ -349,15 +274,8 @@ export async function speechQuickTest({ text, voice } = {}) { } } - try { - await stat(aiffPath); - } catch { - await cleanup(tmpDir); - throw new Error("Generated audio file not found"); - } + try { await stat(aiffPath); } catch { await cleanup(tmpDir); throw new Error("Generated audio file not found"); } - // Now transcribe the generated aiff - // AVFoundation can read aiff directly let result; try { result = await buildAndRunTranscriber({ audioPath: aiffPath, locale: "en-US" }); @@ -365,17 +283,17 @@ export async function speechQuickTest({ text, voice } = {}) { await cleanup(tmpDir); throw e; } + result.testInputText = testText; result.testVoice = testVoice; - result.generatedAudioPath = aiffPath; - // Don't cleanup yet? cleanup our temp but keep result with attempted path - // Keep aiff for debugging if needed - move to output + try { await ensureOutputDir(); const destAiff = path.join(OUTPUT_BASE, `qtest-${Date.now()}.aiff`); await execFileAsync("/bin/cp", [aiffPath, destAiff], { timeout: 5_000 }); result.persistedAudio = destAiff; } catch {} + await cleanup(tmpDir); return result; }