From 1fe9e49d8aee143292543c59ea58b9ad054d1060 Mon Sep 17 00:00:00 2001 From: aeroreyna Date: Mon, 13 Jul 2026 17:12:34 -0400 Subject: [PATCH] Fix speech-transcribe: use real macOS 26.5 API from swiftinterface - No AssetInputSequenceProvider (doesn't exist), use SpeechAnalyzer(inputAudioFile:modules:finishAfterFile:) - That initializer starts analysis automatically with finishAfterFile:true - All locale APIs are async in 26.5 - Verified from Xcode SDK swiftinterface at MacOSX26.5 --- src/integrations/speech-transcribe.js | 224 ++++++++++++-------------- 1 file changed, 101 insertions(+), 123 deletions(-) diff --git a/src/integrations/speech-transcribe.js b/src/integrations/speech-transcribe.js index 68e1bd0..2e49715 100644 --- a/src/integrations/speech-transcribe.js +++ b/src/integrations/speech-transcribe.js @@ -1,18 +1,17 @@ import { execFile } from "node:child_process"; import { promisify } from "node:util"; -import { writeFile, unlink, mkdir, readFile, stat, rm } from "node:fs/promises"; +import { writeFile, readFile, stat, rm, mkdir } from "node:fs/promises"; import path from "node:path"; import os from "node:os"; -import fs from "node:fs"; const execFileAsync = promisify(execFile); const OUTPUT_BASE = path.join(os.homedir(), "Projects", "MacMiniMCP", "generated-transcriptions"); -// ---------- SWIFT SOURCE BUILDER ---------- -// Follows Apple docs https://docs.developer.apple.com/tutorials/data/documentation/speech/speechanalyzer.md -// Key bug from Inscribe benchmark: MUST call finalizeAndFinishThroughEndOfInput() +// Swift source for file transcription using the REAL macOS 26.5 API +// From swiftinterface: SpeechAnalyzer has init(inputAudioFile:modules:) and analyzeSequence(from:) +// No AssetInputSequenceProvider - it's direct AVAudioFile -function swiftSourceForFileTranscription() { +function swiftSourceFileTranscriber() { return `import Speech import AVFoundation import Foundation @@ -26,37 +25,36 @@ struct TranscribeCLI { let jsonOut = args.count > 3 ? args[3] : "" if audioPath.isEmpty { - fputs("Usage: transcriber [locale] [jsonOutPath]\\n", stderr) + fputs("Usage: transcriber [locale] [jsonOut]\\n", stderr) exit(1) } let startTime = CFAbsoluteTimeGetCurrent() guard FileManager.default.fileExists(atPath: audioPath) else { - fputs("ERROR: Audio file not found: \\(audioPath)\\n", stderr) + fputs("ERROR: File not found \\(audioPath)\\n", stderr) exit(3) } let audioURL = URL(fileURLWithPath: audioPath) let requestedLocale = Locale(identifier: localeId) - // In macOS 26, supportedLocale and supportedLocales are async let resolvedLocale: Locale - if let loc = await SpeechTranscriber.supportedLocale(equivalentTo: requestedLocale) { - resolvedLocale = loc - } else if let fallback = await SpeechTranscriber.supportedLocale(equivalentTo: Locale(identifier: "en-US")) { - resolvedLocale = fallback + if let l = await SpeechTranscriber.supportedLocale(equivalentTo: requestedLocale) { + resolvedLocale = l + } else if let fb = await SpeechTranscriber.supportedLocale(equivalentTo: Locale(identifier: "en-US")) { + resolvedLocale = fb } else { fputs("ERROR: No supported locale for \\(localeId)\\n", stderr) exit(2) } let isAvail = SpeechTranscriber.isAvailable - fputs("Locale: \\(resolvedLocale.identifier) (requested \\(localeId)) isAvailable=\\(isAvail)\\n", stderr) + fputs("Locale \\(resolvedLocale.identifier) isAvailable=\\(isAvail)\\n", stderr) let transcriber = SpeechTranscriber(locale: resolvedLocale, preset: .transcription) - // Asset install + // Assets do { if let req = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { fputs("Downloading assets for \\(resolvedLocale.identifier)...\\n", stderr) @@ -66,25 +64,25 @@ struct TranscribeCLI { fputs("No asset download needed\\n", stderr) } } catch { - fputs("Asset install note (continuing): \\(error)\\n", stderr) + fputs("Asset note (continuing): \\(error)\\n", stderr) } - // Open with AVAudioFile for info let avFile: AVAudioFile do { avFile = try AVAudioFile(forReading: audioURL) + fputs("File: frames=\\(avFile.length) sr=\\(avFile.processingFormat.sampleRate) fmt=\\(avFile.fileFormat)\\n", stderr) } catch { - fputs("ERROR opening audio: \\(error)\\n", stderr) + fputs("ERROR opening file: \\(error)\\n", stderr) exit(4) } - fputs("Audio: frames=\\(avFile.length) sr=\\(avFile.processingFormat.sampleRate) format=\\(avFile.fileFormat)\\n", stderr) + + var allSegments: [String] = [] do { - let provider = try AssetInputSequenceProvider(fileURL: audioURL) - let analyzer = SpeechAnalyzer(modules: [transcriber]) - - var allSegments: [String] = [] + // Use the new convenience: analyzer from audio file + let analyzer = try await SpeechAnalyzer(inputAudioFile: avFile, modules: [transcriber], finishAfterFile: true) + // Collect results - must be concurrent let collector = Task { do { for try await r in transcriber.results { @@ -99,20 +97,22 @@ struct TranscribeCLI { } } - fputs("Starting analysis...\\n", stderr) - try await analyzer.start(inputSequence: provider.analyzerInputs) - try await analyzer.finalizeAndFinishThroughEndOfInput() + // analysis was already started by init with finishAfterFile=true, just wait + // Alternatively use analyzeSequence(from:) pattern: + // But since init already starts with file, we just wait for collector + // The analyzer will finish automatically due to finishAfterFile:true + + // Wait for collector - it finishes when analyzer finishes file and finalizes await collector.value let elapsed = CFAbsoluteTimeGetCurrent() - startTime let full = allSegments.joined(separator: " ") let durationSec = avFile.length > 0 ? Double(avFile.length) / avFile.processingFormat.sampleRate : 0 - let installed = await SpeechTranscriber.installedLocales.map { $0.identifier }.sorted() let payload: [String: Any] = [ "ok": true, - "engine": "SpeechAnalyzer+SpeechTranscriber", + "engine": "SpeechAnalyzer+SpeechTranscriber/macOS26.5", "locale": resolvedLocale.identifier, "requestedLocale": localeId, "transcript": full, @@ -121,6 +121,7 @@ struct TranscribeCLI { "audioPath": audioPath, "durationSeconds": durationSec, "realtimeFactor": durationSec > 0 ? elapsed / durationSec : 0, + "rtfx": durationSec > 0 ? durationSec / elapsed : 0, "frames": Int(avFile.length), "sampleRate": avFile.processingFormat.sampleRate, "macOS": ProcessInfo.processInfo.operatingSystemVersionString, @@ -136,6 +137,12 @@ struct TranscribeCLI { } catch { fputs("Analysis failed: \\(error)\\n", stderr) + // Dump chain + var cur: Error? = error + while let e = cur { + fputs(" -> \\(e)\\n", stderr) + cur = (e as NSError).userInfo[NSUnderlyingErrorKey] as? Error + } exit(6) } } @@ -147,75 +154,59 @@ async function ensureOutputDir() { await mkdir(OUTPUT_BASE, { recursive: true }); } -async function buildAndRunTranscriber({ audioPath, locale = "en-US" }) { - const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "speech-transcribe-")); - const swiftFile = path.join(tmpDir, "TranscriberMain.swift"); - const binFile = path.join(tmpDir, "transcriber_bin"); - const jsonOut = path.join(tmpDir, "result.json"); - - await writeFile(swiftFile, swiftSourceForFileTranscription(), "utf8"); - - try { - const comp = await execFileAsync("/usr/bin/swiftc", [ - "-O", "-parse-as-library", swiftFile, "-o", binFile, - "-framework", "AVFoundation", "-framework", "Speech", - ], { timeout: 60_000, maxBuffer: 10 * 1024 * 1024 }); - if (comp.stderr) { try { process.stderr.write(comp.stderr); } catch {} } - } catch (e) { - await cleanup(tmpDir); - throw new Error(`swiftc compile failed: ${e.stderr || e.message}\n${e.stdout || ""}`); - } - - try { await stat(binFile); } catch { await cleanup(tmpDir); throw new Error("Compiled binary not found"); } - - try { - const { stdout, stderr } = await execFileAsync(binFile, [audioPath, locale, jsonOut], { - timeout: 120_000, maxBuffer: 20 * 1024 * 1024, - }); - if (stderr) { try { process.stderr.write(`transcriber stderr: ${stderr.slice(0,4000)}\n`); } catch {} } - - let resultJson; - try { - const raw = await readFile(jsonOut, "utf8"); - resultJson = JSON.parse(raw); - } catch { - resultJson = JSON.parse(stdout); - } - - await ensureOutputDir(); - const persistName = `transcription-${Date.now()}.json`; - const persistPath = path.join(OUTPUT_BASE, persistName); - try { - await writeFile(persistPath, JSON.stringify(resultJson, null, 2), "utf8"); - resultJson.persistedTo = persistPath; - } catch {} - await cleanup(tmpDir); - return resultJson; - } catch (e) { - const errOut = e.stdout || ""; - const errErr = e.stderr || e.message || ""; - try { - const maybe = JSON.parse(e.stdout || ""); - if (maybe && maybe.ok) { await cleanup(tmpDir); return maybe; } - } catch {} - await cleanup(tmpDir); - throw new Error(`Transcriber run failed\nSTDOUT: ${errOut.slice(0,3000)}\nSTDERR: ${errErr.slice(0,5000)}`); - } -} - -function fputsStderr(s) { try { process.stderr.write(s); } catch {} } - async function cleanup(dir) { try { await rm(dir, { recursive: true, force: true }); } catch {} } +async function buildAndRun({ audioPath, locale = "en-US" }) { + const tmpDir = await (await import("node:fs/promises")).mkdtemp.call(null, path.join(os.tmpdir(), "speech-t-")); + // compatible mkdtemp + const { mkdtemp } = await import("node:fs/promises"); + const dir = await mkdtemp(path.join(os.tmpdir(), "speech-t-")); + const swiftFile = path.join(dir, "Main.swift"); + const binFile = path.join(dir, "transcriber"); + const jsonOut = path.join(dir, "result.json"); + + await writeFile(swiftFile, swiftSourceFileTranscriber(), "utf8"); + + try { + await execFileAsync("/usr/bin/swiftc", ["-O", "-parse-as-library", swiftFile, "-o", binFile, "-framework", "AVFoundation", "-framework", "Speech"], { timeout: 60_000, maxBuffer: 20*1024*1024 }); + } catch (e) { + await cleanup(dir); + throw new Error(`swiftc compile failed:\n${e.stderr || e.message}\n${e.stdout||""}`); + } + + try { await stat(binFile); } catch { await cleanup(dir); throw new Error("Binary not built"); } + + try { + const { stdout, stderr } = await execFileAsync(binFile, [audioPath, locale, jsonOut], { timeout: 120_000, maxBuffer: 20*1024*1024 }); + if (stderr) { try { process.stderr.write(stderr.slice(0,4000)); } catch {} } + let result; + try { result = JSON.parse(await readFile(jsonOut, "utf8")); } catch { result = JSON.parse(stdout); } + await ensureOutputDir(); + const persistPath = path.join(OUTPUT_BASE, `transcription-${Date.now()}.json`); + try { await writeFile(persistPath, JSON.stringify(result, null, 2), "utf8"); result.persistedTo = persistPath; } catch {} + await cleanup(dir); + return result; + } catch (e) { + const out = e.stdout || ""; + const er = e.stderr || e.message || ""; + try { + const partial = JSON.parse(out); + if (partial && partial.ok) { await cleanup(dir); return partial; } + } catch {} + await cleanup(dir); + throw new Error(`Transcribe failed\nSTDOUT:${out.slice(0,4000)}\nSTDERR:${er.slice(0,6000)}`); + } +} + export async function speechListLocales() { - const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "speech-locale-")); - const swiftFile = path.join(tmpDir, "ListLocales.swift"); - const binFile = path.join(tmpDir, "list_bin"); + const { mkdtemp } = await import("node:fs/promises"); + const dir = await mkdtemp(path.join(os.tmpdir(), "speech-loc-")); + const swiftFile = path.join(dir, "List.swift"); + const binFile = path.join(dir, "list"); const swiftSrc = `import Speech import Foundation - @main struct L { static func main() async { @@ -223,16 +214,15 @@ struct L { let supported = await SpeechTranscriber.supportedLocales.map { $0.identifier }.sorted() let installed = await SpeechTranscriber.installedLocales.map { $0.identifier }.sorted() let reserved = await AssetInventory.reservedLocales.map { $0.identifier } - let maxRes = AssetInventory.maximumReservedLocales - let status: [String: Any] = [ + let payload: [String: Any] = [ "isAvailable": isAvail, "supported": supported, "installed": installed, "macOS": ProcessInfo.processInfo.operatingSystemVersionString, - "maxReserved": maxRes, + "maxReserved": AssetInventory.maximumReservedLocales, "reserved": reserved ] - let d = try! JSONSerialization.data(withJSONObject: status, options: [.prettyPrinted, .sortedKeys]) + let d = try! JSONSerialization.data(withJSONObject: payload, options: [.prettyPrinted, .sortedKeys]) FileHandle.standardOutput.write(d) } } @@ -241,59 +231,47 @@ struct L { try { await execFileAsync("/usr/bin/swiftc", ["-O", "-parse-as-library", swiftFile, "-o", binFile, "-framework", "Speech"], { timeout: 30_000 }); const { stdout } = await execFileAsync(binFile, [], { timeout: 15_000 }); - await cleanup(tmpDir); + await cleanup(dir); return JSON.parse(stdout); } catch (e) { - await cleanup(tmpDir); - throw new Error(`List locales failed: ${e.stderr || e.message}\n${e.stdout || ""}`); + await cleanup(dir); + throw new Error(`List locales failed: ${e.stderr||e.message}\n${e.stdout||""}`); } } export async function speechTranscribeFile({ filePath, locale }) { if (!filePath) throw new Error("filePath required"); - const resolved = path.resolve(filePath.replace(/^~/, os.homedir())); + const resolved = path.resolve(filePath.replace(/^~(?=$|\/)/, os.homedir())); try { await stat(resolved); } catch { throw new Error(`File not found: ${resolved}`); } - return await buildAndRunTranscriber({ audioPath: resolved, locale: locale || "en-US" }); + return await buildAndRun({ audioPath: resolved, locale: locale || "en-US" }); } export async function speechQuickTest({ text, voice } = {}) { const testText = text || "Hello world this is a test of Apple SpeechAnalyzer on the Mac mini M four"; const testVoice = voice || "Alex"; - const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "speech-qtest-")); - - const aiffPath = path.join(tmpDir, "test.aiff"); - + const { mkdtemp } = await import("node:fs/promises"); + const tmp = await mkdtemp(path.join(os.tmpdir(), "speech-qtest-")); + const aiffPath = path.join(tmp, "test.aiff"); try { await execFileAsync("/usr/bin/say", ["-v", testVoice, "-o", aiffPath, testText], { timeout: 15_000 }); } catch { - try { - await execFileAsync("/usr/bin/say", ["-o", aiffPath, testText], { timeout: 15_000 }); - } catch (e2) { - await cleanup(tmpDir); - throw new Error(`say command failed: ${e2.stderr || e2.message}`); - } + try { await execFileAsync("/usr/bin/say", ["-o", aiffPath, testText], { timeout: 15_000 }); } + catch (e2) { await cleanup(tmp); throw new Error(`say failed: ${e2.stderr||e2.message}`); } } - - try { await stat(aiffPath); } catch { await cleanup(tmpDir); throw new Error("Generated audio file not found"); } + try { await stat(aiffPath); } catch { await cleanup(tmp); throw new Error("Generated audio not found"); } let result; - try { - result = await buildAndRunTranscriber({ audioPath: aiffPath, locale: "en-US" }); - } catch (e) { - await cleanup(tmpDir); - throw e; - } + try { result = await buildAndRun({ audioPath: aiffPath, locale: "en-US" }); } + catch (e) { await cleanup(tmp); throw e; } result.testInputText = testText; result.testVoice = testVoice; - try { await ensureOutputDir(); - const destAiff = path.join(OUTPUT_BASE, `qtest-${Date.now()}.aiff`); - await execFileAsync("/bin/cp", [aiffPath, destAiff], { timeout: 5_000 }); - result.persistedAudio = destAiff; + const dest = path.join(OUTPUT_BASE, `qtest-${Date.now()}.aiff`); + await execFileAsync("/bin/cp", [aiffPath, dest], { timeout: 5_000 }); + result.persistedAudio = dest; } catch {} - - await cleanup(tmpDir); + await cleanup(tmp); return result; }