Preserve macOS app permissions via dynamic launcher, .env workspace path, and session/audio tools
This commit is contained in:
Adolfo Reyna
+302
-36
@@ -8,6 +8,8 @@ identical to the upstream transport.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
import sys
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import sounddevice as sd
|
||||
@@ -18,6 +20,7 @@ from pipecat.processors.frame_processor import FrameProcessor
|
||||
from pipecat.transports.base_input import BaseInputTransport
|
||||
from pipecat.transports.base_output import BaseOutputTransport
|
||||
from pipecat.transports.base_transport import BaseTransport, TransportParams
|
||||
from audio_device_monitor import AudioDeviceSnapshot, create_macos_audio_monitor
|
||||
|
||||
|
||||
class SoundDeviceTransportParams(TransportParams):
|
||||
@@ -36,11 +39,14 @@ class SoundDeviceInputTransport(BaseInputTransport):
|
||||
"""Captures microphone audio and pushes it into the pipeline."""
|
||||
|
||||
_params: SoundDeviceTransportParams
|
||||
_transport: "SoundDeviceTransport"
|
||||
|
||||
def __init__(self, params: SoundDeviceTransportParams):
|
||||
super().__init__(params)
|
||||
self._in_stream: sd.RawInputStream | None = None
|
||||
self._sample_rate = 0
|
||||
self._stream_generation = 0
|
||||
self._stream_lock = asyncio.Lock()
|
||||
|
||||
async def start(self, frame: StartFrame):
|
||||
await super().start(frame)
|
||||
@@ -51,29 +57,74 @@ class SoundDeviceInputTransport(BaseInputTransport):
|
||||
self._sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate
|
||||
blocksize = int(self._sample_rate / 100) * 2 # 20ms
|
||||
|
||||
self._in_stream = sd.RawInputStream(
|
||||
samplerate=self._sample_rate,
|
||||
blocksize=blocksize,
|
||||
device=self._params.input_device,
|
||||
channels=self._params.audio_in_channels,
|
||||
dtype="int16",
|
||||
callback=self._audio_in_callback,
|
||||
)
|
||||
self._in_stream.start()
|
||||
|
||||
device_name = sd.query_devices(self._in_stream.device, "input")["name"]
|
||||
logger.info(f"Microphone: {device_name} @ {self._sample_rate} Hz")
|
||||
await self._open_stream()
|
||||
|
||||
await self.set_transport_ready(frame)
|
||||
|
||||
if hasattr(self, "_transport"):
|
||||
await self._transport.start_device_monitor(self)
|
||||
|
||||
async def reopen(self, *, device=None):
|
||||
"""Recreate an unset-device stream so PortAudio resolves the new default."""
|
||||
if not self._in_stream:
|
||||
return
|
||||
async with self._stream_lock:
|
||||
old_stream = self._in_stream
|
||||
old_device = old_stream.device
|
||||
# Fail before disrupting an otherwise healthy conversation whenever
|
||||
# PortAudio can already tell us that the new default is unavailable.
|
||||
sd.check_input_settings(
|
||||
device=self._params.input_device if device is None else device, samplerate=self._sample_rate,
|
||||
channels=self._params.audio_in_channels, dtype="int16",
|
||||
)
|
||||
self._stream_generation += 1 # makes callbacks from the old stream inert
|
||||
try:
|
||||
old_stream.stop()
|
||||
old_stream.close()
|
||||
self._in_stream = None
|
||||
await self._open_stream(device=device)
|
||||
except Exception as exc:
|
||||
logger.warning(f"Audio input route change failed; restoring prior stream: {type(exc).__name__}")
|
||||
try:
|
||||
self._in_stream = None
|
||||
await self._open_stream(device=old_device)
|
||||
except Exception as restore_exc:
|
||||
logger.error(f"Audio input fallback unavailable: {type(restore_exc).__name__}")
|
||||
raise
|
||||
|
||||
async def _open_stream(self, *, device=None):
|
||||
blocksize = int(self._sample_rate / 100) * 2
|
||||
self._stream_generation += 1
|
||||
generation = self._stream_generation
|
||||
stream = sd.RawInputStream(
|
||||
samplerate=self._sample_rate, blocksize=blocksize,
|
||||
device=self._params.input_device if device is None else device,
|
||||
channels=self._params.audio_in_channels, dtype="int16",
|
||||
callback=lambda *args: self._audio_in_callback(generation, *args),
|
||||
)
|
||||
try:
|
||||
stream.start()
|
||||
except Exception:
|
||||
stream.close()
|
||||
raise
|
||||
self._in_stream = stream
|
||||
device_name = sd.query_devices(self._in_stream.device, "input")["name"]
|
||||
logger.info(f"Microphone: {device_name} @ {self._sample_rate} Hz")
|
||||
|
||||
async def cleanup(self):
|
||||
await super().cleanup()
|
||||
if self._in_stream:
|
||||
self._in_stream.stop()
|
||||
self._in_stream.close()
|
||||
self._in_stream = None
|
||||
async with self._stream_lock:
|
||||
self._stream_generation += 1
|
||||
if self._in_stream:
|
||||
self._in_stream.stop()
|
||||
self._in_stream.close()
|
||||
self._in_stream = None
|
||||
if hasattr(self, "_transport"):
|
||||
await self._transport.stop_device_monitor(self)
|
||||
|
||||
def _audio_in_callback(self, indata, frame_count, time_info, status):
|
||||
def _audio_in_callback(self, generation, indata, frame_count, time_info, status):
|
||||
if generation != self._stream_generation:
|
||||
return
|
||||
if status:
|
||||
logger.trace(f"Audio input status: {status}")
|
||||
|
||||
@@ -83,7 +134,16 @@ class SoundDeviceInputTransport(BaseInputTransport):
|
||||
num_channels=self._params.audio_in_channels,
|
||||
)
|
||||
|
||||
asyncio.run_coroutine_threadsafe(self.push_audio_frame(frame), self.get_event_loop())
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self._push_audio_frame_if_current(generation, frame), self.get_event_loop()
|
||||
)
|
||||
|
||||
async def _push_audio_frame_if_current(self, generation, frame):
|
||||
"""Serialize frame delivery with replacement so a closed route cannot leak audio."""
|
||||
async with self._stream_lock:
|
||||
if generation != self._stream_generation:
|
||||
return
|
||||
await self.push_audio_frame(frame)
|
||||
|
||||
|
||||
class SoundDeviceOutputTransport(BaseOutputTransport):
|
||||
@@ -97,6 +157,7 @@ class SoundDeviceOutputTransport(BaseOutputTransport):
|
||||
self._sample_rate = 0
|
||||
# Writes are serialized by the pipeline, so one worker is enough.
|
||||
self._executor = ThreadPoolExecutor(max_workers=1)
|
||||
self._stream_lock = asyncio.Lock()
|
||||
|
||||
async def start(self, frame: StartFrame):
|
||||
await super().start(frame)
|
||||
@@ -105,53 +166,258 @@ class SoundDeviceOutputTransport(BaseOutputTransport):
|
||||
return
|
||||
|
||||
self._sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
|
||||
await self._open_stream()
|
||||
|
||||
self._out_stream = sd.RawOutputStream(
|
||||
await self.set_transport_ready(frame)
|
||||
if hasattr(self, "_transport"):
|
||||
await self._transport.start_device_monitor(self)
|
||||
|
||||
async def reopen(self, *, device=None):
|
||||
"""Recreate an unset-device stream so PortAudio resolves the new default."""
|
||||
if not self._out_stream:
|
||||
return
|
||||
async with self._stream_lock:
|
||||
old_stream = self._out_stream
|
||||
old_device = old_stream.device
|
||||
# Keep the current output route intact when the selected default
|
||||
# cannot satisfy this stream's negotiated format.
|
||||
sd.check_output_settings(
|
||||
device=self._params.output_device if device is None else device, samplerate=self._sample_rate,
|
||||
channels=self._params.audio_out_channels, dtype="int16",
|
||||
)
|
||||
try:
|
||||
old_stream.stop()
|
||||
old_stream.close()
|
||||
self._out_stream = None
|
||||
await self._open_stream(device=device)
|
||||
except Exception as exc:
|
||||
logger.warning(f"Audio output route change failed; restoring prior stream: {type(exc).__name__}")
|
||||
try:
|
||||
self._out_stream = None
|
||||
await self._open_stream(device=old_device)
|
||||
except Exception as restore_exc:
|
||||
logger.error(f"Audio output fallback unavailable: {type(restore_exc).__name__}")
|
||||
raise
|
||||
|
||||
async def _open_stream(self, *, device=None):
|
||||
stream = sd.RawOutputStream(
|
||||
samplerate=self._sample_rate,
|
||||
device=self._params.output_device,
|
||||
channels=self._params.audio_out_channels,
|
||||
dtype="int16",
|
||||
device=self._params.output_device if device is None else device,
|
||||
channels=self._params.audio_out_channels, dtype="int16",
|
||||
)
|
||||
self._out_stream.start()
|
||||
|
||||
try:
|
||||
stream.start()
|
||||
except Exception:
|
||||
stream.close()
|
||||
raise
|
||||
self._out_stream = stream
|
||||
device_name = sd.query_devices(self._out_stream.device, "output")["name"]
|
||||
logger.info(f"Speaker: {device_name} @ {self._sample_rate} Hz")
|
||||
|
||||
await self.set_transport_ready(frame)
|
||||
|
||||
async def cleanup(self):
|
||||
await super().cleanup()
|
||||
if self._out_stream:
|
||||
self._out_stream.stop()
|
||||
self._out_stream.close()
|
||||
self._out_stream = None
|
||||
async with self._stream_lock:
|
||||
if self._out_stream:
|
||||
self._out_stream.stop()
|
||||
self._out_stream.close()
|
||||
self._out_stream = None
|
||||
if hasattr(self, "_transport"):
|
||||
await self._transport.stop_device_monitor(self)
|
||||
|
||||
async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool:
|
||||
if not self._out_stream:
|
||||
return False
|
||||
await self.get_event_loop().run_in_executor(
|
||||
self._executor, self._out_stream.write, frame.audio
|
||||
)
|
||||
async with self._stream_lock:
|
||||
if not self._out_stream:
|
||||
return False
|
||||
await self.get_event_loop().run_in_executor(
|
||||
self._executor, self._out_stream.write, frame.audio
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
class SoundDeviceTransport(BaseTransport):
|
||||
"""Local microphone + speaker transport."""
|
||||
|
||||
def __init__(self, params: SoundDeviceTransportParams):
|
||||
def __init__(self, params: SoundDeviceTransportParams, *, device_monitor=None,
|
||||
device_event_sink=None):
|
||||
super().__init__()
|
||||
self._params = params
|
||||
self._input: SoundDeviceInputTransport | None = None
|
||||
self._output: SoundDeviceOutputTransport | None = None
|
||||
self._device_monitor = device_monitor
|
||||
self._device_event_sink = device_event_sink
|
||||
self._last_snapshot: AudioDeviceSnapshot | None = None
|
||||
self._restart_lock = asyncio.Lock()
|
||||
self._monitor_started = False
|
||||
self._monitor_owners: set[object] = set()
|
||||
self._monitor_lock = asyncio.Lock()
|
||||
|
||||
def _monitor_if_needed(self):
|
||||
if self._device_monitor is not None:
|
||||
return self._device_monitor
|
||||
if sys.platform != "darwin":
|
||||
return None
|
||||
if self._params.input_device is None or self._params.output_device is None:
|
||||
try:
|
||||
self._device_monitor = create_macos_audio_monitor()
|
||||
except Exception as exc:
|
||||
logger.warning(f"Audio default monitoring unavailable: {exc}")
|
||||
return self._device_monitor
|
||||
|
||||
async def start_device_monitor(self, owner=None):
|
||||
"""Keep the shared monitor running while any transport side is active."""
|
||||
owner = self if owner is None else owner
|
||||
async with self._monitor_lock:
|
||||
self._monitor_owners.add(owner)
|
||||
if self._monitor_started:
|
||||
return
|
||||
monitor = self._monitor_if_needed()
|
||||
if monitor:
|
||||
await monitor.start(self._on_device_change)
|
||||
self._monitor_started = True
|
||||
|
||||
async def stop_device_monitor(self, owner=None):
|
||||
"""Release one transport side; stop only after the final release."""
|
||||
owner = self if owner is None else owner
|
||||
async with self._monitor_lock:
|
||||
self._monitor_owners.discard(owner)
|
||||
if self._monitor_owners or not self._device_monitor or not self._monitor_started:
|
||||
return
|
||||
await self._device_monitor.stop()
|
||||
self._monitor_started = False
|
||||
|
||||
async def _on_device_change(self, snapshot: AudioDeviceSnapshot):
|
||||
if self._last_snapshot and snapshot.generation <= self._last_snapshot.generation:
|
||||
logger.debug(f"Ignoring stale audio route event generation={snapshot.generation}")
|
||||
return
|
||||
if self._device_event_sink:
|
||||
try:
|
||||
result = self._device_event_sink(snapshot)
|
||||
if inspect.isawaitable(result):
|
||||
await result
|
||||
except Exception as exc:
|
||||
logger.warning(f"Audio device event sink failed: {type(exc).__name__}")
|
||||
old = self._last_snapshot
|
||||
self._last_snapshot = snapshot
|
||||
if old is None:
|
||||
return
|
||||
input_changed = old.default_input_uid != snapshot.default_input_uid
|
||||
output_changed = old.default_output_uid != snapshot.default_output_uid
|
||||
if not (input_changed or output_changed):
|
||||
return
|
||||
async with self._restart_lock:
|
||||
if input_changed and self._params.input_device is None and self._input:
|
||||
try:
|
||||
await self._reopen_default(self._input, snapshot, snapshot.default_input_uid, "input")
|
||||
except Exception as exc:
|
||||
logger.warning(f"Audio input route refresh failed: {type(exc).__name__}")
|
||||
if output_changed and self._params.output_device is None and self._output:
|
||||
try:
|
||||
await self._reopen_default(self._output, snapshot, snapshot.default_output_uid, "output")
|
||||
except Exception as exc:
|
||||
logger.warning(f"Audio output route refresh failed: {type(exc).__name__}")
|
||||
|
||||
async def _reopen_default(self, stream, snapshot: AudioDeviceSnapshot, uid: str | None, direction: str) -> None:
|
||||
# Empty snapshots are supported for legacy/injected monitors. Native
|
||||
# snapshots always carry devices and therefore get an explicit index.
|
||||
if not snapshot.devices:
|
||||
await stream.reopen()
|
||||
return
|
||||
await stream.reopen(device=self._portaudio_device(snapshot, uid, direction))
|
||||
|
||||
@staticmethod
|
||||
def _portaudio_device(snapshot: AudioDeviceSnapshot, uid: str | None, direction: str) -> int:
|
||||
"""Map Core Audio's current default to an explicit PortAudio index.
|
||||
|
||||
``device=None`` in a long-lived sounddevice process retains PortAudio's
|
||||
startup default. Reopening with the current index is what makes a
|
||||
default-device event actually move the live stream.
|
||||
"""
|
||||
if not uid or uid not in snapshot.devices:
|
||||
raise RuntimeError(f"No available Core Audio default {direction} device")
|
||||
native = snapshot.devices[uid]
|
||||
capability = "max_input_channels" if direction == "input" else "max_output_channels"
|
||||
matches = [index for index, candidate in enumerate(sd.query_devices())
|
||||
if candidate["name"] == native.name and candidate[capability] > 0]
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError(f"No unique PortAudio {direction} device for {native.name!r}: {matches}")
|
||||
return matches[0]
|
||||
|
||||
@staticmethod
|
||||
def available_devices() -> list[dict]:
|
||||
return [
|
||||
{"id": index, "name": device["name"], "input": bool(device["max_input_channels"]),
|
||||
"output": bool(device["max_output_channels"])}
|
||||
for index, device in enumerate(sd.query_devices())
|
||||
if device["max_input_channels"] or device["max_output_channels"]
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def _select_device(cls, request: int | str, direction: str) -> int:
|
||||
capability = "input" if direction == "input" else "output"
|
||||
devices = cls.available_devices()
|
||||
if isinstance(request, int) or (isinstance(request, str) and request.isdecimal()):
|
||||
index = int(request)
|
||||
if any(device["id"] == index and device[capability] for device in devices):
|
||||
return index
|
||||
else:
|
||||
needle = str(request).casefold().strip()
|
||||
matches = [device["id"] for device in devices
|
||||
if device[capability] and needle in device["name"].casefold()]
|
||||
if len(matches) == 1:
|
||||
return matches[0]
|
||||
raise ValueError(f"No unique available {direction} device matches {request!r}")
|
||||
|
||||
async def set_runtime_device(self, direction: str, request: int | str | None) -> dict:
|
||||
"""Pin one route live, or pass ``default``/None to follow macOS again."""
|
||||
if direction not in {"input", "output"}:
|
||||
raise ValueError("direction must be input or output")
|
||||
following_default = request is None or str(request).casefold().strip() in {"default", "mac default", "system default"}
|
||||
selected = None if following_default else self._select_device(request, direction) # type: ignore[arg-type]
|
||||
previous = self._params.input_device if direction == "input" else self._params.output_device
|
||||
async with self._restart_lock:
|
||||
stream = self._input if direction == "input" else self._output
|
||||
try:
|
||||
# Temporarily clear this pin so an all-pinned transport can
|
||||
# create its native monitor and take a fresh default snapshot.
|
||||
if following_default:
|
||||
if direction == "input":
|
||||
self._params.input_device = None
|
||||
else:
|
||||
self._params.output_device = None
|
||||
await self.start_device_monitor()
|
||||
if stream:
|
||||
if following_default:
|
||||
snapshot = self._last_snapshot
|
||||
if not snapshot:
|
||||
raise RuntimeError(f"No macOS default-{direction} snapshot is available yet")
|
||||
uid = snapshot.default_input_uid if direction == "input" else snapshot.default_output_uid
|
||||
await self._reopen_default(stream, snapshot, uid, direction)
|
||||
else:
|
||||
await stream.reopen(device=selected)
|
||||
if direction == "input":
|
||||
self._params.input_device = selected
|
||||
else:
|
||||
self._params.output_device = selected
|
||||
except Exception:
|
||||
if direction == "input":
|
||||
self._params.input_device = previous
|
||||
else:
|
||||
self._params.output_device = previous
|
||||
raise
|
||||
name = "macOS default" if following_default else next(device["name"] for device in self.available_devices() if device["id"] == selected)
|
||||
logger.info(f"Runtime {direction} device changed to {name}")
|
||||
return {"direction": direction, "device": selected, "name": name, "following_default": following_default}
|
||||
|
||||
def input(self) -> FrameProcessor:
|
||||
if not self._input:
|
||||
self._input = SoundDeviceInputTransport(self._params)
|
||||
self._input._transport = self
|
||||
return self._input
|
||||
|
||||
def output(self) -> FrameProcessor:
|
||||
if not self._output:
|
||||
self._output = SoundDeviceOutputTransport(self._params)
|
||||
self._output._transport = self
|
||||
return self._output
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user