"""Pipecat local-audio transport backed by sounddevice instead of PyAudio. Pipecat ships `pipecat.transports.local.audio.LocalAudioTransport`, but it needs PyAudio, which has no macOS wheel and must be compiled against a Homebrew portaudio. This machine blocks both, so we talk to the same portaudio through sounddevice, whose wheel bundles a prebuilt dylib. The frame contract is identical to the upstream transport. """ import asyncio import inspect import sys from concurrent.futures import ThreadPoolExecutor import sounddevice as sd from loguru import logger from pipecat.frames.frames import InputAudioRawFrame, OutputAudioRawFrame, StartFrame from pipecat.processors.frame_processor import FrameProcessor from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams from audio_device_monitor import AudioDeviceSnapshot, create_macos_audio_monitor class SoundDeviceTransportParams(TransportParams): """Configuration for the sounddevice transport. Parameters: input_device: sounddevice device index or name substring. None uses the default. output_device: sounddevice device index or name substring. None uses the default. """ input_device: int | str | None = None output_device: int | str | None = None class SoundDeviceInputTransport(BaseInputTransport): """Captures microphone audio and pushes it into the pipeline.""" _params: SoundDeviceTransportParams _transport: "SoundDeviceTransport" def __init__(self, params: SoundDeviceTransportParams): super().__init__(params) self._in_stream: sd.RawInputStream | None = None self._sample_rate = 0 self._stream_generation = 0 self._stream_lock = asyncio.Lock() async def start(self, frame: StartFrame): await super().start(frame) if self._in_stream: return self._sample_rate = self._params.audio_in_sample_rate or frame.audio_in_sample_rate blocksize = int(self._sample_rate / 100) * 2 # 20ms await self._open_stream() await self.set_transport_ready(frame) if hasattr(self, "_transport"): await self._transport.start_device_monitor(self) async def reopen(self, *, device=None): """Recreate an unset-device stream so PortAudio resolves the new default.""" if not self._in_stream: return async with self._stream_lock: old_stream = self._in_stream old_device = old_stream.device # Fail before disrupting an otherwise healthy conversation whenever # PortAudio can already tell us that the new default is unavailable. sd.check_input_settings( device=self._params.input_device if device is None else device, samplerate=self._sample_rate, channels=self._params.audio_in_channels, dtype="int16", ) self._stream_generation += 1 # makes callbacks from the old stream inert try: old_stream.stop() old_stream.close() self._in_stream = None await self._open_stream(device=device) except Exception as exc: logger.warning(f"Audio input route change failed; restoring prior stream: {type(exc).__name__}") try: self._in_stream = None await self._open_stream(device=old_device) except Exception as restore_exc: logger.error(f"Audio input fallback unavailable: {type(restore_exc).__name__}") raise async def _open_stream(self, *, device=None): blocksize = int(self._sample_rate / 100) * 2 self._stream_generation += 1 generation = self._stream_generation stream = sd.RawInputStream( samplerate=self._sample_rate, blocksize=blocksize, device=self._params.input_device if device is None else device, channels=self._params.audio_in_channels, dtype="int16", callback=lambda *args: self._audio_in_callback(generation, *args), ) try: stream.start() except Exception: stream.close() raise self._in_stream = stream device_name = sd.query_devices(self._in_stream.device, "input")["name"] logger.info(f"Microphone: {device_name} @ {self._sample_rate} Hz") async def cleanup(self): await super().cleanup() async with self._stream_lock: self._stream_generation += 1 if self._in_stream: self._in_stream.stop() self._in_stream.close() self._in_stream = None if hasattr(self, "_transport"): await self._transport.stop_device_monitor(self) def _audio_in_callback(self, generation, indata, frame_count, time_info, status): if generation != self._stream_generation: return if status: logger.trace(f"Audio input status: {status}") frame = InputAudioRawFrame( audio=bytes(indata), sample_rate=self._sample_rate, num_channels=self._params.audio_in_channels, ) asyncio.run_coroutine_threadsafe( self._push_audio_frame_if_current(generation, frame), self.get_event_loop() ) async def _push_audio_frame_if_current(self, generation, frame): """Serialize frame delivery with replacement so a closed route cannot leak audio.""" async with self._stream_lock: if generation != self._stream_generation: return await self.push_audio_frame(frame) class SoundDeviceOutputTransport(BaseOutputTransport): """Plays pipeline audio out through the speakers.""" _params: SoundDeviceTransportParams def __init__(self, params: SoundDeviceTransportParams): super().__init__(params) self._out_stream: sd.RawOutputStream | None = None self._sample_rate = 0 # Writes are serialized by the pipeline, so one worker is enough. self._executor = ThreadPoolExecutor(max_workers=1) self._stream_lock = asyncio.Lock() async def start(self, frame: StartFrame): await super().start(frame) if self._out_stream: return self._sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate await self._open_stream() await self.set_transport_ready(frame) if hasattr(self, "_transport"): await self._transport.start_device_monitor(self) async def reopen(self, *, device=None): """Recreate an unset-device stream so PortAudio resolves the new default.""" if not self._out_stream: return async with self._stream_lock: old_stream = self._out_stream old_device = old_stream.device # Keep the current output route intact when the selected default # cannot satisfy this stream's negotiated format. sd.check_output_settings( device=self._params.output_device if device is None else device, samplerate=self._sample_rate, channels=self._params.audio_out_channels, dtype="int16", ) try: old_stream.stop() old_stream.close() self._out_stream = None await self._open_stream(device=device) except Exception as exc: logger.warning(f"Audio output route change failed; restoring prior stream: {type(exc).__name__}") try: self._out_stream = None await self._open_stream(device=old_device) except Exception as restore_exc: logger.error(f"Audio output fallback unavailable: {type(restore_exc).__name__}") raise async def _open_stream(self, *, device=None): stream = sd.RawOutputStream( samplerate=self._sample_rate, device=self._params.output_device if device is None else device, channels=self._params.audio_out_channels, dtype="int16", ) try: stream.start() except Exception: stream.close() raise self._out_stream = stream device_name = sd.query_devices(self._out_stream.device, "output")["name"] logger.info(f"Speaker: {device_name} @ {self._sample_rate} Hz") async def cleanup(self): await super().cleanup() async with self._stream_lock: if self._out_stream: self._out_stream.stop() self._out_stream.close() self._out_stream = None if hasattr(self, "_transport"): await self._transport.stop_device_monitor(self) async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool: async with self._stream_lock: if not self._out_stream: return False await self.get_event_loop().run_in_executor( self._executor, self._out_stream.write, frame.audio ) return True class SoundDeviceTransport(BaseTransport): """Local microphone + speaker transport.""" def __init__(self, params: SoundDeviceTransportParams, *, device_monitor=None, device_event_sink=None): super().__init__() self._params = params self._input: SoundDeviceInputTransport | None = None self._output: SoundDeviceOutputTransport | None = None self._device_monitor = device_monitor self._device_event_sink = device_event_sink self._last_snapshot: AudioDeviceSnapshot | None = None self._restart_lock = asyncio.Lock() self._monitor_started = False self._monitor_owners: set[object] = set() self._monitor_lock = asyncio.Lock() def _monitor_if_needed(self): if self._device_monitor is not None: return self._device_monitor if sys.platform != "darwin": return None if self._params.input_device is None or self._params.output_device is None: try: self._device_monitor = create_macos_audio_monitor() except Exception as exc: logger.warning(f"Audio default monitoring unavailable: {exc}") return self._device_monitor async def start_device_monitor(self, owner=None): """Keep the shared monitor running while any transport side is active.""" owner = self if owner is None else owner async with self._monitor_lock: self._monitor_owners.add(owner) if self._monitor_started: return monitor = self._monitor_if_needed() if monitor: await monitor.start(self._on_device_change) self._monitor_started = True async def stop_device_monitor(self, owner=None): """Release one transport side; stop only after the final release.""" owner = self if owner is None else owner async with self._monitor_lock: self._monitor_owners.discard(owner) if self._monitor_owners or not self._device_monitor or not self._monitor_started: return await self._device_monitor.stop() self._monitor_started = False async def _on_device_change(self, snapshot: AudioDeviceSnapshot): if self._last_snapshot and snapshot.generation <= self._last_snapshot.generation: logger.debug(f"Ignoring stale audio route event generation={snapshot.generation}") return if self._device_event_sink: try: result = self._device_event_sink(snapshot) if inspect.isawaitable(result): await result except Exception as exc: logger.warning(f"Audio device event sink failed: {type(exc).__name__}") old = self._last_snapshot self._last_snapshot = snapshot if old is None: return input_changed = old.default_input_uid != snapshot.default_input_uid output_changed = old.default_output_uid != snapshot.default_output_uid if not (input_changed or output_changed): return async with self._restart_lock: if input_changed and self._params.input_device is None and self._input: try: await self._reopen_default(self._input, snapshot, snapshot.default_input_uid, "input") except Exception as exc: logger.warning(f"Audio input route refresh failed: {type(exc).__name__}") if output_changed and self._params.output_device is None and self._output: try: await self._reopen_default(self._output, snapshot, snapshot.default_output_uid, "output") except Exception as exc: logger.warning(f"Audio output route refresh failed: {type(exc).__name__}") async def _reopen_default(self, stream, snapshot: AudioDeviceSnapshot, uid: str | None, direction: str) -> None: # Empty snapshots are supported for legacy/injected monitors. Native # snapshots always carry devices and therefore get an explicit index. if not snapshot.devices: await stream.reopen() return await stream.reopen(device=self._portaudio_device(snapshot, uid, direction)) @staticmethod def _portaudio_device(snapshot: AudioDeviceSnapshot, uid: str | None, direction: str) -> int: """Map Core Audio's current default to an explicit PortAudio index. ``device=None`` in a long-lived sounddevice process retains PortAudio's startup default. Reopening with the current index is what makes a default-device event actually move the live stream. """ if not uid or uid not in snapshot.devices: raise RuntimeError(f"No available Core Audio default {direction} device") native = snapshot.devices[uid] capability = "max_input_channels" if direction == "input" else "max_output_channels" matches = [index for index, candidate in enumerate(sd.query_devices()) if candidate["name"] == native.name and candidate[capability] > 0] if len(matches) != 1: raise RuntimeError(f"No unique PortAudio {direction} device for {native.name!r}: {matches}") return matches[0] @staticmethod def available_devices() -> list[dict]: return [ {"id": index, "name": device["name"], "input": bool(device["max_input_channels"]), "output": bool(device["max_output_channels"])} for index, device in enumerate(sd.query_devices()) if device["max_input_channels"] or device["max_output_channels"] ] @classmethod def _select_device(cls, request: int | str, direction: str) -> int: capability = "input" if direction == "input" else "output" devices = cls.available_devices() if isinstance(request, int) or (isinstance(request, str) and request.isdecimal()): index = int(request) if any(device["id"] == index and device[capability] for device in devices): return index else: needle = str(request).casefold().strip() matches = [device["id"] for device in devices if device[capability] and needle in device["name"].casefold()] if len(matches) == 1: return matches[0] raise ValueError(f"No unique available {direction} device matches {request!r}") async def set_runtime_device(self, direction: str, request: int | str | None) -> dict: """Pin one route live, or pass ``default``/None to follow macOS again.""" if direction not in {"input", "output"}: raise ValueError("direction must be input or output") following_default = request is None or str(request).casefold().strip() in {"default", "mac default", "system default"} selected = None if following_default else self._select_device(request, direction) # type: ignore[arg-type] previous = self._params.input_device if direction == "input" else self._params.output_device async with self._restart_lock: stream = self._input if direction == "input" else self._output try: # Temporarily clear this pin so an all-pinned transport can # create its native monitor and take a fresh default snapshot. if following_default: if direction == "input": self._params.input_device = None else: self._params.output_device = None await self.start_device_monitor() if stream: if following_default: snapshot = self._last_snapshot if not snapshot: raise RuntimeError(f"No macOS default-{direction} snapshot is available yet") uid = snapshot.default_input_uid if direction == "input" else snapshot.default_output_uid await self._reopen_default(stream, snapshot, uid, direction) else: await stream.reopen(device=selected) if direction == "input": self._params.input_device = selected else: self._params.output_device = selected except Exception: if direction == "input": self._params.input_device = previous else: self._params.output_device = previous raise name = "macOS default" if following_default else next(device["name"] for device in self.available_devices() if device["id"] == selected) logger.info(f"Runtime {direction} device changed to {name}") return {"direction": direction, "device": selected, "name": name, "following_default": following_default} def input(self) -> FrameProcessor: if not self._input: self._input = SoundDeviceInputTransport(self._params) self._input._transport = self return self._input def output(self) -> FrameProcessor: if not self._output: self._output = SoundDeviceOutputTransport(self._params) self._output._transport = self return self._output def list_devices() -> str: """Render the audio devices sounddevice can see, for `--list-devices`.""" lines = [] for index, device in enumerate(sd.query_devices()): capability = [] if device["max_input_channels"]: capability.append(f"in:{device['max_input_channels']}") if device["max_output_channels"]: capability.append(f"out:{device['max_output_channels']}") lines.append(f" [{index}] {device['name']} ({', '.join(capability)})") return "\n".join(lines)