feat: add QwenTTSService for MLX Qwen3-TTS 1.7B integration

qwen_tts_service.py

@@ -0,0 +1,118 @@
+"""qwen_tts_service.py
+
+Qwen3-TTS Service for Pipecat using MLX on Apple Silicon.
+Provides local speech synthesis using Alibaba Qwen3-TTS 1.7B
+with zero-shot voice cloning capabilities via mlx-audio.
+"""
+
+import sys
+import os
+import asyncio
+import numpy as np
+from pathlib import Path
+from collections.abc import AsyncGenerator
+from loguru import logger
+
+from pipecat.frames.frames import ErrorFrame, Frame, TTSAudioRawFrame
+from pipecat.services.tts_service import TTSService
+
+CUSTOM_VOICES_DIR = Path(__file__).resolve().parent / "custom_voices"
+JV_SAMPLE_WAV = Path.home() / "Library/Application Support/sh.voicebox.app/profiles/2f4e8f2e-dbc4-43fc-b940-c6ca7ac694c7/c496be86-267d-4793-b26a-7beead57fbd4.wav"
+JV_SAMPLE_TEXT = "I have completed a diagnostic scan of your current schedule, and it appears several conflicts have arisen. While I have taken the liberty of reorganizing your morning appointments to ensure maximum efficiency, I cannot account for human fatigue. Perhaps a second cup of coffee would be a logical next step."
+
+MODEL_ID = "mlx-community/Qwen3-TTS-12Hz-1.7B-Base-bf16"
+
+
+class QwenTTSService(TTSService):
+    def __init__(
+        self,
+        *,
+        voice: str = "qwen_jv",
+        model_id: str = MODEL_ID,
+        sample_rate: int = 24000,
+        temperature: float = 0.3,
+        **kwargs,
+    ):
+        super().__init__(sample_rate=sample_rate, **kwargs)
+        self._voice_name = voice
+        self._model_id = model_id
+        self._temperature = temperature
+        self._model = None
+
+    def _ensure_model_loaded(self):
+        if self._model is not None:
+            return
+        from mlx_audio.tts import load_model
+        logger.info(f"Loading local MLX Qwen3-TTS model ({self._model_id})...")
+        self._model = load_model(self._model_id)
+        logger.info("Qwen3-TTS model loaded successfully on Metal GPU.")
+
+    def set_voice(self, voice: str):
+        self._voice_name = voice
+        logger.info(f"QwenTTSService active voice set to '{voice}'")
+
+    async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
+        try:
+            await self.start_tts_usage_metrics(text)
+            self._ensure_model_loaded()
+
+            from mlx_audio.tts.generate import generate_audio
+
+            ref_audio = str(JV_SAMPLE_WAV) if JV_SAMPLE_WAV.exists() else None
+            ref_text = JV_SAMPLE_TEXT if ref_audio else None
+
+            # Generate audio using MLX Qwen3-TTS in background thread
+            loop = asyncio.get_running_loop()
+
+            output_file = "/tmp/qwen_gen_temp.wav"
+            await loop.run_in_executor(
+                None,
+                lambda: generate_audio(
+                    text=text,
+                    model=self._model,
+                    ref_audio=ref_audio,
+                    ref_text=ref_text,
+                    temperature=self._temperature,
+                    output_path=output_file,
+                    verbose=False,
+                ),
+            )
+
+            await self.stop_ttfb_metrics()
+
+            actual_wav = None
+            if os.path.isdir(output_file):
+                sub_wav = os.path.join(output_file, "audio_000.wav")
+                if os.path.exists(sub_wav):
+                    actual_wav = sub_wav
+            elif os.path.exists(output_file):
+                actual_wav = output_file
+
+            if actual_wav:
+                import scipy.io.wavfile as wavfile
+                sr, audio_data = wavfile.read(actual_wav)
+                if audio_data.dtype != np.int16:
+                    audio_data = (np.clip(audio_data, -1.0, 1.0) * 32767).astype(np.int16)
+                audio_bytes = audio_data.tobytes()
+
+                if os.path.isdir(output_file):
+                    import shutil
+                    shutil.rmtree(output_file, ignore_errors=True)
+                else:
+                    try:
+                        os.remove(output_file)
+                    except Exception:
+                        pass
+
+                yield TTSAudioRawFrame(
+                    audio=audio_bytes,
+                    sample_rate=sr,
+                    num_channels=1,
+                    context_id=context_id,
+                )
+
+        except Exception as e:
+            logger.error(f"Error in QwenTTSService: {e}")
+            yield ErrorFrame(error=f"Qwen TTS error: {e}")
+        finally:
+            await self.stop_ttfb_metrics()