diff --git a/bot.py b/bot.py index e49ac6d..e996593 100644 --- a/bot.py +++ b/bot.py @@ -392,6 +392,18 @@ def build_tts(args: argparse.Namespace, voice_manager=None): voice_manager.set_tts_processor(tts) return tts + if "qwen" in voice or voice == "qwen_jv": + from qwen_tts_service import QwenTTSService + logger.info(f"Text to speech: MLX Qwen3-TTS (Voice Clone: {voice})") + tts = QwenTTSService( + voice=voice, + sample_rate=TTS_SAMPLE_RATE, + text_filters=[SpokenTextFilter(voice_manager=voice_manager)], + ) + if voice_manager: + voice_manager.set_tts_processor(tts) + return tts + logger.info(f"Text to speech: Kokoro {voice}") tts = KokoroTTSService( settings=KokoroTTSService.Settings(voice=voice, language=Language.EN), diff --git a/qwen_tts_service.py b/qwen_tts_service.py new file mode 100644 index 0000000..78cb24b --- /dev/null +++ b/qwen_tts_service.py @@ -0,0 +1,118 @@ +"""qwen_tts_service.py + +Qwen3-TTS Service for Pipecat using MLX on Apple Silicon. +Provides local speech synthesis using Alibaba Qwen3-TTS 1.7B +with zero-shot voice cloning capabilities via mlx-audio. +""" + +import sys +import os +import asyncio +import numpy as np +from pathlib import Path +from collections.abc import AsyncGenerator +from loguru import logger + +from pipecat.frames.frames import ErrorFrame, Frame, TTSAudioRawFrame +from pipecat.services.tts_service import TTSService + +CUSTOM_VOICES_DIR = Path(__file__).resolve().parent / "custom_voices" +JV_SAMPLE_WAV = Path.home() / "Library/Application Support/sh.voicebox.app/profiles/2f4e8f2e-dbc4-43fc-b940-c6ca7ac694c7/c496be86-267d-4793-b26a-7beead57fbd4.wav" +JV_SAMPLE_TEXT = "I have completed a diagnostic scan of your current schedule, and it appears several conflicts have arisen. While I have taken the liberty of reorganizing your morning appointments to ensure maximum efficiency, I cannot account for human fatigue. Perhaps a second cup of coffee would be a logical next step." + +MODEL_ID = "mlx-community/Qwen3-TTS-12Hz-1.7B-Base-bf16" + + +class QwenTTSService(TTSService): + def __init__( + self, + *, + voice: str = "qwen_jv", + model_id: str = MODEL_ID, + sample_rate: int = 24000, + temperature: float = 0.3, + **kwargs, + ): + super().__init__(sample_rate=sample_rate, **kwargs) + self._voice_name = voice + self._model_id = model_id + self._temperature = temperature + self._model = None + + def _ensure_model_loaded(self): + if self._model is not None: + return + from mlx_audio.tts import load_model + logger.info(f"Loading local MLX Qwen3-TTS model ({self._model_id})...") + self._model = load_model(self._model_id) + logger.info("Qwen3-TTS model loaded successfully on Metal GPU.") + + def set_voice(self, voice: str): + self._voice_name = voice + logger.info(f"QwenTTSService active voice set to '{voice}'") + + async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]: + try: + await self.start_tts_usage_metrics(text) + self._ensure_model_loaded() + + from mlx_audio.tts.generate import generate_audio + + ref_audio = str(JV_SAMPLE_WAV) if JV_SAMPLE_WAV.exists() else None + ref_text = JV_SAMPLE_TEXT if ref_audio else None + + # Generate audio using MLX Qwen3-TTS in background thread + loop = asyncio.get_running_loop() + + output_file = "/tmp/qwen_gen_temp.wav" + await loop.run_in_executor( + None, + lambda: generate_audio( + text=text, + model=self._model, + ref_audio=ref_audio, + ref_text=ref_text, + temperature=self._temperature, + output_path=output_file, + verbose=False, + ), + ) + + await self.stop_ttfb_metrics() + + actual_wav = None + if os.path.isdir(output_file): + sub_wav = os.path.join(output_file, "audio_000.wav") + if os.path.exists(sub_wav): + actual_wav = sub_wav + elif os.path.exists(output_file): + actual_wav = output_file + + if actual_wav: + import scipy.io.wavfile as wavfile + sr, audio_data = wavfile.read(actual_wav) + if audio_data.dtype != np.int16: + audio_data = (np.clip(audio_data, -1.0, 1.0) * 32767).astype(np.int16) + audio_bytes = audio_data.tobytes() + + if os.path.isdir(output_file): + import shutil + shutil.rmtree(output_file, ignore_errors=True) + else: + try: + os.remove(output_file) + except Exception: + pass + + yield TTSAudioRawFrame( + audio=audio_bytes, + sample_rate=sr, + num_channels=1, + context_id=context_id, + ) + + except Exception as e: + logger.error(f"Error in QwenTTSService: {e}") + yield ErrorFrame(error=f"Qwen TTS error: {e}") + finally: + await self.stop_ttfb_metrics() diff --git a/test_qwen_tts_service.py b/test_qwen_tts_service.py new file mode 100644 index 0000000..37d88c8 --- /dev/null +++ b/test_qwen_tts_service.py @@ -0,0 +1,30 @@ +"""test_qwen_tts_service.py + +Unit test for QwenTTSService. +Verifies loading MLX Qwen3-TTS 1.7B model and generating audio frames. +""" + +import asyncio +from qwen_tts_service import QwenTTSService +from pipecat.frames.frames import TTSAudioRawFrame + + +async def main(): + print("Testing QwenTTSService with MLX Qwen3-TTS 1.7B...") + service = QwenTTSService(voice="qwen_jv") + + frames = [] + async for frame in service.run_tts("Hello! This is a test of Qwen 1.7B TTS service.", context_id="test_ctx"): + frames.append(frame) + + assert len(frames) > 0, "No frames generated by QwenTTSService" + audio_frames = [f for f in frames if isinstance(f, TTSAudioRawFrame)] + assert len(audio_frames) > 0, "No TTSAudioRawFrame generated" + + total_bytes = sum(len(f.audio) for f in audio_frames) + duration_s = total_bytes / (24000 * 2) + print(f"PASS: Generated {len(audio_frames)} audio frame(s), total {total_bytes} bytes ({duration_s:.2f}s audio at 24kHz)!") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/voice_manager.py b/voice_manager.py index cff873f..5a5d985 100644 --- a/voice_manager.py +++ b/voice_manager.py @@ -30,7 +30,8 @@ KOKORO_VOICES = { "bm_fable": "British Male - Expressive", "bm_stuart": "British Male - Stuart Bell", "custom_pocket": "Pocket Voice Clone (Default)", - "jv_pocket": "JV Voice Profile (Cloned via Voicebox sample)", + "jv_pocket": "JV Voice Profile (Kyutai Pocket TTS)", + "qwen_jv": "JV Voice Profile (MLX Qwen 1.7B TTS)", } MACOS_VOICES = { @@ -71,6 +72,10 @@ VOICE_ALIASES = { "jv pocket": "jv_pocket", "jv profile": "jv_pocket", "jv voice": "jv_pocket", + "qwen": "qwen_jv", + "qwen_jv": "qwen_jv", + "qwen 1.7b": "qwen_jv", + "qwen3": "qwen_jv", }