Adolfo Reyna
370 lines
17 KiB
Python
370 lines
17 KiB
Python
import logging, time, collections, numpy as np
|
|
from pathlib import Path
|
|
|
|
from ..config import load_config
|
|
from ..audio.capture import AudioCapture
|
|
from ..audio.vad import create_vad
|
|
from ..audio.wakeword import WakeWordDetector, KeywordSpotterFallback
|
|
from ..stt.faster_whisper_engine import FasterWhisperSTT
|
|
from ..speaker.identifier import SpeakerIdentifier
|
|
from ..vision.face_identifier import FaceIdentifier
|
|
from ..llm.echo import EchoLLM
|
|
from ..tts.piper import PiperTTS, EchoTTS
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
class JarvisPipeline:
|
|
def __init__(self, config_path=None):
|
|
self.cfg = load_config(config_path)
|
|
self.samplerate = self.cfg.get('audio',{}).get('samplerate',16000)
|
|
self.block_size = self.cfg.get('audio',{}).get('block_size',512)
|
|
self.device = self.cfg.get('audio',{}).get('device',None)
|
|
self.backend = self.cfg.get('audio',{}).get('backend','auto')
|
|
|
|
self.capture = AudioCapture(samplerate=self.samplerate, block_size=self.block_size, device=self.device, backend=self.backend)
|
|
self.vad = create_vad(self.cfg)
|
|
self.wakeword = WakeWordDetector(
|
|
models=self.cfg.get('wakeword',{}).get('models',['hey_jarvis']),
|
|
threshold=self.cfg.get('wakeword',{}).get('threshold',0.45),
|
|
cooldown=self.cfg.get('wakeword',{}).get('cooldown_sec',1.0)
|
|
)
|
|
self.keyword_fallback = KeywordSpotterFallback(["hey jarvis","jarvis"])
|
|
|
|
stt_cfg = self.cfg.get('stt',{})
|
|
self.stt = FasterWhisperSTT(
|
|
model=stt_cfg.get('model','base'),
|
|
language=stt_cfg.get('language','en'),
|
|
device=stt_cfg.get('device','cpu'),
|
|
compute_type=stt_cfg.get('compute_type','int8'),
|
|
vad_filter=stt_cfg.get('vad_filter',False)
|
|
)
|
|
|
|
self.speaker_id = None
|
|
if self.cfg.get('speaker',{}).get('enabled',True):
|
|
try:
|
|
self.speaker_id = SpeakerIdentifier(
|
|
enrollment_dir=self.cfg.get('speaker',{}).get('enrollment_dir','data/speakers'),
|
|
threshold=self.cfg.get('speaker',{}).get('threshold',0.60),
|
|
embedding_model=self.cfg.get('speaker',{}).get('embedding_model','speechbrain/spkrec-ecapa-voxceleb')
|
|
)
|
|
except Exception as e:
|
|
log.warning(f"Speaker ID disabled: {e}")
|
|
|
|
self.face_id = None
|
|
if self.cfg.get('vision',{}).get('enabled',True):
|
|
try:
|
|
self.face_id = FaceIdentifier(
|
|
camera_id=self.cfg.get('vision',{}).get('camera_id',0),
|
|
enrollment_dir=self.cfg.get('vision',{}).get('enrollment_dir','data/faces'),
|
|
threshold=self.cfg.get('vision',{}).get('threshold',0.55)
|
|
)
|
|
except Exception as e:
|
|
log.warning(f"Face ID disabled: {e}")
|
|
|
|
self.llm = self._create_llm()
|
|
self.tts = self._create_tts()
|
|
|
|
self.pre_roll_sec = self.cfg.get('wakeword',{}).get('pre_roll_sec',0.8)
|
|
self.pre_roll_samples = int(self.pre_roll_sec * self.samplerate)
|
|
self.pre_roll_buffer = collections.deque(maxlen=self.pre_roll_samples)
|
|
|
|
self.silence_ms = self.cfg.get('vad',{}).get('min_silence_ms',800)
|
|
self.followup_timeout = self.cfg.get('conversation',{}).get('followup_timeout_sec',12)
|
|
self.max_followup = self.cfg.get('conversation',{}).get('max_followup_turns',5)
|
|
|
|
self.history = []
|
|
self.transcription_dir = Path(self.cfg.get('logging',{}).get('transcription_dir','data/transcriptions'))
|
|
self.transcription_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
self.running = False
|
|
self.in_followup = False
|
|
self.followup_turns = 0
|
|
self.last_interaction = 0
|
|
self._last_face_result = {"name":"unknown","face_conf":0.0}
|
|
|
|
def _create_llm(self):
|
|
provider = self.cfg.get('llm',{}).get('provider','echo')
|
|
log.info(f"LLM provider: {provider}")
|
|
if provider == 'ollama':
|
|
try:
|
|
from ..llm.ollama import OllamaLLM
|
|
return OllamaLLM(
|
|
model=self.cfg.get('llm',{}).get('model','qwen2.5:3b'),
|
|
host=self.cfg.get('llm',{}).get('ollama_host','http://localhost:11434'),
|
|
temperature=self.cfg.get('llm',{}).get('temperature',0.7)
|
|
)
|
|
except Exception as e:
|
|
log.warning(f"Ollama failed: {e}, echo")
|
|
return EchoLLM()
|
|
elif provider == 'gemini_cli':
|
|
try:
|
|
from ..llm.gemini_cli import GeminiCLILLM
|
|
return GeminiCLILLM(workspace_dir="data", system_file=self.cfg.get('llm',{}).get('system_prompt_file','data/soul.md'))
|
|
except Exception as e:
|
|
log.warning(f"Gemini CLI failed: {e}")
|
|
return EchoLLM()
|
|
elif provider == 'hermes_api':
|
|
try:
|
|
from ..llm.hermes_api import HermesAPILLM
|
|
return HermesAPILLM(base_url=self.cfg.get('llm',{}).get('base_url','http://192.168.68.110:8000'))
|
|
except Exception as e:
|
|
log.warning(f"Hermes API failed: {e}")
|
|
return EchoLLM()
|
|
else:
|
|
return EchoLLM()
|
|
|
|
def _create_tts(self):
|
|
engine = self.cfg.get('tts',{}).get('engine','echo')
|
|
if engine == 'piper':
|
|
try:
|
|
return PiperTTS(voice=self.cfg.get('tts',{}).get('voice','en_US-lessac-medium'), models_dir="models", volume=self.cfg.get('tts',{}).get('volume',0.9))
|
|
except Exception as e:
|
|
log.warning(f"Piper failed: {e}, echo fallback")
|
|
return EchoTTS()
|
|
else:
|
|
return EchoTTS()
|
|
|
|
def log_transcription(self, text, speaker_info=None):
|
|
if not text:
|
|
return
|
|
from datetime import datetime
|
|
date_str = datetime.now().strftime("%Y-%m-%d")
|
|
time_str = datetime.now().strftime("%H:%M:%S")
|
|
speaker = speaker_info.get('name','unknown') if speaker_info else 'unknown'
|
|
log_file = self.transcription_dir / f"{date_str}.log"
|
|
try:
|
|
with open(log_file,'a') as f:
|
|
f.write(f"[{time_str}] ({speaker}) {text}\n")
|
|
except Exception as e:
|
|
log.warning(f"Log failed: {e}")
|
|
|
|
def run(self):
|
|
print("=== Jarvis v2 Starting ===")
|
|
print(f"Config: wakeword={self.cfg.get('wakeword',{}).get('models')} stt={self.cfg.get('stt',{}).get('model')} llm={self.cfg.get('llm',{}).get('provider')} backend={self.backend}")
|
|
print(f"Speaker ID: {self.speaker_id is not None} Face ID: {self.face_id is not None}")
|
|
print(f"Wakeword model loaded: {self.wakeword.oww_model is not None} names={self.wakeword.model_names}")
|
|
|
|
self.capture.start()
|
|
self.running = True
|
|
|
|
recording_buffer = []
|
|
is_recording = False
|
|
silence_start = None
|
|
|
|
system_prompt = ""
|
|
soul_path = self.cfg.get('llm',{}).get('system_prompt_file','data/soul.md')
|
|
if soul_path and Path(soul_path).exists():
|
|
system_prompt = Path(soul_path).read_text()[:4000]
|
|
|
|
print("Listening... say 'Hey Jarvis' (or just speak if wakeword model missing, keyword fallback via STT)")
|
|
# For keyword fallback mode, we need continuous STT? We'll use VAD-gated recording + STT keyword check
|
|
# If wakeword model missing, we go into VAD recording mode and check transcript for keyword
|
|
|
|
try:
|
|
while self.running:
|
|
chunk = self.capture.read(timeout=0.2)
|
|
if chunk is None:
|
|
if self.in_followup and (time.time() - self.last_interaction > self.followup_timeout):
|
|
print("Followup window expired, idle")
|
|
self.in_followup = False
|
|
self.followup_turns = 0
|
|
continue
|
|
|
|
self.pre_roll_buffer.extend(chunk)
|
|
|
|
if not is_recording:
|
|
if not self.vad.is_speech(chunk, self.samplerate):
|
|
if not self.in_followup:
|
|
continue
|
|
|
|
detected = None
|
|
if not self.in_followup:
|
|
detected = self.wakeword.detect(chunk)
|
|
# If wakeword model not available, use VAD to start recording and let STT keyword check handle it
|
|
if self.wakeword.oww_model is None and self.vad.is_speech(chunk):
|
|
# Start recording for keyword fallback
|
|
detected = {"model": "vad_fallback", "score": 0.5}
|
|
else:
|
|
if self.vad.is_speech(chunk, self.samplerate):
|
|
detected = {"model": "followup", "score": 1.0}
|
|
|
|
if detected:
|
|
print(f"\n>>> Wake detected: {detected['model']} score={detected.get('score',0):.2f}")
|
|
try:
|
|
import numpy as np
|
|
import sounddevice as sd
|
|
sr=16000
|
|
t=np.linspace(0,0.15,sr//6)
|
|
beep = 0.3*np.sin(2*np.pi*880*t)
|
|
# try pipewire playback? ignore
|
|
try:
|
|
sd.play(beep, sr)
|
|
sd.wait()
|
|
except:
|
|
pass
|
|
except:
|
|
pass
|
|
|
|
is_recording = True
|
|
recording_buffer = [np.array(self.pre_roll_buffer, dtype=np.float32)]
|
|
silence_start = None
|
|
|
|
face_result = {"name":"unknown","face_conf":0.0}
|
|
if self.face_id and self.cfg.get('vision',{}).get('check_on_wake',True):
|
|
try:
|
|
face_result = self.face_id.check()
|
|
print(f"Face: {face_result.get('name')} conf={face_result.get('face_conf',0):.2f} found={face_result.get('faces_found',0)}")
|
|
except Exception as e:
|
|
log.warning(f"Face check error: {e}")
|
|
self._last_face_result = face_result
|
|
continue
|
|
else:
|
|
recording_buffer.append(chunk.copy())
|
|
if self.vad.is_speech(chunk, self.samplerate):
|
|
silence_start = None
|
|
else:
|
|
if silence_start is None:
|
|
silence_start = time.time()
|
|
else:
|
|
if (time.time() - silence_start)*1000 > self.silence_ms:
|
|
total_len = sum(len(c) for c in recording_buffer)
|
|
if total_len > self.samplerate * 0.4:
|
|
is_recording = False
|
|
audio = np.concatenate(recording_buffer) if recording_buffer else np.array([])
|
|
recording_buffer = []
|
|
self._process_utterance(audio, system_prompt)
|
|
self.wakeword.reset()
|
|
self.pre_roll_buffer.clear()
|
|
else:
|
|
is_recording = False
|
|
recording_buffer = []
|
|
|
|
total_len = sum(len(c) for c in recording_buffer)
|
|
if total_len > self.samplerate * 12:
|
|
print("Max recording length, processing")
|
|
is_recording = False
|
|
audio = np.concatenate(recording_buffer) if recording_buffer else np.array([])
|
|
recording_buffer = []
|
|
self._process_utterance(audio, system_prompt)
|
|
self.wakeword.reset()
|
|
self.pre_roll_buffer.clear()
|
|
|
|
except KeyboardInterrupt:
|
|
print("\nShutting down...")
|
|
finally:
|
|
self.capture.stop()
|
|
if self.face_id:
|
|
self.face_id.close()
|
|
|
|
def _process_utterance(self, audio, system_prompt):
|
|
import numpy as np
|
|
if len(audio) < self.samplerate * 0.2:
|
|
print("Audio too short")
|
|
return
|
|
|
|
print(f"Transcribing {len(audio)/self.samplerate:.2f}s...")
|
|
stt_result = self.stt.transcribe(audio, self.samplerate)
|
|
text = stt_result.get('text','').strip()
|
|
print(f"STT: '{text}' conf={stt_result.get('confidence',0):.2f}")
|
|
|
|
if not text:
|
|
print("Empty transcription")
|
|
return
|
|
|
|
# Keyword fallback check when wakeword model missing or followup not active
|
|
is_keyword_trigger = self.keyword_fallback.check_transcript(text)
|
|
|
|
if not self.in_followup:
|
|
# If we were triggered by VAD fallback, we need keyword in text to confirm it's for Jarvis
|
|
if self.wakeword.oww_model is None:
|
|
if not is_keyword_trigger:
|
|
print(f"No wake keyword in '{text}', ignoring (keyword fallback)")
|
|
return
|
|
# Extract command after keyword
|
|
low = text.lower()
|
|
for kw in ["hey jarvis", "jarvis"]:
|
|
if kw in low:
|
|
idx = low.find(kw)
|
|
text = text[idx+len(kw):].strip(" ,:")
|
|
break
|
|
if not text:
|
|
print("Only wakeword, waiting for command in followup")
|
|
self.in_followup = True
|
|
self.followup_turns = 0
|
|
self.last_interaction = time.time()
|
|
return
|
|
else:
|
|
# in followup, allow empty? no
|
|
pass
|
|
|
|
speaker_info = {"name": "unknown", "voice_conf": 0.0, "face_conf": 0.0, "fused_conf": 0.0}
|
|
if hasattr(self, '_last_face_result'):
|
|
speaker_info['face_conf'] = self._last_face_result.get('face_conf',0.0)
|
|
|
|
if self.speaker_id:
|
|
try:
|
|
voice_result = self.speaker_id.identify(audio, self.samplerate)
|
|
speaker_info['name'] = voice_result.get('name','unknown')
|
|
speaker_info['voice_conf'] = voice_result.get('voice_conf',0.0)
|
|
speaker_info['voice_scores'] = voice_result.get('scores',{})
|
|
except Exception as e:
|
|
log.warning(f"Speaker ID error: {e}")
|
|
|
|
face_name = getattr(self, '_last_face_result', {}).get('name','unknown')
|
|
face_conf = getattr(self, '_last_face_result', {}).get('face_conf',0.0) if hasattr(self, '_last_face_result') else 0.0
|
|
final_name = speaker_info.get('name','unknown')
|
|
final_conf = speaker_info.get('voice_conf',0.0)
|
|
|
|
if face_name != "unknown" and final_name != "unknown":
|
|
if face_name == final_name:
|
|
final_conf = min(max(face_conf, final_conf) * 1.1, 0.99)
|
|
final_name = face_name
|
|
else:
|
|
if face_conf > final_conf:
|
|
final_name = face_name
|
|
final_conf = face_conf
|
|
elif face_name != "unknown" and final_name == "unknown":
|
|
final_name = face_name
|
|
final_conf = face_conf
|
|
|
|
speaker_info['name'] = final_name
|
|
speaker_info['fused_conf'] = final_conf
|
|
speaker_info['face_conf'] = face_conf
|
|
|
|
print(f"Speaker: {final_name} fused={final_conf:.2f} voice={speaker_info.get('voice_conf',0):.2f} face={face_conf:.2f}")
|
|
|
|
self.log_transcription(text, speaker_info)
|
|
|
|
messages = []
|
|
if system_prompt:
|
|
messages.append({"role": "system", "content": system_prompt})
|
|
for h in self.history[-self.cfg.get('llm',{}).get('max_history',8):]:
|
|
messages.append(h)
|
|
messages.append({"role": "user", "content": text})
|
|
|
|
print("Thinking...")
|
|
try:
|
|
response = self.llm.chat(messages, speaker=speaker_info)
|
|
except Exception as e:
|
|
log.error(f"LLM error: {e}")
|
|
response = f"Error: {e}"
|
|
|
|
print(f"Jarvis: {response}")
|
|
self.history.append({"role": "user", "content": text})
|
|
self.history.append({"role": "assistant", "content": response})
|
|
|
|
try:
|
|
self.tts.speak(response)
|
|
except Exception as e:
|
|
log.error(f"TTS error: {e}")
|
|
print(f"[TTS error] {response}")
|
|
|
|
self.in_followup = True
|
|
self.followup_turns += 1
|
|
self.last_interaction = time.time()
|
|
|
|
if self.followup_turns >= self.max_followup:
|
|
print("Max followup reached, idle")
|
|
self.in_followup = False
|
|
self.followup_turns = 0
|