Audio & Voice

import asyncio
from gaia.audio.audio_client import AudioClient

# Initialize audio client
audio = AudioClient(
    whisper_model_size="base",    # ASR model: base, small, medium, large
    audio_device_index=None,       # Auto-select default input device
    silence_threshold=0.5,         # Seconds of silence before processing speech
    mic_threshold=0.003,           # Microphone energy threshold for voice detection
    enable_tts=True,               # Enable text-to-speech
)

# Define message processor. It is awaited once per transcription, so it
# MUST be `async def` — AudioClient invokes it via asyncio.run(callback(text)).
# The return value is ignored; do your work (respond, speak, print) inside it.
async def process_user_message(message: str) -> None:
    # Your agent logic
    print(f"You said: {message}")

# Start voice chat (async — must be awaited)
async def main():
    await audio.start_voice_chat(message_processor_callback=process_user_message)

asyncio.run(main())

from gaia.audio.whisper_asr import WhisperAsr
from queue import Queue

# List available audio input devices (inherited from AudioRecorder)
tmp = WhisperAsr(model_size="base")
tmp.list_audio_devices()   # prints available input devices with index + name

# Create a dedicated transcription queue
transcription_queue = Queue()

# Initialize Whisper
asr = WhisperAsr(
    model_size="base",              # base, small, medium, large
    device_index=0,                 # Input device (None = default)
    transcription_queue=transcription_queue,
    enable_cuda=False,              # Use CUDA if available
    silence_threshold=0.003,        # Voice-activity energy threshold
    min_audio_length=16000 * 0.5,   # 0.5s minimum at 16kHz to trigger transcription
)

# Start streaming recording (runs in a worker thread)
asr.start_recording_streaming()

# Consume transcriptions
while True:
    if not transcription_queue.empty():
        text = transcription_queue.get()
        print(f"Transcribed: {text}")
        # Process text...

from gaia.audio.diarize import diarize, ensure_ready, is_available

if not is_available():
    ensure_ready(progress=print)   # downloads sherpa-onnx + models, once

# wav_path must be 16 kHz mono — the same file transcription reads
spans = diarize("meeting.wav", num_speakers=4)   # pass the count when known
for span in spans:
    print(f"{span.speaker}: {span.start:.1f}s - {span.end:.1f}s")

from gaia.audio.kokoro_tts import KokoroTTS

# Initialize TTS (voice defaults to "af_bella")
tts = KokoroTTS()

# List available voices
voices = tts.list_available_voices()
for voice_id, voice_data in voices.items():
    print(f"{voice_id}: {voice_data['name']} (quality: {voice_data['quality']})")

# Select a voice (must be called before generate_speech)
tts.set_voice("af_sarah")  # American female voice

# Generate speech (returns: (audio_samples: list[float], phonemes: str, stats: dict))
text = "Hello! This is a test of the Kokoro text-to-speech system."
audio_samples, phonemes, stats = tts.generate_speech(text)

# audio_samples are 24 kHz float32 PCM in the range [-1.0, 1.0].
# Save to WAV:
import numpy as np
import wave
audio_int16 = (np.array(audio_samples) * 32767).astype(np.int16)
with wave.open("output.wav", "wb") as wav_file:
    wav_file.setnchannels(1)
    wav_file.setsampwidth(2)
    wav_file.setframerate(24000)
    wav_file.writeframes(audio_int16.tobytes())

# Or play directly using sounddevice
import sounddevice as sd
sd.play(np.array(audio_samples), 24000)
sd.wait()

# Streaming synthesis (plays audio as chunks become available)
tts.generate_speech_streaming(text)