# Examples

Complete code examples for common RoohAI use cases.

---

## 1. Minimal Voice Agent (Python Only)

Use the pipeline directly without the web server:

```python
"""Minimal voice agent — record from mic, get AI response, play audio."""

import asyncio
import soundfile as sf
import numpy as np
from roohai.pipeline import Rooh, register_all_models

async def main():
    # Initialize
    register_all_models()
    pipeline = Rooh("config.yaml")
    pipeline.load()

    # Load audio from file (or record from mic)
    audio, sr = sf.read("question.wav", dtype="float32")

    # Run the full pipeline
    transcription, response, audio_bytes = await pipeline.process_audio(
        audio_to_wav_bytes(audio, sr)
    )

    print(f"You said: {transcription}")
    print(f"AI said:  {response}")

    # Save the response audio
    with open("response.wav", "wb") as f:
        f.write(audio_bytes)

asyncio.run(main())
```

---

## 2. Step-by-Step Pipeline Control

Run each pipeline stage independently:

```python
"""Step-by-step pipeline — full control over each stage."""

import asyncio
import numpy as np
import soundfile as sf
from roohai.pipeline import Rooh, register_all_models
from roohai.audio import audio_to_wav_bytes, read_audio

async def main():
    register_all_models()
    pipeline = Rooh("config.yaml")
    pipeline.load()

    # Step 1: Transcribe
    audio_bytes = open("input.wav", "rb").read()
    transcription = await pipeline.transcribe(audio_bytes)
    print(f"Transcription: {transcription}")

    # Step 2: Get LLM response
    response = await pipeline.chat(transcription)
    print(f"Response: {response}")

    # Step 3: Synthesize speech
    wav_bytes = await pipeline.synthesize(response)
    with open("output.wav", "wb") as f:
        f.write(wav_bytes)
    print("Audio saved to output.wav")

asyncio.run(main())
```

---

## 3. Custom Pipeline from Code (No YAML)

Build a pipeline entirely in Python:

```python
"""Build a pipeline from code without config files."""

import asyncio
from roohai.pipeline import Rooh, register_all_models
from roohai.registry import registry

async def main():
    register_all_models()

    # Create pipeline from in-memory config
    config = {
        "pipeline": {
            "stt": "deepgram",
            "llm": "bedrock-claude",
            "tts": "piper",
            "vad": "silero",
        },
        "models": {
            "deepgram": {
                "type": "stt",
                "class": "deepgram",
                "api_key": "dg-xxxx",
                "model": "nova-3",
            },
            "bedrock-claude": {
                "type": "llm",
                "class": "bedrock",
                "model_id": "anthropic.claude-3-haiku-20240307-v1:0",
                "region": "us-east-1",
            },
            "piper": {
                "type": "tts",
                "class": "piper",
                "voice": "en_US-lessac-medium",
            },
            "silero": {
                "type": "vad",
                "class": "silero",
            },
        },
        "audio": {"sample_rate": 16000},
    }

    pipeline = Rooh.from_config_dict(config)
    pipeline.load()

    # Use the pipeline
    response = await pipeline.chat("What is the capital of France?")
    print(response)

    audio = await pipeline.synthesize(response)
    with open("paris.wav", "wb") as f:
        f.write(audio)

asyncio.run(main())
```

---

## 4. Hot-Swapping Models at Runtime

Switch models without restarting:

```python
"""Hot-swap models at runtime."""

import asyncio
from roohai.pipeline import Rooh, register_all_models

async def main():
    register_all_models()
    pipeline = Rooh("config.yaml")
    pipeline.load()

    # Start with SpeechT5
    audio1 = await pipeline.synthesize("Hello from SpeechT5!")
    with open("speecht5.wav", "wb") as f:
        f.write(audio1)

    # Swap to Piper
    pipeline.swap_model("tts", "piper")
    audio2 = await pipeline.synthesize("Hello from Piper!")
    with open("piper.wav", "wb") as f:
        f.write(audio2)

    # Swap to Cartesia (cloud)
    pipeline.swap_model("tts", "cartesia")
    audio3 = await pipeline.synthesize("Hello from Cartesia!")
    with open("cartesia.wav", "wb") as f:
        f.write(audio3)

    print("Saved three audio files with different TTS engines")

asyncio.run(main())
```

---

## 5. Using the Registry Directly

Work with individual models outside the pipeline:

```python
"""Use individual models via the registry."""

import asyncio
import numpy as np
from roohai.pipeline import register_all_models
from roohai.registry import registry

async def main():
    register_all_models()

    # Create and load a TTS model
    tts = registry.create_tts("piper")
    tts.load()
    audio_array, sample_rate = tts.synthesize("Hello, world!")
    print(f"Generated {len(audio_array)} samples at {sample_rate}Hz")
    tts.unload()

    # Create and load a VAD model
    vad = registry.create_vad("silero")
    vad.load()

    # Check a chunk for speech
    silence = np.zeros(512, dtype=np.float32)
    speech_prob = vad.is_speech(silence, 16000)
    print(f"Speech probability of silence: {speech_prob:.2f}")
    vad.unload()

    # Create an LLM
    llm = registry.create_llm("bedrock")
    llm.load()
    response = await llm.chat("Say hello in three languages.")
    print(response)
    llm.unload()

asyncio.run(main())
```

---

## 6. Creating an Agent via the REST API

Use `curl` or any HTTP client:

```bash
# Create an agent
curl -X POST http://localhost:8000/api/agents \
  -H "Content-Type: application/json" \
  -d '{
    "name": "fast-cloud",
    "pipeline": {
      "stt": "deepgram",
      "llm": "bedrock-claude",
      "tts": "cartesia",
      "vad": "silero"
    },
    "models": {
      "deepgram": {"class": "deepgram", "type": "stt", "model": "nova-3", "language": "en"},
      "bedrock-claude": {"class": "bedrock-claude", "type": "llm", "region": "us-east-1"},
      "cartesia": {"class": "cartesia", "type": "tts", "model_id": "sonic-2"},
      "silero": {"class": "silero", "type": "vad"}
    },
    "secrets": {
      "deepgram": {"api_key": "dg-xxxx"},
      "cartesia": {"api_key": "sk-xxxx"}
    }
  }'

# List agents
curl http://localhost:8000/api/agents

# Activate the agent
curl -X POST http://localhost:8000/api/agents/fast-cloud/activate

# Check health (shows active models)
curl http://localhost:8000/api/health
```

---

## 7. WebSocket Client (Python)

Connect to the voice streaming endpoint programmatically:

```python
"""WebSocket client for voice streaming."""

import asyncio
import base64
import json
import soundfile as sf
import numpy as np
import websockets

async def main():
    uri = "ws://localhost:8000/ws/voice-stream"

    async with websockets.connect(uri) as ws:
        # Read audio file and send as chunks
        audio, sr = sf.read("question.wav", dtype="float32")
        pcm16 = (audio * 32768).astype(np.int16)

        # Send in ~32ms chunks (512 samples at 16kHz)
        chunk_size = 512
        for i in range(0, len(pcm16), chunk_size):
            chunk = pcm16[i:i + chunk_size]
            b64 = base64.b64encode(chunk.tobytes()).decode()
            await ws.send(json.dumps({"type": "audio_chunk", "audio": b64}))
            await asyncio.sleep(0.032)  # Simulate real-time

        # Listen for responses
        audio_chunks = []
        session_id = None
        while True:
            try:
                msg = json.loads(await asyncio.wait_for(ws.recv(), timeout=10))
            except asyncio.TimeoutError:
                break

            if msg["type"] == "session":
                session_id = msg["session_id"]
                print(f"Session: {session_id}")
            elif msg["type"] == "transcription":
                print(f"You said: {msg['text']}")
            elif msg["type"] == "llm_response":
                print(f"AI said: {msg['text']}")
            elif msg["type"] == "llm_response_chunk":
                print(msg["text"], end="", flush=True)
            elif msg["type"] == "llm_response_done":
                print()  # newline after streaming
            elif msg["type"] == "metrics":
                parts = [f"{k}: {v:.0f}ms" for k, v in msg.items() if k != "type"]
                print(f"  [{', '.join(parts)}]")
            elif msg["type"] == "audio":
                audio_chunks.append(base64.b64decode(msg["audio"]))
            elif msg["type"] == "status" and msg["state"] == "listening":
                if audio_chunks:
                    break  # Done

        # Save response audio
        if audio_chunks:
            with open("response.wav", "wb") as f:
                for chunk in audio_chunks:
                    f.write(chunk)
            print("Response audio saved")

        # Notify server playback is done
        await ws.send(json.dumps({"type": "playback_done"}))

asyncio.run(main())
```

---

## 8. Fully Open-Source Voice Pipeline (No Cloud APIs)

Build a voice agent using only local, open-source models — no API keys needed:

```python
"""Fully open-source voice pipeline — Whisper + TinyLlama + Piper + Silero."""

import asyncio
from roohai.pipeline import Rooh, register_all_models
from roohai.registry import registry

async def main():
    register_all_models()

    # All local, all open-source — no API keys required
    config = {
        "pipeline": {
            "stt": "whisper-base",
            "llm": "local-llm",
            "tts": "piper",
            "vad": "silero",
        },
        "models": {
            "whisper-base": {
                "type": "stt",
                "class": "whisper",
                "model_id": "openai/whisper-base",
                "language": "en",
            },
            "local-llm": {
                "type": "llm",
                "class": "local",
                "model_id": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
            },
            "piper": {
                "type": "tts",
                "class": "piper",
                "voice": "en_US-lessac-medium",
            },
            "silero": {
                "type": "vad",
                "class": "silero",
            },
        },
        "audio": {"sample_rate": 16000},
    }

    pipeline = Rooh.from_config_dict(config)
    pipeline.load()
    pipeline.set_system_prompt("You are a helpful voice assistant. Be concise.")

    # Transcribe from file
    audio_bytes = open("question.wav", "rb").read()
    transcription = await pipeline.transcribe(audio_bytes)
    print(f"You said: {transcription}")

    # Generate response
    response = await pipeline.chat(transcription)
    print(f"AI said: {response}")

    # Synthesize speech
    wav_bytes = await pipeline.synthesize(response)
    with open("response.wav", "wb") as f:
        f.write(wav_bytes)

asyncio.run(main())
```

Available open-source model variants:
- **STT:** `whisper-tiny`, `whisper-base`, `whisper-small`, `whisper-large`, `wav2vec2`, `nvidia-canary`, `nvidia-parakeet`
- **LLM:** `local-llm` (TinyLlama)
- **TTS:** `speecht5`, `bark-small`, `bark-large`, `piper`
- **VAD:** `silero`

---

## 9. WebRTC Voice Agent (Browser JavaScript)

Connect to RoohAI using WebRTC for lower-latency audio:

```html
<!-- Include the WebRTC client -->
<script src="/static/webrtc-client.js"></script>

<script>
// Handle messages from the server (same format as WebSocket)
function onMessage(data) {
    switch (data.type) {
        case "status":
            console.log("State:", data.state, data.vad || "");
            break;
        case "interim_transcription":
            console.log("Interim:", data.text);
            break;
        case "transcription":
            console.log("You said:", data.text);
            break;
        case "llm_response":
            console.log("AI said:", data.text);
            break;
        case "interrupt":
            console.log("Barge-in — playback interrupted");
            // Mute briefly to avoid hearing stale audio
            WebRTCClient.muteRemoteAudio(300);
            WebRTCClient.sendPlaybackDone();
            break;
        case "playback_complete":
            // Server finished sending TTS frames.
            // Wait for the audio to actually play, then notify.
            setTimeout(function () {
                WebRTCClient.sendPlaybackDone();
            }, 200);
            break;
        case "error":
            console.error("Error:", data.message);
            break;
    }
}

function onDisconnect() {
    console.log("WebRTC disconnected");
}

// Connect (uses default microphone)
async function start() {
    try {
        await WebRTCClient.connect(null, onMessage, onDisconnect);
        console.log("WebRTC connected — speak into your microphone");
    } catch (err) {
        console.error("Connection failed:", err);
    }
}

// Disconnect
function stop() {
    WebRTCClient.disconnect();
}

start();
</script>
```

Key differences from WebSocket:
- **Audio transport:** Native Opus codec via RTCPeerConnection (no base64 encoding)
- **Playback:** Remote audio track plays automatically via `&lt;audio&gt;` element
- **Barge-in:** Server sends `interrupt` via data channel; client mutes remote audio briefly

---

## 10. WebRTC Agent via REST API

Create and activate a WebRTC agent:

```bash
# Create agent with WebRTC transport
curl -X POST http://localhost:8000/api/agents \
  -H "Content-Type: application/json" \
  -d '{
    "name": "webrtc-agent",
    "system_prompt": "You are a helpful voice assistant. Be concise.",
    "transport": "webrtc",
    "pipeline": {
      "stt": "deepgram",
      "llm": "bedrock-claude-haiku",
      "tts": "piper",
      "vad": "silero"
    },
    "models": {
      "deepgram": {"class": "deepgram", "type": "stt", "model": "nova-3"},
      "bedrock-claude-haiku": {"class": "bedrock-claude-haiku", "type": "llm", "region": "us-east-1"},
      "piper": {"class": "piper", "type": "tts", "voice": "en_US-lessac-medium"},
      "silero": {"class": "silero", "type": "vad"}
    },
    "secrets": {
      "deepgram": {"api_key": "dg-xxxx"}
    }
  }'

# Activate it
curl -X POST http://localhost:8000/api/agents/webrtc-agent/activate

# Verify — the frontend will now use WebRTC transport
curl http://localhost:8000/api/health
```

---

## 11. Barge-In (Interrupt) Implementation

Barge-in lets users interrupt the AI mid-response. Here's how it works end-to-end:

### Server-Side (Python)

The `BargeInState` class in `server/voice_common.py` tracks generation and playback:

```python
from server.voice_common import BargeInState

barge_in = BargeInState()

# When a new utterance arrives, cancel any in-flight generation
gen_id = await barge_in.start_generation(send_interrupt_callback)
cancel = barge_in.cancel_event

# Check cancel between pipeline stages
transcription = await pipeline.transcribe(wav_bytes)
if cancel.is_set():
    return  # User interrupted — abort

response = await pipeline.chat(transcription)
if cancel.is_set():
    return  # User interrupted — abort

# TTS with cancel support (skips remaining chunks if cancelled)
await synthesize_chunks(pipeline, response, send_audio, cancel)

# Mark server-side work as done (client may still be playing audio)
barge_in.finish_generation()

# When client reports playback finished:
barge_in.playback_done()
```

### State Machine

```
      ┌──────────┐   new utterance    ┌────────────┐
      │   Idle   │ ─────────────────> │ Generating │
      │          │                    │            │
      └──────────┘                    └─────┬──────┘
           ^                                │ LLM+TTS done
           │ playback_done                  v
      ┌──────────┐                    ┌────────────┐
      │          │ <───────────────── │  Playback  │
      │          │                    │  Pending   │
      └──────────┘                    └────────────┘

  At any point where is_active=true:
    VAD detects speech → cancel_event.set() → interrupt sent
```

### Client-Side (JavaScript)

```javascript
// WebSocket barge-in handling
ws.onmessage = function(event) {
    var data = JSON.parse(event.data);
    if (data.type === "interrupt") {
        // Stop audio immediately
        audioPlayer.pause();
        audioQueue = [];
        ws.send(JSON.stringify({ type: "playback_done" }));
    }
};

// WebRTC barge-in handling
function onMessage(data) {
    if (data.type === "interrupt") {
        // Mute remote track briefly (server stops sending TTS frames)
        WebRTCClient.muteRemoteAudio(300);
        WebRTCClient.sendPlaybackDone();
    }
}
```

### TTS Chunking with Cancel

Long responses are split into sentence-sized chunks. If the user interrupts mid-response, remaining chunks are skipped:

```python
from server.voice_common import synthesize_chunks

# synthesize_chunks checks cancel between each sentence chunk
await synthesize_chunks(
    pipeline,
    "This is a long response. It has many sentences. Each one is synthesized separately.",
    send_audio_callback,
    cancel=barge_in.cancel_event,  # set by VAD when speech detected
)
```

---

## 12. Agent Config Files

### WebRTC Agent (Deepgram + Claude + Piper)

```yaml
# agents/webrtc-agent.yaml
name: webrtc-agent
status: active
transport: webrtc
system_prompt: You are a helpful voice assistant. Be concise and friendly.
pipeline:
  stt: deepgram
  llm: bedrock-claude
  tts: piper
  vad: silero
models:
  deepgram:
    class: deepgram
    type: stt
    model: nova-3
    language: en
  bedrock-claude:
    class: bedrock-claude
    type: llm
    model_id: anthropic.claude-3-haiku-20240307-v1:0
    region: us-east-1
    auth_mode: environment
  piper:
    class: piper
    type: tts
    voice: en_US-lessac-medium
  silero:
    class: silero
    type: vad
```

### Cloud-First Agent (Deepgram + Claude + Cartesia)

```yaml
# agents/cloud-agent.yaml
name: cloud-agent
status: active
pipeline:
  stt: deepgram
  llm: bedrock-claude
  tts: cartesia
  vad: silero
models:
  deepgram:
    class: deepgram
    type: stt
    model: nova-3
    language: en
  bedrock-claude:
    class: bedrock-claude
    type: llm
    model_id: anthropic.claude-3-haiku-20240307-v1:0
    region: us-east-1
    auth_mode: environment
  cartesia:
    class: cartesia
    type: tts
    model_id: sonic-2
    voice_id: a0e99841-438c-4a64-b679-ae501e7d6091
    language: en
    sample_rate: 24000
  silero:
    class: silero
    type: vad
```

### Fully Local Agent (Whisper + TinyLlama + Piper)

```yaml
# agents/local-agent.yaml
name: local-agent
status: inactive
pipeline:
  stt: whisper
  llm: local-llm
  tts: piper
  vad: silero
models:
  whisper:
    class: whisper
    type: stt
    model_id: openai/whisper-tiny
    language: en
  local-llm:
    class: local
    type: llm
    model_id: TinyLlama/TinyLlama-1.1B-Chat-v1.0
  piper:
    class: piper
    type: tts
    voice: en_US-lessac-medium
  silero:
    class: silero
    type: vad
```

### Hybrid Agent (Deepgram + Claude + Piper)

```yaml
# agents/hybrid-agent.yaml
name: hybrid-agent
status: inactive
pipeline:
  stt: deepgram
  llm: bedrock-claude
  tts: piper
  vad: silero
models:
  deepgram:
    class: deepgram
    type: stt
    model: nova-3
    language: en
  bedrock-claude:
    class: bedrock-claude
    type: llm
    model_id: anthropic.claude-3-haiku-20240307-v1:0
    region: us-east-1
  piper:
    class: piper
    type: tts
    voice: en_GB-alan-medium
  silero:
    class: silero
    type: vad
```

---

## 13. Browsing the Model Catalog

Discover all available models and their configuration options:

```python
"""Explore available models and their config fields."""

from roohai.pipeline import register_all_models
from roohai.registry import registry

register_all_models()
catalog = registry.catalog()

for model_type, models in catalog.items():
    print(f"\n{'=' * 40}")
    print(f"  {model_type.upper()} Models")
    print(f"{'=' * 40}")
    for model in models:
        meta = model.get("meta", {})
        print(f"\n  {meta.get('display_name', model['name'])}")
        print(f"  Registry name: {model['name']}")
        print(f"  Kind: {meta.get('kind', 'unknown')}")
        print(f"  Dependencies: {', '.join(meta.get('pip_dependencies', []))}")

        fields = meta.get("config_fields", [])
        if fields:
            print(f"  Config fields:")
            for f in fields:
                req = " (required)" if f.get("required") else ""
                default = f" [default: {f['default']}]" if "default" in f else ""
                print(f"    - {f['name']}: {f['type']}{req}{default}")
```

---

## 14. Running the Server

```bash
# Default (localhost:8000)
roohai

# Bind to all interfaces
roohai --host 0.0.0.0

# Custom port
roohai --port 9000

# With debug logging
roohai --log-level debug
```

> **Important:** RoohAI runs a single worker by default. Each worker loads all models into memory independently, so multiple workers multiply memory usage.
