Voice Agent API Guide: How to Choose the Right Stack for Production

const ws = new WebSocket(
  'wss://api.inworld.ai/api/v1/realtime/session?key=my-session&protocol=realtime',
  { headers: { Authorization: `Basic ${process.env.INWORLD_API_KEY}` } }
);

ws.on('open', () => {
  console.log('Connected to Realtime API');
});

ws.on('message', (raw) => {
  const event = JSON.parse(raw.toString());

  if (event.type === 'session.created') {
    // Configure the voice agent
    ws.send(JSON.stringify({
      type: 'session.update',
      session: {
        type: 'realtime',
        instructions: 'You are a helpful support agent.',
        model: 'gpt-5.4',
        output_modalities: ['audio', 'text'],
        temperature: 0.7,
        audio: {
          output: {
            voice: 'Sarah',
            model: 'inworld-tts-1.5-max',
            speed: 1.0,
          },
        },
      },
    }));
  }

  if (event.type === 'session.updated') {
    console.log('Session configured. Ready to stream audio.');
  }

  if (event.type === 'response.output_audio.delta') {
    // Decode and play base64 PCM16 audio chunk
    const audio = Buffer.from(event.delta, 'base64');
    // ... write to speaker or forward to client
  }

  if (event.type === 'input_audio_buffer.speech_started') {
    // User is speaking - stop playback for barge-in
  }

  if (event.type === 'response.function_call_arguments.done') {
    // Handle tool call, send result back
    const args = JSON.parse(event.arguments);
    ws.send(JSON.stringify({
      type: 'conversation.item.create',
      item: {
        type: 'function_call_output',
        call_id: event.call_id,
        output: JSON.stringify({ result: 'your tool output here' }),
      },
    }));
    ws.send(JSON.stringify({ type: 'response.create' }));
  }
});

// Switch to Claude mid-session without reconnecting
ws.send(JSON.stringify({
  type: 'session.update',
  session: {
    model: 'anthropic/claude-opus-4-7',
  },
}));

// Or use Realtime Router for automatic selection
ws.send(JSON.stringify({
  type: 'session.update',
  session: {
    model: 'auto',
  },
}));

# Inworld Realtime TTS as a provider in Pipecat
# Works with any framework that accepts an OpenAI-compatible TTS endpoint

import aiohttp
import base64
import json

async def inworld_tts_stream(text: str, voice: str = 'Sarah'):
    """Stream TTS audio from Inworld REST API."""
    async with aiohttp.ClientSession() as session:
        resp = await session.post(
            'https://api.inworld.ai/tts/v1/voice:stream',
            headers={
                'Authorization': f'Basic {INWORLD_API_KEY}',
                'Content-Type': 'application/json',
            },
            json={
                'text': text,
                'voiceId': 'Sarah',
                'modelId': 'inworld-tts-1.5-max',
                'audioConfig': {
                    'audioEncoding': 'PCM',
                    'sampleRateHertz': 24000,
                },
            },
        )
        async for line in resp.content:
            if line.strip():
                chunk = json.loads(line)
                audio = base64.b64decode(
                    chunk['result']['audioContent']
                )
                yield audio