Voice AI for AI Companions: How to Build Expressive, Low-Latency Voice Into Consumer Apps at Scale

const WebSocket = require('ws');

const ws = new WebSocket(
  `wss://api.inworld.ai/api/v1/realtime/session?key=companion-${Date.now()}&protocol=realtime`,
  { headers: { Authorization: `Basic ${process.env.INWORLD_API_KEY}` } }
);

ws.on('open', () => console.log('Connected'));

ws.on('message', (raw) => {
  const msg = JSON.parse(raw.toString());

  if (msg.type === 'session.created') {
    // Configure the companion session
    ws.send(JSON.stringify({
      type: 'session.update',
      session: {
        instructions: 'You are Luna, a warm and curious companion. You remember previous conversations and ask thoughtful follow-up questions. Use [happy] and [laughing] tags when the mood is light.',
        model: 'anthropic/claude-sonnet-4-6',
        audio: {
          input: {
            transcription: { model: 'inworld/inworld-stt-1' },
            turn_detection: {
              type: 'semantic_vad',
              eagerness: 'medium',
              interrupt_response: true
            }
          },
          output: {
            voice: 'Sarah',
            model: 'inworld-tts-1.5-max'
          }
        }
      }
    }));
  }

  if (msg.type === 'response.output_audio.delta') {
    // Play audio chunk to the user
    playAudio(msg.delta); // base64-encoded PCM16
  }

  if (msg.type === 'response.output_audio_transcript.delta') {
    // Display live captions
    updateCaptions(msg.delta);
  }
});

// Stream microphone audio to the companion
function sendMicrophoneChunk(audioChunk) {
  ws.send(JSON.stringify({
    type: 'input_audio_buffer.append',
    audio: audioChunk // base64-encoded PCM16 from mic
  }));
}

import requests
import base64
import os

INWORLD_API_KEY = os.environ['INWORLD_API_KEY']

# Basic companion TTS with emotion markup
response = requests.post(
    'https://api.inworld.ai/tts/v1/voice',
    headers={
        'Authorization': f'Basic {INWORLD_API_KEY}',
        'Content-Type': 'application/json'
    },
    json={
        'text': '[happy] Hey! I was just thinking about what you said yesterday about that hiking trail. Did you end up going?',
        'voiceId': 'Sarah',
        'modelId': 'inworld-tts-1.5-max',
        'audioConfig': {
            'audioEncoding': 'MP3',
            'sampleRateHertz': 24000
        }
    },
    timeout=30
)
response.raise_for_status()

audio_bytes = base64.b64decode(response.json()['audioContent'])
with open('companion_greeting.mp3', 'wb') as f:
    f.write(audio_bytes)

print(f'Generated {len(audio_bytes)} bytes of audio')

import requests
import base64
import os

INWORLD_API_KEY = os.environ['INWORLD_API_KEY']

# Step 1: Clone a voice from 5-15 seconds of reference audio
with open('character_voice_sample.wav', 'rb') as f:
    audio_b64 = base64.b64encode(f.read()).decode('utf-8')

clone_response = requests.post(
    'https://api.inworld.ai/voices/v1/voices:clone',
    headers={
        'Authorization': f'Basic {INWORLD_API_KEY}',
        'Content-Type': 'application/json'
    },
    json={
        'displayName': 'CompanionVoice',
        'langCode': 'EN_US',
        'voiceSamples': [{'audioData': audio_b64}]
    },
    timeout=60
)
clone_response.raise_for_status()

cloned_voice_id = clone_response.json()['voice']['voiceId']
print(f'Cloned voice ID: {cloned_voice_id}')

# Step 2: Generate speech in the cloned voice
response = requests.post(
    'https://api.inworld.ai/tts/v1/voice',
    headers={
        'Authorization': f'Basic {INWORLD_API_KEY}',
        'Content-Type': 'application/json'
    },
    json={
        'text': '[whispering] I have something important to tell you...',
        'voiceId': cloned_voice_id,
        'modelId': 'inworld-tts-1.5-max',
        'audioConfig': {
            'audioEncoding': 'MP3',
            'sampleRateHertz': 24000
        }
    },
    timeout=30
)
response.raise_for_status()

audio_bytes = base64.b64decode(response.json()['audioContent'])
with open('cloned_companion_voice.mp3', 'wb') as f:
    f.write(audio_bytes)

import requests
import base64
import json
import os

INWORLD_API_KEY = os.environ['INWORLD_API_KEY']

# Streaming TTS for low-latency companion responses
response = requests.post(
    'https://api.inworld.ai/tts/v1/voice:stream',
    headers={
        'Authorization': f'Basic {INWORLD_API_KEY}',
        'Content-Type': 'application/json'
    },
    json={
        'text': '[happy] That sounds amazing! Tell me more about what happened after you got to the summit. I bet the view was incredible.',
        'voiceId': 'Sarah',
        'modelId': 'inworld-tts-1.5-max',
        'audioConfig': {
            'audioEncoding': 'MP3',
            'sampleRateHertz': 24000
        }
    },
    stream=True,
    timeout=30
)
response.raise_for_status()

audio_chunks = []
for line in response.iter_lines():
    if line:
        chunk = json.loads(line)
        audio_data = base64.b64decode(chunk['result']['audioContent'])
        audio_chunks.append(audio_data)
        # Play each chunk immediately for lowest latency
        play_audio_chunk(audio_data)

print(f'Streamed {len(audio_chunks)} chunks')