Realtime voice conversations over WebSocket

import WebSocket from "ws";

const ASSISTANT_ID = "assistant-0f4e8b2a";
const url = `wss://api.telnyx.com/v2/ai/assistants/${ASSISTANT_ID}/conversation?input_sample_rate=16000`;

const ws = new WebSocket(url, {
  headers: { Authorization: `Bearer ${process.env.TELNYX_API_KEY}` },
});

ws.on("open", () => console.log("Connected"));

import asyncio
import os
import websockets

ASSISTANT_ID = "assistant-0f4e8b2a"
URL = f"wss://api.telnyx.com/v2/ai/assistants/{ASSISTANT_ID}/conversation?input_sample_rate=16000"

async def main():
    headers = {"Authorization": f"Bearer {os.environ['TELNYX_API_KEY']}"}
    async with websockets.connect(URL, additional_headers=headers) as ws:
        print("Connected")

asyncio.run(main())

ws.on("open", () => {
  ws.send(
    JSON.stringify({
      type: "session.update",
      session: {
        assistant: {
          dynamic_variables: { customer_name: "Ada", account_tier: "pro" },
        },
      },
    })
  );
});

async with websockets.connect(URL, additional_headers=headers) as ws:
    await ws.send(json.dumps({
        "type": "session.update",
        "session": {
            "assistant": {
                "dynamic_variables": {"customer_name": "Ada", "account_tier": "pro"},
            },
        },
    }))

import fs from "node:fs";

const SAMPLE_RATE = 16000;
const CHUNK_MS = 100;
const CHUNK_BYTES = (SAMPLE_RATE * 2 * CHUNK_MS) / 1000; // 16-bit mono

// speech.raw is raw little-endian PCM16 at 16 kHz
const pcm = fs.readFileSync("speech.raw");
let offset = 0;

const timer = setInterval(() => {
  if (offset >= pcm.length) {
    clearInterval(timer);
    return;
  }
  const chunk = pcm.subarray(offset, offset + CHUNK_BYTES);
  offset += CHUNK_BYTES;
  ws.send(
    JSON.stringify({
      type: "input_audio_buffer.append",
      audio: chunk.toString("base64"),
    })
  );
}, CHUNK_MS);

import base64
import json

SAMPLE_RATE = 16000
CHUNK_MS = 100
CHUNK_BYTES = SAMPLE_RATE * 2 * CHUNK_MS // 1000  # 16-bit mono

async def stream_audio(ws):
    # speech.raw is raw little-endian PCM16 at 16 kHz
    with open("speech.raw", "rb") as f:
        while chunk := f.read(CHUNK_BYTES):
            await ws.send(json.dumps({
                "type": "input_audio_buffer.append",
                "audio": base64.b64encode(chunk).decode(),
            }))
            await asyncio.sleep(CHUNK_MS / 1000)  # real-time pacing

let outputRate = 24000;
const playback = [];

ws.on("message", async (raw) => {
  const event = JSON.parse(raw.toString());

  switch (event.type) {
    case "session.created":
      outputRate = event.session.audio.output.format.rate;
      break;

    case "conversation.item.input_audio_transcription.completed":
      console.log(`You said: ${event.transcript}`);
      break;

    case "response.output_audio.delta":
      // Feed to your audio output as it arrives for lowest latency
      playback.push(Buffer.from(event.delta, "base64"));
      break;

    case "response.output_audio_transcript.delta":
      process.stdout.write(event.delta);
      break;

    case "response.done":
      console.log(`\n[turn ${event.response.id}: ${event.response.status}]`);
      break;

    case "error":
      console.error(`${event.error.code}: ${event.error.message}`);
      break;
  }
});

import base64
import json

async def handle_events(ws):
    output_rate = 24000
    playback = bytearray()

    async for raw in ws:
        event = json.loads(raw)
        match event["type"]:
            case "session.created":
                output_rate = event["session"]["audio"]["output"]["format"]["rate"]

            case "conversation.item.input_audio_transcription.completed":
                print(f"You said: {event['transcript']}")

            case "response.output_audio.delta":
                # Feed to your audio output as it arrives for lowest latency
                playback.extend(base64.b64decode(event["delta"]))

            case "response.output_audio_transcript.delta":
                print(event["delta"], end="", flush=True)

            case "response.done":
                print(f"\n[turn {event['response']['id']}: {event['response']['status']}]")

            case "error":
                print(f"{event['error']['code']}: {event['error']['message']}")

case "conversation.item.created": {
  const item = event.item;
  if (item.type === "function_call") {
    const args = JSON.parse(item.arguments);
    const result = await runTool(item.name, args); // your implementation
    ws.send(
      JSON.stringify({
        type: "conversation.item.create",
        item: {
          type: "function_call_output",
          call_id: item.call_id,
          output: JSON.stringify(result),
        },
      })
    );
  }
  break;
}

case "conversation.item.created":
    item = event["item"]
    if item["type"] == "function_call":
        args = json.loads(item["arguments"])
        result = await run_tool(item["name"], args)  # your implementation
        await ws.send(json.dumps({
            "type": "conversation.item.create",
            "item": {
                "type": "function_call_output",
                "call_id": item["call_id"],
                "output": json.dumps(result),
            },
        }))