Realtime voice and avatar conversations over WebRTC

Browser                         Application server                  Telnyx
   │ 1. create offer (mic, data channel,                               │
   │    optional video), gather ICE                                    │
   │── 2. SDP offer ──────────────────▶│                               │
   │                                   │── 3. POST /calls (API key) ──▶│
   │                                   │◀─ 4. 201 SDP answer ──────────│
   │◀─ 5. SDP answer ──────────────────│                               │
   │══ 6. media + oai-events data channel, relayed through Telnyx TURN ═│

const pc = new RTCPeerConnection({
  iceServers: [
    {
      urls: [
        "turn:turn.telnyx.com:3478?transport=udp",
        "turn:turn.telnyx.com:3478?transport=tcp",
      ],
      username: TURN_USERNAME,
      credential: TURN_CREDENTIAL,
    },
  ],
  iceTransportPolicy: "relay",
});

const mic = await navigator.mediaDevices.getUserMedia({
  audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
});
pc.addTrack(mic.getAudioTracks()[0], mic);

// Receive the avatar's video. Omit for a voice-only call.
pc.addTransceiver("video", { direction: "recvonly" });

const events = pc.createDataChannel("oai-events");
events.onmessage = (message) => handleEvent(JSON.parse(message.data));

// One stream per element: the audio and video tracks can share a remote
// stream, and the video element must not play the audio a second time.
pc.ontrack = ({ track }) => {
  const element = track.kind === "video" ? avatarVideo : assistantAudio;
  element.srcObject = new MediaStream([track]);
};

const iceGatheringComplete = (pc) =>
  new Promise((resolve) => {
    if (pc.iceGatheringState === "complete") return resolve();
    pc.addEventListener("icegatheringstatechange", () => {
      if (pc.iceGatheringState === "complete") resolve();
    });
  });

try {
  await pc.setLocalDescription(await pc.createOffer());
  await iceGatheringComplete(pc);

  const response = await fetch("/session", {
    method: "POST",
    headers: { "Content-Type": "application/sdp" },
    body: pc.localDescription.sdp,
  });
  if (!response.ok) throw new Error(await response.text());
  await pc.setRemoteDescription({ type: "answer", sdp: await response.text() });
} catch (error) {
  // Release the microphone (and camera) when the call cannot be set up.
  pc.getSenders().forEach((sender) => sender.track?.stop());
  pc.close();
  throw error;
}

POST https://api.telnyx.com/v2/ai/assistants/{assistant_id}/calls
Authorization: Bearer YOUR_TELNYX_API_KEY

{
  "assistant": {
    "dynamic_variables": { "customer_name": "Ada", "plan": "premium" }
  }
}

// Node.js (Express): relay the browser's offer to Telnyx.
// requireUser is the application's own authentication middleware.
app.post("/session", requireUser, express.text({ type: "application/sdp" }), async (req, res) => {
  const form = new FormData();
  form.set("sdp", req.body);
  form.set("session", JSON.stringify({ assistant: { dynamic_variables: { customer_name: "Ada" } } }));

  const telnyx = await fetch(
    `https://api.telnyx.com/v2/ai/assistants/${process.env.ASSISTANT_ID}/calls`,
    { method: "POST", headers: { Authorization: `Bearer ${process.env.TELNYX_API_KEY}` }, body: form },
  );

  res.status(telnyx.status).type(telnyx.headers.get("content-type") ?? "text/plain").send(await telnyx.text());
});

{ "type": "avatar.updated", "state": "active" }

const camera = await navigator.mediaDevices.getUserMedia({ video: true });
pc.addTrack(camera.getVideoTracks()[0], camera);

pc.addTransceiver("video", { direction: "recvonly" });
await pc.setLocalDescription(await pc.createOffer());
await iceGatheringComplete(pc);
events.send(JSON.stringify({ type: "webrtc.offer", sdp: pc.localDescription.sdp }));

// In the data channel's message handler:
if (event.type === "webrtc.answer") {
  await pc.setRemoteDescription({ type: "answer", sdp: event.sdp });
}

// Stop every local track (microphone and, if sent, camera), then close.
pc.getSenders().forEach((sender) => sender.track?.stop());
pc.close();