Gemini 2.5 Flash Native Audio (Live)

{
  "setup": {
    "generationConfig": {
      "responseModalities": [
        "AUDIO"
      ],
      "speechConfig": {
        "voiceConfig": {
          "prebuiltVoiceConfig": {
            "voiceName": "Puck"
          }
        }
      }
    },
    "inputAudioTranscription": {},
    "outputAudioTranscription": {}
  }
}

// Node.js 22+. speech.pcm is 16 kHz, 16-bit, mono PCM; make one with:
// ffmpeg -i speech.mp3 -ar 16000 -ac 1 -f s16le speech.pcm
import { readFile, writeFile } from "node:fs/promises";

const socket = new WebSocket("wss://api.ohmygpt.com/v1/gemini/live?model=google/gemini-2.5-flash-native-audio", ["realtime", "openai-insecure-api-key." + process.env.OHMYGPT_API_KEY]);
const voice = [];
let saved = false;
const finish = async () => {
  if (saved) return;
  saved = true;
  // Play it with: ffplay -f s16le -ar 24000 -ac 1 reply.pcm
  await writeFile("reply.pcm", Buffer.concat(voice));
  socket.close();
};
socket.onopen = () => {
  socket.send(JSON.stringify({
    setup: {
      generationConfig: {
        responseModalities: ["AUDIO"],
        speechConfig: {
          voiceConfig: { prebuiltVoiceConfig: { voiceName: "Puck" } },
        },
      },
      inputAudioTranscription: {},
      outputAudioTranscription: {},
    },
  }));
};
socket.onmessage = async (event) => {
  const data = JSON.parse(typeof event.data === "string" ? event.data : await event.data.text());
  const content = data.serverContent;
  for (const part of content?.modelTurn?.parts ?? []) {
    if (part.inlineData) voice.push(Buffer.from(part.inlineData.data, "base64"));
  }
  if (content?.outputTranscription?.text) process.stdout.write(content.outputTranscription.text);
  if (content?.turnComplete) await finish();
  if (!data.setupComplete) return;
  const audio = await readFile("speech.pcm");
  // Send at real-time speed, 100 ms per chunk, as a microphone would
  for (let offset = 0; offset < audio.length; offset += 3200) {
    socket.send(JSON.stringify({ realtimeInput: { audio: { data: audio.subarray(offset, offset + 3200).toString("base64"), mimeType: "audio/pcm;rate=16000" } } }));
    await new Promise((resolve) => setTimeout(resolve, 100));
  }
  socket.send(JSON.stringify({ realtimeInput: { audioStreamEnd: true } }));
  setTimeout(finish, 8000);
};
socket.onclose = (event) => event.reason && console.error(event.reason);