104 lines
3.5 KiB
TypeScript
104 lines
3.5 KiB
TypeScript
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { ServerWebSocket } from "bun";
|
|
import { LIVE_TUNING, LiveTranscriber, type LiveUpdate } from "../lib/live";
|
|
import { sidecar } from "./transcribe";
|
|
import { getSessionUser } from "./auth";
|
|
import { getFasterWhisperModel } from "../lib/models";
|
|
import type { WhisperTask } from "../worker/protocol";
|
|
|
|
interface LiveSessionData {
|
|
user: any;
|
|
model: string;
|
|
language?: string;
|
|
task?: WhisperTask;
|
|
tempDir: string;
|
|
}
|
|
|
|
export const liveWebSocket = {
|
|
open(ws: ServerWebSocket<LiveSessionData>) {
|
|
// Initialize the transcriber for this connection.
|
|
const transcriber = new LiveTranscriber({
|
|
...LIVE_TUNING,
|
|
transcribe: async (audio: Float32Array) => {
|
|
const audioPath = join(
|
|
ws.data.tempDir,
|
|
`whisper-live-${crypto.randomUUID()}.f32`
|
|
);
|
|
const sink = Bun.file(audioPath).writer();
|
|
// audio.buffer is an ArrayBuffer, we need a Uint8Array view
|
|
sink.write(new Uint8Array(audio.buffer as ArrayBuffer, audio.byteOffset, audio.byteLength));
|
|
await sink.end();
|
|
|
|
return new Promise((resolve, reject) => {
|
|
let text = "";
|
|
let chunks: any[] = [];
|
|
|
|
sidecar.transcribe(
|
|
{
|
|
model: ws.data.model,
|
|
audioPath,
|
|
language: ws.data.language,
|
|
task: ws.data.task,
|
|
},
|
|
(segment) => {
|
|
text += segment.text;
|
|
chunks.push({
|
|
text: segment.text,
|
|
timestamp: [segment.start, segment.end],
|
|
});
|
|
}
|
|
)
|
|
.then(() => resolve({ text: text.trim(), chunks }))
|
|
.catch(reject)
|
|
.finally(() => Bun.file(audioPath).unlink().catch(() => {}));
|
|
});
|
|
},
|
|
onUpdate: (update: LiveUpdate) => {
|
|
ws.send(JSON.stringify({ type: "update", update }));
|
|
},
|
|
});
|
|
|
|
// Store it on the websocket
|
|
(ws as any).transcriber = transcriber;
|
|
},
|
|
|
|
message(ws: ServerWebSocket<LiveSessionData>, message: string | Buffer | Uint8Array) {
|
|
if (typeof message === "string") {
|
|
let parsed: any;
|
|
try {
|
|
parsed = JSON.parse(message);
|
|
} catch {
|
|
// Not a frame we understand — ignore rather than crash the handler
|
|
// (which would otherwise take the whole connection down for every
|
|
// client sharing this Bun.serve process, not just this session).
|
|
return;
|
|
}
|
|
if (parsed.type === "stop") {
|
|
const transcriber = (ws as any).transcriber as LiveTranscriber;
|
|
transcriber.stop().then((result) => {
|
|
ws.send(JSON.stringify({ type: "result", result }));
|
|
ws.close();
|
|
}).catch((err) => {
|
|
ws.send(JSON.stringify({ type: "error", error: err instanceof Error ? err.message : String(err) }));
|
|
ws.close();
|
|
});
|
|
}
|
|
} else {
|
|
// Audio chunk arrived
|
|
const transcriber = (ws as any).transcriber as LiveTranscriber;
|
|
// Convert raw bytes to Float32Array (16kHz mono)
|
|
const buffer = message instanceof Buffer ? message.buffer : message.buffer;
|
|
const byteOffset = message instanceof Buffer ? message.byteOffset : (message as Uint8Array).byteOffset;
|
|
const byteLength = message.byteLength;
|
|
const audio = new Float32Array(buffer, byteOffset, byteLength / 4);
|
|
transcriber.push(audio);
|
|
}
|
|
},
|
|
|
|
close(ws: ServerWebSocket<LiveSessionData>) {
|
|
const transcriber = (ws as any).transcriber as LiveTranscriber;
|
|
transcriber?.stop().catch(() => {});
|
|
},
|
|
};
|