Agent completed work jubii
This commit is contained in:
Lucas Winther
+162
@@ -0,0 +1,162 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import "./index.css";
|
||||
|
||||
import { FileDropzone } from "./components/FileDropzone";
|
||||
import { ModelLoadProgress } from "./components/ModelLoadProgress";
|
||||
import { TranscriptView } from "./components/TranscriptView";
|
||||
import { decodeAudioFile } from "./lib/audio";
|
||||
import { DEFAULT_DEVICE, DEFAULT_MODEL_ID } from "./lib/models";
|
||||
import {
|
||||
applyLoadProgress,
|
||||
initialLoadProgress,
|
||||
isWorkerToUiMessage,
|
||||
type TranscriptChunk,
|
||||
type UiToWorkerMessage,
|
||||
} from "./worker/protocol";
|
||||
|
||||
interface TranscriptResult {
|
||||
text: string;
|
||||
chunks: TranscriptChunk[];
|
||||
}
|
||||
|
||||
export function App() {
|
||||
const workerRef = useRef<Worker | null>(null);
|
||||
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [decoding, setDecoding] = useState(false);
|
||||
const [loadStarted, setLoadStarted] = useState(false);
|
||||
const [loadProgress, setLoadProgress] = useState(initialLoadProgress);
|
||||
const [partialText, setPartialText] = useState("");
|
||||
const [result, setResult] = useState<TranscriptResult | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [fileName, setFileName] = useState<string | null>(null);
|
||||
|
||||
// Terminate the worker when the app unmounts (also keeps HMR tidy).
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
workerRef.current?.terminate();
|
||||
workerRef.current = null;
|
||||
};
|
||||
}, []);
|
||||
|
||||
function ensureWorker(): Worker {
|
||||
if (workerRef.current) return workerRef.current;
|
||||
// Inference always runs in a dedicated Web Worker (D-003), never on the
|
||||
// main thread (Architecture.md §5). The worker script is bundled by the
|
||||
// server at startup and served at a stable URL (see src/index.ts) because
|
||||
// Bun's bundler does not follow `new Worker(new URL(...))` graphs yet.
|
||||
const worker = new Worker("/inference.worker.js", { type: "module" });
|
||||
worker.addEventListener("message", (event: MessageEvent<unknown>) => {
|
||||
const message = event.data;
|
||||
if (!isWorkerToUiMessage(message)) return;
|
||||
switch (message.type) {
|
||||
case "load-progress":
|
||||
setLoadProgress((state) => applyLoadProgress(state, message));
|
||||
break;
|
||||
case "partial":
|
||||
setPartialText((text) => text + message.text);
|
||||
break;
|
||||
case "result":
|
||||
setResult({ text: message.text, chunks: message.chunks });
|
||||
setBusy(false);
|
||||
break;
|
||||
case "error":
|
||||
setError(message.message);
|
||||
setBusy(false);
|
||||
break;
|
||||
}
|
||||
});
|
||||
worker.addEventListener("error", (event) => {
|
||||
setError(event.message || "The inference worker crashed unexpectedly.");
|
||||
setBusy(false);
|
||||
});
|
||||
workerRef.current = worker;
|
||||
return worker;
|
||||
}
|
||||
|
||||
function postToWorker(worker: Worker, message: UiToWorkerMessage, transfer?: Transferable[]) {
|
||||
worker.postMessage(message, transfer ?? []);
|
||||
}
|
||||
|
||||
async function handleFile(file: File) {
|
||||
if (busy) return;
|
||||
setBusy(true);
|
||||
setError(null);
|
||||
setPartialText("");
|
||||
setResult(null);
|
||||
setFileName(file.name);
|
||||
|
||||
// Kick off the (idempotent) model load immediately so the download runs
|
||||
// while we decode the audio locally.
|
||||
const worker = ensureWorker();
|
||||
postToWorker(worker, {
|
||||
type: "load",
|
||||
modelId: DEFAULT_MODEL_ID,
|
||||
device: DEFAULT_DEVICE,
|
||||
});
|
||||
setLoadStarted(true);
|
||||
|
||||
setDecoding(true);
|
||||
try {
|
||||
// Decoded entirely on-device via Web Audio (D-012); the audio never
|
||||
// touches the network (Architecture.md §5).
|
||||
const audio = await decodeAudioFile(file);
|
||||
// Transfer the buffer to the worker — zero copy.
|
||||
postToWorker(worker, { type: "transcribe", audio }, [audio.buffer as ArrayBuffer]);
|
||||
} catch (decodeError) {
|
||||
setError(decodeError instanceof Error ? decodeError.message : String(decodeError));
|
||||
setBusy(false);
|
||||
} finally {
|
||||
setDecoding(false);
|
||||
}
|
||||
}
|
||||
|
||||
const modelReady = loadProgress.ready;
|
||||
const showModelProgress = loadStarted && !modelReady && !error;
|
||||
const transcribing = busy && !decoding && modelReady;
|
||||
|
||||
return (
|
||||
<main className="app">
|
||||
<header className="app-header">
|
||||
<h1>Whisper Web</h1>
|
||||
<p className="tagline">
|
||||
Speech-to-text that runs entirely in your browser. Your audio never
|
||||
leaves this device.
|
||||
</p>
|
||||
</header>
|
||||
|
||||
<FileDropzone onFile={handleFile} disabled={busy} />
|
||||
|
||||
{fileName && (
|
||||
<p className="file-name">
|
||||
File: <strong>{fileName}</strong>
|
||||
{decoding && " — decoding audio…"}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{error && (
|
||||
<p className="error" role="alert">
|
||||
{error}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{showModelProgress && (
|
||||
<ModelLoadProgress modelId={DEFAULT_MODEL_ID} progress={loadProgress} />
|
||||
)}
|
||||
|
||||
<TranscriptView
|
||||
partialText={partialText}
|
||||
result={result}
|
||||
transcribing={transcribing}
|
||||
/>
|
||||
|
||||
<footer className="app-footer">
|
||||
<p>
|
||||
Model: <code>{DEFAULT_MODEL_ID}</code> · Engine: WASM (on-device)
|
||||
</p>
|
||||
</footer>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
|
||||
export default App;
|
||||
@@ -0,0 +1,68 @@
|
||||
import { useRef, useState, type DragEvent, type KeyboardEvent } from "react";
|
||||
|
||||
interface FileDropzoneProps {
|
||||
onFile: (file: File) => void;
|
||||
disabled?: boolean;
|
||||
}
|
||||
|
||||
/** Drag-and-drop target + file picker for audio/video uploads. */
|
||||
export function FileDropzone({ onFile, disabled = false }: FileDropzoneProps) {
|
||||
const inputRef = useRef<HTMLInputElement>(null);
|
||||
const [dragOver, setDragOver] = useState(false);
|
||||
|
||||
function submit(file: File | undefined | null) {
|
||||
if (file && !disabled) onFile(file);
|
||||
}
|
||||
|
||||
function onDrop(event: DragEvent<HTMLDivElement>) {
|
||||
event.preventDefault();
|
||||
setDragOver(false);
|
||||
submit(event.dataTransfer.files.item(0));
|
||||
}
|
||||
|
||||
function onDragOver(event: DragEvent<HTMLDivElement>) {
|
||||
event.preventDefault(); // required to allow dropping
|
||||
if (!disabled) setDragOver(true);
|
||||
}
|
||||
|
||||
function openPicker() {
|
||||
if (!disabled) inputRef.current?.click();
|
||||
}
|
||||
|
||||
function onKeyDown(event: KeyboardEvent<HTMLDivElement>) {
|
||||
if (event.key === "Enter" || event.key === " ") {
|
||||
event.preventDefault();
|
||||
openPicker();
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
className={`dropzone${dragOver ? " dropzone-active" : ""}${disabled ? " dropzone-disabled" : ""}`}
|
||||
role="button"
|
||||
tabIndex={disabled ? -1 : 0}
|
||||
aria-disabled={disabled}
|
||||
aria-label="Upload an audio or video file"
|
||||
onClick={openPicker}
|
||||
onKeyDown={onKeyDown}
|
||||
onDrop={onDrop}
|
||||
onDragOver={onDragOver}
|
||||
onDragLeave={() => setDragOver(false)}
|
||||
>
|
||||
<input
|
||||
ref={inputRef}
|
||||
type="file"
|
||||
accept="audio/*,video/*"
|
||||
hidden
|
||||
onChange={(event) => {
|
||||
submit(event.target.files?.item(0));
|
||||
event.target.value = ""; // allow re-selecting the same file
|
||||
}}
|
||||
/>
|
||||
<p className="dropzone-title">
|
||||
{disabled ? "Working…" : "Drop an audio or video file here"}
|
||||
</p>
|
||||
<p className="dropzone-hint">or click to choose a file</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import type { LoadProgressState } from "../worker/protocol";
|
||||
|
||||
interface ModelLoadProgressProps {
|
||||
modelId: string;
|
||||
progress: LoadProgressState;
|
||||
}
|
||||
|
||||
function formatBytes(bytes: number): string {
|
||||
if (bytes >= 1_000_000) return `${(bytes / 1_000_000).toFixed(1)} MB`;
|
||||
if (bytes >= 1_000) return `${(bytes / 1_000).toFixed(0)} kB`;
|
||||
return `${bytes} B`;
|
||||
}
|
||||
|
||||
/** Per-file model download progress (bytes and %), driven by load-progress messages. */
|
||||
export function ModelLoadProgress({ modelId, progress }: ModelLoadProgressProps) {
|
||||
const files = Object.values(progress.files);
|
||||
return (
|
||||
<section className="model-progress" aria-label="Model download progress">
|
||||
<h2>
|
||||
Downloading model <code>{modelId}</code>
|
||||
</h2>
|
||||
<p className="model-progress-hint">
|
||||
This happens once — the weights are cached in your browser afterwards.
|
||||
</p>
|
||||
{files.length === 0 && <p className="model-progress-hint">Preparing…</p>}
|
||||
<ul>
|
||||
{files.map((file) => {
|
||||
const percent =
|
||||
file.done ? 100 : file.total > 0 ? (file.loaded / file.total) * 100 : 0;
|
||||
return (
|
||||
<li key={file.file} className="model-progress-file">
|
||||
<div className="model-progress-label">
|
||||
<span className="model-progress-name">{file.file}</span>
|
||||
<span className="model-progress-bytes">
|
||||
{file.total > 0
|
||||
? `${formatBytes(file.loaded)} / ${formatBytes(file.total)} (${percent.toFixed(0)}%)`
|
||||
: file.done
|
||||
? "done"
|
||||
: "…"}
|
||||
</span>
|
||||
</div>
|
||||
<progress max={100} value={percent} />
|
||||
</li>
|
||||
);
|
||||
})}
|
||||
</ul>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import type { TranscriptChunk } from "../worker/protocol";
|
||||
|
||||
interface TranscriptViewProps {
|
||||
/** Text streamed so far (accumulated partial messages). */
|
||||
partialText: string;
|
||||
/** Final transcript, once available. */
|
||||
result: { text: string; chunks: TranscriptChunk[] } | null;
|
||||
/** True while a transcription is in flight. */
|
||||
transcribing: boolean;
|
||||
}
|
||||
|
||||
function formatTimestamp(seconds: number): string {
|
||||
const m = Math.floor(seconds / 60);
|
||||
const s = seconds - m * 60;
|
||||
return `${String(m).padStart(2, "0")}:${s.toFixed(1).padStart(4, "0")}`;
|
||||
}
|
||||
|
||||
/** Streaming transcript: live partials while running, final text + copy button after. */
|
||||
export function TranscriptView({ partialText, result, transcribing }: TranscriptViewProps) {
|
||||
const [copied, setCopied] = useState(false);
|
||||
const text = result?.text ?? partialText;
|
||||
|
||||
useEffect(() => {
|
||||
if (!copied) return;
|
||||
const timer = setTimeout(() => setCopied(false), 1500);
|
||||
return () => clearTimeout(timer);
|
||||
}, [copied]);
|
||||
|
||||
if (!transcribing && !text) return null;
|
||||
|
||||
async function copy() {
|
||||
if (!text) return;
|
||||
await navigator.clipboard.writeText(text);
|
||||
setCopied(true);
|
||||
}
|
||||
|
||||
return (
|
||||
<section className="transcript">
|
||||
<div className="transcript-header">
|
||||
<h2>{result ? "Transcript" : "Transcribing…"}</h2>
|
||||
<button
|
||||
type="button"
|
||||
className="copy-button"
|
||||
onClick={copy}
|
||||
disabled={!text}
|
||||
>
|
||||
{copied ? "Copied!" : "Copy"}
|
||||
</button>
|
||||
</div>
|
||||
<p className="transcript-text" aria-live="polite">
|
||||
{text || "Listening for the first words…"}
|
||||
{transcribing && <span className="cursor" aria-hidden="true" />}
|
||||
</p>
|
||||
{result && result.chunks.length > 0 && (
|
||||
<details className="transcript-chunks">
|
||||
<summary>Timestamps ({result.chunks.length} segments)</summary>
|
||||
<ol>
|
||||
{result.chunks.map((chunk, i) => (
|
||||
<li key={i}>
|
||||
<span className="chunk-time">
|
||||
{formatTimestamp(chunk.timestamp[0])}
|
||||
{chunk.timestamp[1] !== null &&
|
||||
` – ${formatTimestamp(chunk.timestamp[1])}`}
|
||||
</span>
|
||||
<span className="chunk-text">{chunk.text}</span>
|
||||
</li>
|
||||
))}
|
||||
</ol>
|
||||
</details>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
/**
|
||||
* This file is the entry point for the React app, it sets up the root
|
||||
* element and renders the App component to the DOM.
|
||||
*
|
||||
* It is included in `src/index.html`.
|
||||
*/
|
||||
|
||||
import { StrictMode } from "react";
|
||||
import { createRoot } from "react-dom/client";
|
||||
import { App } from "./App.tsx";
|
||||
|
||||
const elem = document.getElementById("root")!;
|
||||
const app = (
|
||||
<StrictMode>
|
||||
<App />
|
||||
</StrictMode>
|
||||
);
|
||||
|
||||
// https://bun.com/docs/bundler/hot-reloading#import-meta-hot-data
|
||||
(import.meta.hot.data.root ??= createRoot(elem)).render(app);
|
||||
+258
@@ -0,0 +1,258 @@
|
||||
:root {
|
||||
font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
|
||||
line-height: 1.5;
|
||||
color-scheme: light dark;
|
||||
color: #e8e8e8;
|
||||
background-color: #16181d;
|
||||
--panel: #1f232b;
|
||||
--border: #343a46;
|
||||
--accent: #7aa2f7;
|
||||
--muted: #9aa3b2;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
min-width: 320px;
|
||||
min-height: 100vh;
|
||||
}
|
||||
|
||||
.app {
|
||||
max-width: 720px;
|
||||
margin: 0 auto;
|
||||
padding: 2.5rem 1.25rem 3rem;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 1.5rem;
|
||||
}
|
||||
|
||||
.app-header h1 {
|
||||
margin: 0 0 0.25rem;
|
||||
font-size: 2.2rem;
|
||||
line-height: 1.15;
|
||||
}
|
||||
|
||||
.tagline {
|
||||
margin: 0;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
/* --- FileDropzone --- */
|
||||
|
||||
.dropzone {
|
||||
border: 2px dashed var(--border);
|
||||
border-radius: 14px;
|
||||
background: var(--panel);
|
||||
padding: 2.5rem 1.5rem;
|
||||
text-align: center;
|
||||
cursor: pointer;
|
||||
transition: border-color 0.15s, background-color 0.15s;
|
||||
}
|
||||
|
||||
.dropzone:hover,
|
||||
.dropzone:focus-visible {
|
||||
border-color: var(--accent);
|
||||
outline: none;
|
||||
}
|
||||
|
||||
.dropzone-active {
|
||||
border-color: var(--accent);
|
||||
background: #232a38;
|
||||
}
|
||||
|
||||
.dropzone-disabled {
|
||||
cursor: default;
|
||||
opacity: 0.6;
|
||||
}
|
||||
|
||||
.dropzone-title {
|
||||
margin: 0 0 0.25rem;
|
||||
font-size: 1.1rem;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.dropzone-hint {
|
||||
margin: 0;
|
||||
color: var(--muted);
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.file-name {
|
||||
margin: 0;
|
||||
color: var(--muted);
|
||||
font-size: 0.95rem;
|
||||
}
|
||||
|
||||
.error {
|
||||
margin: 0;
|
||||
padding: 0.75rem 1rem;
|
||||
border-radius: 10px;
|
||||
border: 1px solid #8b3a3a;
|
||||
background: #2a1c1e;
|
||||
color: #f2b8b5;
|
||||
}
|
||||
|
||||
/* --- ModelLoadProgress --- */
|
||||
|
||||
.model-progress {
|
||||
background: var(--panel);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 14px;
|
||||
padding: 1rem 1.25rem;
|
||||
}
|
||||
|
||||
.model-progress h2 {
|
||||
margin: 0 0 0.25rem;
|
||||
font-size: 1rem;
|
||||
}
|
||||
|
||||
.model-progress-hint {
|
||||
margin: 0 0 0.75rem;
|
||||
color: var(--muted);
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
.model-progress ul {
|
||||
list-style: none;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.6rem;
|
||||
}
|
||||
|
||||
.model-progress-label {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: 1rem;
|
||||
font-size: 0.85rem;
|
||||
margin-bottom: 0.2rem;
|
||||
}
|
||||
|
||||
.model-progress-name {
|
||||
font-family: ui-monospace, monospace;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.model-progress-bytes {
|
||||
color: var(--muted);
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.model-progress progress {
|
||||
width: 100%;
|
||||
height: 6px;
|
||||
accent-color: var(--accent);
|
||||
}
|
||||
|
||||
/* --- TranscriptView --- */
|
||||
|
||||
.transcript {
|
||||
background: var(--panel);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 14px;
|
||||
padding: 1rem 1.25rem;
|
||||
}
|
||||
|
||||
.transcript-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.transcript-header h2 {
|
||||
margin: 0;
|
||||
font-size: 1rem;
|
||||
}
|
||||
|
||||
.copy-button {
|
||||
border: 1px solid var(--border);
|
||||
background: transparent;
|
||||
color: inherit;
|
||||
border-radius: 8px;
|
||||
padding: 0.3rem 0.9rem;
|
||||
font-size: 0.85rem;
|
||||
cursor: pointer;
|
||||
transition: border-color 0.15s;
|
||||
}
|
||||
|
||||
.copy-button:hover:enabled {
|
||||
border-color: var(--accent);
|
||||
}
|
||||
|
||||
.copy-button:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: default;
|
||||
}
|
||||
|
||||
.transcript-text {
|
||||
margin: 0.75rem 0 0;
|
||||
white-space: pre-wrap;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.cursor {
|
||||
display: inline-block;
|
||||
width: 0.55em;
|
||||
height: 1em;
|
||||
margin-left: 2px;
|
||||
vertical-align: text-bottom;
|
||||
background: var(--accent);
|
||||
animation: blink 1s steps(2, start) infinite;
|
||||
}
|
||||
|
||||
@keyframes blink {
|
||||
to {
|
||||
visibility: hidden;
|
||||
}
|
||||
}
|
||||
|
||||
.transcript-chunks {
|
||||
margin-top: 1rem;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.transcript-chunks summary {
|
||||
cursor: pointer;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.transcript-chunks ol {
|
||||
list-style: none;
|
||||
margin: 0.5rem 0 0;
|
||||
padding: 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.3rem;
|
||||
}
|
||||
|
||||
.chunk-time {
|
||||
font-family: ui-monospace, monospace;
|
||||
color: var(--accent);
|
||||
margin-right: 0.75rem;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.app-footer {
|
||||
color: var(--muted);
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
.app-footer p {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
code {
|
||||
font-family: ui-monospace, monospace;
|
||||
background: #262b34;
|
||||
padding: 0.1em 0.35em;
|
||||
border-radius: 5px;
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion) {
|
||||
*,
|
||||
::before,
|
||||
::after {
|
||||
animation: none !important;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<link
|
||||
rel="icon"
|
||||
href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'%3E%3Ctext y='.9em' font-size='90'%3E%F0%9F%8E%99%EF%B8%8F%3C/text%3E%3C/svg%3E"
|
||||
/>
|
||||
<title>Whisper Web — on-device speech-to-text</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="frontend.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,43 @@
|
||||
import { serve } from "bun";
|
||||
import index from "./index.html";
|
||||
|
||||
// Bun's bundler does not currently follow `new Worker(...)` graphs (verified
|
||||
// with Bun 1.3: both `bun build` and the dev server leave the call untouched),
|
||||
// so the inference worker is bundled here at startup and served from a stable
|
||||
// URL. The UI spawns it with `new Worker("/inference.worker.js")`.
|
||||
const workerBuild = await Bun.build({
|
||||
entrypoints: [
|
||||
Bun.fileURLToPath(new URL("./worker/inference.worker.ts", import.meta.url)),
|
||||
],
|
||||
target: "browser",
|
||||
format: "esm",
|
||||
minify: process.env.NODE_ENV === "production",
|
||||
});
|
||||
const workerArtifact = workerBuild.outputs[0];
|
||||
if (!workerArtifact) {
|
||||
throw new Error("Failed to bundle src/worker/inference.worker.ts");
|
||||
}
|
||||
const workerScript = await workerArtifact.text();
|
||||
|
||||
// Minimal Bun.serve entry (Architecture.md §2). API routes (auth, server
|
||||
// transcription) arrive in M4/M5 — see src/server/.
|
||||
const server = serve({
|
||||
routes: {
|
||||
"/inference.worker.js": new Response(workerScript, {
|
||||
headers: { "Content-Type": "text/javascript; charset=utf-8" },
|
||||
}),
|
||||
|
||||
// Serve the bundled React app for all other routes.
|
||||
"/*": index,
|
||||
},
|
||||
|
||||
development: process.env.NODE_ENV !== "production" && {
|
||||
// Enable browser hot reloading in development
|
||||
hmr: true,
|
||||
|
||||
// Echo console logs from the browser to the server
|
||||
console: true,
|
||||
},
|
||||
});
|
||||
|
||||
console.log(`Server running at ${server.url}`);
|
||||
@@ -0,0 +1,120 @@
|
||||
/**
|
||||
* Audio decoding for the browser engine (Decisions.md D-012).
|
||||
*
|
||||
* Uploaded audio/video File → `AudioContext.decodeAudioData` → mono downmix →
|
||||
* resample to 16 kHz via `OfflineAudioContext` → `Float32Array` (transferred
|
||||
* to the inference worker, zero-copy).
|
||||
*
|
||||
* Browser-engine audio never touches the network (Architecture.md §5): all
|
||||
* work happens locally with Web Audio APIs.
|
||||
*
|
||||
* `downmixToMono` / `resampleLinear` are pure and unit-testable under
|
||||
* `bun test`; the browser path uses `OfflineAudioContext` (higher-quality
|
||||
* resampler) and only needs the pure helpers as a documented fallback.
|
||||
*
|
||||
* No ffmpeg.wasm so far — `decodeAudioData` has covered every container we
|
||||
* have tried. Note here if we hit a real gap (per D-012).
|
||||
*/
|
||||
|
||||
/** Whisper models consume 16 kHz mono PCM. */
|
||||
export const TARGET_SAMPLE_RATE = 16_000;
|
||||
|
||||
/**
|
||||
* Average N channels into one. Pure — safe to call anywhere.
|
||||
* Channels of unequal length are truncated to the shortest.
|
||||
*/
|
||||
export function downmixToMono(channels: readonly Float32Array[]): Float32Array {
|
||||
if (channels.length === 0) {
|
||||
throw new Error("downmixToMono: need at least one channel");
|
||||
}
|
||||
if (channels.length === 1) {
|
||||
return Float32Array.from(channels[0]!);
|
||||
}
|
||||
const length = Math.min(...channels.map((c) => c.length));
|
||||
const out = new Float32Array(length);
|
||||
for (let i = 0; i < length; i++) {
|
||||
let sum = 0;
|
||||
for (const channel of channels) {
|
||||
sum += channel[i]!;
|
||||
}
|
||||
out[i] = sum / channels.length;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Linear-interpolation resampler. Pure — safe to call anywhere.
|
||||
* The browser path prefers `OfflineAudioContext` (D-012); this exists as a
|
||||
* testable reference implementation and non-browser fallback.
|
||||
*/
|
||||
export function resampleLinear(
|
||||
input: Float32Array,
|
||||
fromRate: number,
|
||||
toRate: number,
|
||||
): Float32Array {
|
||||
if (!(fromRate > 0) || !(toRate > 0)) {
|
||||
throw new Error("resampleLinear: sample rates must be positive");
|
||||
}
|
||||
if (fromRate === toRate) {
|
||||
return Float32Array.from(input);
|
||||
}
|
||||
if (input.length === 0) {
|
||||
return new Float32Array(0);
|
||||
}
|
||||
const outLength = Math.max(1, Math.round((input.length * toRate) / fromRate));
|
||||
const out = new Float32Array(outLength);
|
||||
const step = fromRate / toRate;
|
||||
const last = input.length - 1;
|
||||
for (let i = 0; i < outLength; i++) {
|
||||
const position = Math.min(i * step, last);
|
||||
const index = Math.floor(position);
|
||||
const nextIndex = Math.min(index + 1, last);
|
||||
const fraction = position - index;
|
||||
out[i] = input[index]! * (1 - fraction) + input[nextIndex]! * fraction;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Downmix + resample a decoded `AudioBuffer` to 16 kHz mono using
|
||||
* `OfflineAudioContext` (D-012). Browser-only.
|
||||
*/
|
||||
export async function audioBufferToModelInput(
|
||||
buffer: AudioBuffer,
|
||||
): Promise<Float32Array> {
|
||||
if (buffer.sampleRate === TARGET_SAMPLE_RATE && buffer.numberOfChannels === 1) {
|
||||
return buffer.getChannelData(0);
|
||||
}
|
||||
const length = Math.max(1, Math.ceil(buffer.duration * TARGET_SAMPLE_RATE));
|
||||
// Single-channel destination: the OfflineAudioContext performs the mono
|
||||
// downmix and the (windowed-sinc quality) resample in one render pass.
|
||||
const offline = new OfflineAudioContext(1, length, TARGET_SAMPLE_RATE);
|
||||
const source = offline.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
source.connect(offline.destination);
|
||||
source.start(0);
|
||||
const rendered = await offline.startRendering();
|
||||
return rendered.getChannelData(0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode an uploaded audio or video file to 16 kHz mono `Float32Array`,
|
||||
* ready to transfer to the inference worker. Browser-only (Web Audio API).
|
||||
*/
|
||||
export async function decodeAudioFile(file: Blob): Promise<Float32Array> {
|
||||
const bytes = await file.arrayBuffer();
|
||||
// decodeAudioData handles most audio containers and audio-in-video (D-012).
|
||||
// A suspended AudioContext (no user gesture) can still decode.
|
||||
const context = new AudioContext();
|
||||
try {
|
||||
const decoded = await context.decodeAudioData(bytes);
|
||||
return await audioBufferToModelInput(decoded);
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
"Could not decode this file. Make sure it is an audio or video file " +
|
||||
`in a format your browser supports. (${error instanceof Error ? error.message : String(error)})`,
|
||||
);
|
||||
} finally {
|
||||
void context.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
/**
|
||||
* Engine abstraction (Decisions.md D-010).
|
||||
*
|
||||
* TODO(M5): define the `Engine` interface — `{ load(model), transcribe(audio),
|
||||
* onProgress }` — plus `BrowserEngine` (wraps the inference worker) and
|
||||
* `ServerEngine` (wraps the streamed `POST /api/transcribe` fetch, D-011).
|
||||
*
|
||||
* Until then the UI talks to the worker directly via src/worker/protocol.ts.
|
||||
*/
|
||||
|
||||
export {};
|
||||
@@ -0,0 +1,20 @@
|
||||
/**
|
||||
* Model catalog (Decisions.md D-006).
|
||||
*
|
||||
* M1 ships a single hardcoded model. The curated catalog — tiny → large,
|
||||
* English-only vs multilingual, dtype defaults per device, size estimates —
|
||||
* lands in M2.
|
||||
*
|
||||
* TODO(M2): curated `MODELS` catalog with sizes/dtypes/languages (D-006).
|
||||
*/
|
||||
|
||||
import type { InferenceDevice } from "../worker/protocol";
|
||||
|
||||
/**
|
||||
* The one model M1 supports. `onnx-community` is the recommended org for
|
||||
* Transformers.js v3+ ONNX conversions.
|
||||
*/
|
||||
export const DEFAULT_MODEL_ID = "onnx-community/whisper-tiny.en";
|
||||
|
||||
/** M1 runs WASM only; WebGPU detection arrives in M2 (D-004). */
|
||||
export const DEFAULT_DEVICE: InferenceDevice = "wasm";
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"$comment": "TODO(M3): finalize PWA manifest — icons, theme colors, installability (D-007).",
|
||||
"name": "Whisper Web",
|
||||
"short_name": "Whisper",
|
||||
"start_url": "/",
|
||||
"display": "standalone"
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
/**
|
||||
* Service worker (Decisions.md D-007) — TODO(M3).
|
||||
*
|
||||
* Plan: precache the built app shell (hashed assets emitted by `bun build`
|
||||
* into a generated manifest), cache-first for static assets, network-only for
|
||||
* `/api/*`. Model weights are NOT cached here — Transformers.js keeps them in
|
||||
* the browser Cache API itself (D-005).
|
||||
*/
|
||||
|
||||
export {};
|
||||
@@ -0,0 +1,11 @@
|
||||
/**
|
||||
* Auth — TODO(M4).
|
||||
*
|
||||
* Passport.js local strategy behind a connect-compat adapter over Bun.serve
|
||||
* (Decisions.md D-008): wrap Bun's Request in a minimal req/res shim, run the
|
||||
* passport authenticate chain, translate back to a Response. Sessions in
|
||||
* bun:sqlite with HttpOnly cookies (D-009). Routes:
|
||||
* POST /api/auth/register|login|logout, GET /api/auth/me.
|
||||
*/
|
||||
|
||||
export {};
|
||||
@@ -0,0 +1,9 @@
|
||||
/**
|
||||
* Database — TODO(M4).
|
||||
*
|
||||
* bun:sqlite (Decisions.md D-009). Tables: users (id, username, password_hash
|
||||
* via Bun.password Argon2), sessions (id, user_id, expires_at). DB path from
|
||||
* env, default ./data/app.db.
|
||||
*/
|
||||
|
||||
export {};
|
||||
@@ -0,0 +1,10 @@
|
||||
/**
|
||||
* Server-side transcription engine — TODO(M5).
|
||||
*
|
||||
* POST /api/transcribe: auth-guarded, runs the same Transformers.js pipeline
|
||||
* under Bun (ONNX Runtime, CPU), streams progress + partial transcripts as a
|
||||
* chunked JSON-lines response (Decisions.md D-011). Reuses the shared model
|
||||
* catalog in src/lib/models.ts; keep pipelines warm across requests.
|
||||
*/
|
||||
|
||||
export {};
|
||||
@@ -0,0 +1,142 @@
|
||||
/**
|
||||
* Inference Web Worker (Decisions.md D-003 — inference never runs on the main
|
||||
* thread). Hosts the Transformers.js `automatic-speech-recognition` pipeline.
|
||||
*
|
||||
* M1: hardcoded-model, WASM-only. WebGPU detection + fallback lands in M2
|
||||
* (D-004); model switching/disposal also lands in M2.
|
||||
*/
|
||||
|
||||
import {
|
||||
pipeline,
|
||||
WhisperTextStreamer,
|
||||
type AutomaticSpeechRecognitionPipeline,
|
||||
type ProgressInfo,
|
||||
} from "@huggingface/transformers";
|
||||
import {
|
||||
isUiToWorkerMessage,
|
||||
type LoadMessage,
|
||||
type ResultMessage,
|
||||
type TranscriptChunk,
|
||||
type WorkerToUiMessage,
|
||||
} from "./protocol";
|
||||
|
||||
// `lib: ["DOM"]` types `self` as Window; give it the worker-scope shape we use.
|
||||
interface DedicatedWorkerScope {
|
||||
postMessage(message: WorkerToUiMessage): void;
|
||||
addEventListener(
|
||||
type: "message",
|
||||
listener: (event: MessageEvent<unknown>) => void,
|
||||
): void;
|
||||
}
|
||||
const scope = self as unknown as DedicatedWorkerScope;
|
||||
|
||||
function post(message: WorkerToUiMessage): void {
|
||||
scope.postMessage(message);
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
/** Forward Transformers.js progress events to the UI as load-progress messages. */
|
||||
function forwardProgress(info: ProgressInfo): void {
|
||||
switch (info.status) {
|
||||
case "initiate":
|
||||
case "download":
|
||||
case "done":
|
||||
post({ type: "load-progress", status: info.status, file: info.file });
|
||||
break;
|
||||
case "progress":
|
||||
post({
|
||||
type: "load-progress",
|
||||
status: "progress",
|
||||
file: info.file,
|
||||
loaded: info.loaded,
|
||||
total: info.total,
|
||||
progress: info.progress,
|
||||
});
|
||||
break;
|
||||
case "ready":
|
||||
post({ type: "load-progress", status: "ready" });
|
||||
break;
|
||||
default:
|
||||
// e.g. "progress_total" — the UI derives totals per-file itself.
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let asrPromise: Promise<AutomaticSpeechRecognitionPipeline> | null = null;
|
||||
|
||||
function load(message: LoadMessage): Promise<AutomaticSpeechRecognitionPipeline> {
|
||||
// Idempotent for M1's single hardcoded model: repeated "load" messages
|
||||
// reuse the in-flight/completed pipeline.
|
||||
asrPromise ??= pipeline("automatic-speech-recognition", message.modelId, {
|
||||
device: message.device,
|
||||
progress_callback: forwardProgress,
|
||||
});
|
||||
return asrPromise;
|
||||
}
|
||||
|
||||
async function transcribe(audio: Float32Array): Promise<void> {
|
||||
if (asrPromise === null) {
|
||||
post({
|
||||
type: "error",
|
||||
stage: "transcribe",
|
||||
message: "No model loaded — send a \"load\" message first.",
|
||||
});
|
||||
return;
|
||||
}
|
||||
const asr = await asrPromise;
|
||||
|
||||
const streamer = new WhisperTextStreamer(
|
||||
// The ASR pipeline for Whisper models always carries a WhisperTokenizer.
|
||||
asr.tokenizer as ConstructorParameters<typeof WhisperTextStreamer>[0],
|
||||
{
|
||||
skip_prompt: true,
|
||||
callback_function: (text: string) => post({ type: "partial", text }),
|
||||
},
|
||||
);
|
||||
|
||||
const output = await asr(audio, {
|
||||
chunk_length_s: 30,
|
||||
stride_length_s: 5,
|
||||
return_timestamps: true,
|
||||
streamer,
|
||||
});
|
||||
|
||||
const single = (Array.isArray(output) ? output[0] : output) as {
|
||||
text?: string;
|
||||
chunks?: TranscriptChunk[];
|
||||
};
|
||||
const result: ResultMessage = {
|
||||
type: "result",
|
||||
text: (single.text ?? "").trim(),
|
||||
chunks: single.chunks ?? [],
|
||||
};
|
||||
post(result);
|
||||
}
|
||||
|
||||
scope.addEventListener("message", (event) => {
|
||||
const data = event.data;
|
||||
if (!isUiToWorkerMessage(data)) {
|
||||
post({
|
||||
type: "error",
|
||||
stage: "transcribe",
|
||||
message: "Malformed message sent to inference worker.",
|
||||
});
|
||||
return;
|
||||
}
|
||||
switch (data.type) {
|
||||
case "load":
|
||||
load(data).catch((error: unknown) => {
|
||||
asrPromise = null; // allow retry after a failed download
|
||||
post({ type: "error", stage: "load", message: errorMessage(error) });
|
||||
});
|
||||
break;
|
||||
case "transcribe":
|
||||
transcribe(data.audio).catch((error: unknown) => {
|
||||
post({ type: "error", stage: "transcribe", message: errorMessage(error) });
|
||||
});
|
||||
break;
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,189 @@
|
||||
/**
|
||||
* Typed postMessage protocol between the UI and the inference Web Worker
|
||||
* (Decisions.md D-003). Shared by `src/worker/inference.worker.ts` and the UI.
|
||||
*
|
||||
* Everything in this file must stay free of browser/worker globals so it can
|
||||
* be unit-tested with `bun test`.
|
||||
*/
|
||||
|
||||
/** Devices supported by the worker. M1 is WASM-only; M2 adds "webgpu" (D-004). */
|
||||
export type InferenceDevice = "wasm";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// UI → worker
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface LoadMessage {
|
||||
type: "load";
|
||||
/** Hugging Face Hub model id, e.g. "onnx-community/whisper-tiny.en". */
|
||||
modelId: string;
|
||||
device: InferenceDevice;
|
||||
}
|
||||
|
||||
export interface TranscribeMessage {
|
||||
type: "transcribe";
|
||||
/**
|
||||
* Mono PCM at 16 kHz (see src/lib/audio.ts). The underlying ArrayBuffer
|
||||
* should be passed as a transferable for zero-copy handoff.
|
||||
*/
|
||||
audio: Float32Array;
|
||||
}
|
||||
|
||||
export type UiToWorkerMessage = LoadMessage | TranscribeMessage;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Worker → UI
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Model download / initialization progress. Mirrors the Transformers.js
|
||||
* `progress_callback` events we care about; `status: "ready"` marks the end
|
||||
* of a successful load.
|
||||
*/
|
||||
export type LoadProgressMessage =
|
||||
| { type: "load-progress"; status: "initiate"; file: string }
|
||||
| { type: "load-progress"; status: "download"; file: string }
|
||||
| {
|
||||
type: "load-progress";
|
||||
status: "progress";
|
||||
file: string;
|
||||
/** Bytes downloaded so far. */
|
||||
loaded: number;
|
||||
/** Total bytes for this file. */
|
||||
total: number;
|
||||
/** Percentage in [0, 100]. */
|
||||
progress: number;
|
||||
}
|
||||
| { type: "load-progress"; status: "done"; file: string }
|
||||
| { type: "load-progress"; status: "ready" };
|
||||
|
||||
/** Incremental transcript fragment — append to the text streamed so far. */
|
||||
export interface PartialMessage {
|
||||
type: "partial";
|
||||
text: string;
|
||||
}
|
||||
|
||||
/** One timestamped segment of the final transcript. */
|
||||
export interface TranscriptChunk {
|
||||
text: string;
|
||||
/** [start, end] in seconds; end may be null if Whisper never closed the segment. */
|
||||
timestamp: [number, number | null];
|
||||
}
|
||||
|
||||
export interface ResultMessage {
|
||||
type: "result";
|
||||
text: string;
|
||||
chunks: TranscriptChunk[];
|
||||
}
|
||||
|
||||
export interface ErrorMessage {
|
||||
type: "error";
|
||||
stage: "load" | "transcribe";
|
||||
message: string;
|
||||
}
|
||||
|
||||
export type WorkerToUiMessage =
|
||||
| LoadProgressMessage
|
||||
| PartialMessage
|
||||
| ResultMessage
|
||||
| ErrorMessage;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Runtime guards (postMessage data arrives untyped)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null;
|
||||
}
|
||||
|
||||
export function isUiToWorkerMessage(value: unknown): value is UiToWorkerMessage {
|
||||
if (!isRecord(value)) return false;
|
||||
switch (value.type) {
|
||||
case "load":
|
||||
return typeof value.modelId === "string" && value.device === "wasm";
|
||||
case "transcribe":
|
||||
return value.audio instanceof Float32Array;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function isWorkerToUiMessage(value: unknown): value is WorkerToUiMessage {
|
||||
if (!isRecord(value)) return false;
|
||||
switch (value.type) {
|
||||
case "load-progress":
|
||||
switch (value.status) {
|
||||
case "initiate":
|
||||
case "download":
|
||||
case "done":
|
||||
return typeof value.file === "string";
|
||||
case "progress":
|
||||
return (
|
||||
typeof value.file === "string" &&
|
||||
typeof value.loaded === "number" &&
|
||||
typeof value.total === "number" &&
|
||||
typeof value.progress === "number"
|
||||
);
|
||||
case "ready":
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
case "partial":
|
||||
return typeof value.text === "string";
|
||||
case "result":
|
||||
return typeof value.text === "string" && Array.isArray(value.chunks);
|
||||
case "error":
|
||||
return (
|
||||
typeof value.message === "string" &&
|
||||
(value.stage === "load" || value.stage === "transcribe")
|
||||
);
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Load-progress accumulation (pure; drives the download progress UI)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface FileProgress {
|
||||
file: string;
|
||||
loaded: number;
|
||||
total: number;
|
||||
done: boolean;
|
||||
}
|
||||
|
||||
export interface LoadProgressState {
|
||||
/** Per-file progress, keyed by file name, in first-seen order. */
|
||||
files: Record<string, FileProgress>;
|
||||
/** True once the pipeline reported ready. */
|
||||
ready: boolean;
|
||||
}
|
||||
|
||||
export const initialLoadProgress: LoadProgressState = { files: {}, ready: false };
|
||||
|
||||
/** Fold one load-progress message into the accumulated state. Pure. */
|
||||
export function applyLoadProgress(
|
||||
state: LoadProgressState,
|
||||
message: LoadProgressMessage,
|
||||
): LoadProgressState {
|
||||
if (message.status === "ready") {
|
||||
return { ...state, ready: true };
|
||||
}
|
||||
const previous = state.files[message.file];
|
||||
const next: FileProgress = {
|
||||
file: message.file,
|
||||
loaded: previous?.loaded ?? 0,
|
||||
total: previous?.total ?? 0,
|
||||
done: previous?.done ?? false,
|
||||
};
|
||||
if (message.status === "progress") {
|
||||
next.loaded = message.loaded;
|
||||
next.total = message.total;
|
||||
} else if (message.status === "done") {
|
||||
next.done = true;
|
||||
if (next.total > 0) next.loaded = next.total;
|
||||
}
|
||||
return { ...state, files: { ...state.files, [message.file]: next } };
|
||||
}
|
||||
Reference in New Issue
Block a user