WIP: uncommitted work at pause (session limit)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/worker/longform/transformers.ts

@@ -163,13 +163,19 @@ export function createWhisperDeps(
   asr: AutomaticSpeechRecognitionPipeline,
   options: { multilingual: boolean },
 ): Pick<LongFormDeps, "decode" | "detectLanguage"> {
-  const decode = async (audio: Float32Array, request: DecodeRequest): Promise<DecodeResult> => {
+  const decode = async (
+    audio: Float32Array,
+    request: DecodeRequest,
+    encoderOutputs?: Tensor,
+  ): Promise<DecodeResult> => {
     const recorder = new LogProbRecorder();
     const generate: Record<string, unknown> = {
       return_timestamps: true,
       // One window, single-pass generate (D-021's seek-loop note applies).
       max_new_tokens: 440,
       logits_processor: new ListWithLast(recorder),
+      // generate() skips the encoder when handed its output.
+      ...(encoderOutputs && { encoder_outputs: encoderOutputs }),
     };
     if (options.multilingual) {
       if (request.language !== undefined) generate.language = request.language;
@@ -205,20 +211,47 @@ export function createWhisperDeps(
     };
   };
 
-  if (!options.multilingual) return { decode };
+  /** Encoder output of the chunk language detection just ran on. */
+  let encoded: { audio: Float32Array; hidden: Tensor } | null = null;
+  const decodeReusingEncoder = (audio: Float32Array, request: DecodeRequest) => {
+    if (encoded?.audio !== audio) {
+      encoded = null; // a new chunk: drop the stale encoder output
+      return decode(audio, request);
+    }
+    return decode(audio, request, encoded.hidden);
+  };
+
+  if (!options.multilingual) return { decode: decodeReusingEncoder };
 
   const detectLanguage = async (audio: Float32Array): Promise<string> => {
     const model = asr.model as unknown as {
       generation_config: WhisperGenerationConfig;
       generate(args: Record<string, unknown>): Promise<Tensor>;
+      _prepare_encoder_decoder_kwargs_for_generation(args: {
+        inputs_tensor: Tensor;
+        model_inputs: Record<string, unknown>;
+        model_input_name: string;
+        generation_config: unknown;
+      }): Promise<{ encoder_outputs: Tensor }>;
     };
     const config = model.generation_config;
     const languages = Object.entries(config.lang_to_id ?? {});
     const { input_features } = (await asr.processor(audio)) as { input_features: Tensor };
+    // Encode once and keep it: the chunk's real decode reuses this encoder
+    // output, so detection costs one decoder step instead of a second encoder
+    // pass (the encoder dominates a 30 s window's cost).
+    const { encoder_outputs } = await model._prepare_encoder_decoder_kwargs_for_generation({
+      inputs_tensor: input_features,
+      model_inputs: { input_features },
+      model_input_name: "input_features",
+      generation_config: config,
+    });
+    encoded = { audio, hidden: encoder_outputs };
     const list = new LogitsProcessorList();
     list.push(new LanguageOnly(new Set(languages.map(([, id]) => id))));
     const sequences = await model.generate({
       inputs: input_features,
+      encoder_outputs,
       decoder_input_ids: [config.decoder_start_token_id],
       max_new_tokens: 1,
       logits_processor: list,
@@ -228,5 +261,5 @@ export function createWhisperDeps(
     if (!token) throw new Error(`Language detection produced a non-language token (${tokenId}).`);
     return token.slice(2, -2); // "<|da|>" → "da"
   };
-  return { decode, detectLanguage };
+  return { decode: decodeReusingEncoder, detectLanguage };
 }