feat(import): video-folder and pdf-deck jobs, deck cache

Co-Authored-By: Claude Opus 5.5 <[email protected]>

packages/feature-import/src/deck.ts

@@ -0,0 +1,49 @@
+import { basename, extname, join } from "node:path";
+import { copyFile, mkdir } from "node:fs/promises";
+import { coverCacheDir, deckCacheDir, exists, type Deps, type ManagedCourse } from "./context";
+import { putFile } from "./media";
+import { renderPdf } from "./pdf";
+import { slugify, uniqueId } from "./util/text";
+
+export interface AttachedDeck { deckId: string; file: string; pages?: number; warning?: string }
+
+/** All lesson and module ids of a raw manifest, for validating "attach to". */
+export function attachTargets(manifest: Record<string, any>): Set<string> {
+  const ids = new Set<string>();
+  for (const m of manifest.modules ?? []) {
+    ids.add(m.id);
+    for (const l of m.lessons ?? []) ids.add(l.id);
+  }
+  return ids;
+}
+
+/**
+ * Copies a PDF into the course (media/decks/), registers it in the (in-memory) manifest, and builds the derived cache:
+ * page images + text.json under <dataDir>/cache/decks/<course>/<deck>/ and the first page as a cover candidate.
+ * A failure of the PDF tool only degrades to a warning: the deck stays attached and the cache can be rebuilt.
+ * The caller writes the manifest.
+ */
+export async function attachDeck(d: Deps, course: ManagedCourse, pdfPath: string, attachTo: string, opts: { title?: string; mode?: "copy" | "link"; onProgress?: (fraction: number) => void } = {}): Promise<AttachedDeck> {
+  if (!attachTarget(course, attachTo)) throw new Error(`attachTo "${attachTo}" is not a lesson or module of course "${course.id}"`);
+  const decks: any[] = (course.manifest.decks ??= []);
+  const baseName = basename(pdfPath, extname(pdfPath));
+  const deckId = uniqueId(`d-${slugify(baseName)}`, (x) => decks.some((k) => k.id === x));
+  let file = `media/decks/${basename(pdfPath)}`;
+  for (let n = 2; await exists(join(course.dir, file)); n++) file = `media/decks/${baseName}-${n}${extname(pdfPath)}`;
+  await putFile(pdfPath, join(course.dir, file), opts.mode ?? "copy");
+  decks.push({ id: deckId, file, title: opts.title ?? baseName.replace(/[_]+/g, " ").trim(), attachTo });
+
+  const out: AttachedDeck = { deckId, file };
+  try {
+    const r = await renderPdf(d.python, join(course.dir, file), deckCacheDir(d, course.id, deckId), { onPage: (p, t) => opts.onProgress?.(p / t) });
+    out.pages = r.pages;
+    await mkdir(coverCacheDir(d, course.id), { recursive: true });
+    await copyFile(join(deckCacheDir(d, course.id, deckId), "p-001.jpg"), join(coverCacheDir(d, course.id), `deck-${deckId}.jpg`));
+  } catch (e) {
+    out.warning = `deck "${basename(pdfPath)}" is attached, but page images and text could not be built (${e instanceof Error ? e.message.split("\n")[0] : e}); STC_PYTHON must point to Python with PyMuPDF`;
+  }
+  return out;
+}
+
+const attachTarget = (c: ManagedCourse, id: string) => attachTargets(c.manifest).has(id);
+

packages/feature-import/src/jobs/pdf-deck.ts

@@ -0,0 +1,26 @@
+import { stat } from "node:fs/promises";
+import { join } from "node:path";
+import type { ImportResult, JobContext } from "@stc/contracts";
+import { finish, managedCourse, saveManifest, type Deps } from "../context";
+import { attachDeck } from "../deck";
+import { optStr, parsePayload } from "./common";
+
+export async function importPdfDeck(d: Deps, payload: unknown, jc: JobContext): Promise<ImportResult> {
+  const { path, options } = parsePayload(payload, "pdf-deck");
+  const warnings: string[] = [];
+  if (!(await stat(path).catch(() => null))?.isFile() || !/\.pdf$/i.test(path)) throw new Error(`not a PDF file: ${path}`);
+  const course = await managedCourse(d, optStr(options, "courseId"));
+  const attachTo = optStr(options, "attachTo");
+  if (!attachTo) throw new Error("options.attachTo is required: the lesson or module id the deck belongs to");
+  await jc.reportProgress(0.05, "Copying the deck");
+  const r = await attachDeck(d, course, path, attachTo, {
+    title: optStr(options, "title"),
+    mode: options.media === "link" ? "link" : "copy",
+    onProgress: (f) => void jc.reportProgress(0.1 + 0.8 * f, "Rendering slides"),
+  });
+  if (r.warning) warnings.push(r.warning);
+  await saveManifest(course.dir, course.manifest);
+  await finish(d, course.id, warnings);
+  await jc.reportProgress(1, r.pages ? `Deck attached (${r.pages} slides)` : "Deck attached");
+  return { courseId: course.id, manifestPath: join(course.dir, "stcourse.yaml"), warnings };
+}

packages/feature-import/src/jobs/video-folder.ts

@@ -0,0 +1,62 @@
+import { stat } from "node:fs/promises";
+import { basename, extname, join } from "node:path";
+import type { ImportResult, JobContext } from "@stc/contracts";
+import { finish, frameCandidates, freshCourseId, publishCourse, saveManifest, sourceBlock, stageDir, type Deps } from "../context";
+import { probeDuration, putFile } from "../media";
+import { naturalSort } from "../util/text";
+import { openSource } from "../util/source";
+import { adoptCover, maybeQueueTranscription, optBool, optStr, parsePayload, withStage } from "./common";
+
+const VIDEO = /\.(mp4|m4v|webm|mov|mkv|avi)$/i;
+const BROWSER_OK = /\.(mp4|m4v|webm|mov)$/i;
+
+export function lessonTitle(file: string): string {
+  const base = basename(file, extname(file));
+  const t = base.replace(/^\s*\d+\s*[-_.)\s]+(?=\S)/, "").replace(/_/g, " ").replace(/\s+/g, " ").trim();
+  return t || base;
+}
+
+export async function importVideoFolder(d: Deps, payload: unknown, jc: JobContext): Promise<ImportResult> {
+  const { path, options } = parsePayload(payload, "video-folder");
+  const warnings: string[] = [];
+  if (!(await stat(path).catch(() => null))?.isDirectory()) throw new Error(`not a folder: ${path}`);
+  const src = await openSource(path);
+  const files = naturalSort(src.entries.filter((e) => VIDEO.test(e) && !e.split("/").some((p) => p.startsWith("."))));
+  if (files.length === 0) throw new Error(`no video files found in ${path}`);
+  const title = optStr(options, "title") ?? basename(path).replace(/[_]+/g, " ");
+  const id = await freshCourseId(d, optStr(options, "courseId"), title);
+  const mode = options.media === "link" ? "link" : "copy";
+  const stage = await stageDir(d, "videos");
+
+  return withStage(stage, async () => {
+    const lessons: any[] = [];
+    const placed: string[] = [];
+    const used = new Set<string>();
+    for (const [i, f] of files.entries()) {
+      await jc.reportProgress(0.05 + 0.7 * (i / files.length), `Copying ${basename(f)}`);
+      let rel = `media/${f}`;
+      if (used.has(rel.toLowerCase())) rel = `media/${i + 1}-${basename(f)}`;
+      used.add(rel.toLowerCase());
+      await putFile(join(path, f), join(stage, rel), mode);
+      placed.push(join(stage, rel));
+      if (!BROWSER_OK.test(f)) warnings.push(`${f} may not play in browsers (${extname(f)}); consider converting it to MP4`);
+      const duration = await probeDuration(d.tools, join(stage, rel));
+      lessons.push({ id: `l${i + 1}`, title: lessonTitle(f), type: "video", file: rel, ...(duration ? { duration } : {}) });
+    }
+    await jc.reportProgress(0.8, "Making cover candidates");
+    const covers = await frameCandidates(d, id, placed);
+    if (covers.length === 0) warnings.push("could not extract cover frames (is ffmpeg available?)");
+    const cover = await adoptCover(d, id, stage, covers[0]);
+    await saveManifest(stage, {
+      schema: 1, id, title, visibility: "restricted", ...(cover ? { cover } : {}),
+      source: sourceBlock("video-folder", path),
+      modules: [{ id: "m1", title: "Videos", lessons }],
+    });
+    await jc.reportProgress(0.92, "Publishing the course");
+    const dir = await publishCourse(d, stage, id);
+    await finish(d, id, warnings);
+    await maybeQueueTranscription(d, id, optBool(options, "transcribe"), warnings);
+    await jc.reportProgress(1, `Imported ${files.length} videos`);
+    return { courseId: id, manifestPath: join(dir, "stcourse.yaml"), warnings };
+  });
+}

packages/feature-import/test/video-folder-pdf.test.ts

@@ -0,0 +1,75 @@
+import { expect, test } from "bun:test";
+import { readdir, readFile, stat } from "node:fs/promises";
+import { join } from "node:path";
+import { importPdfDeck } from "../src/jobs/pdf-deck";
+import { importVideoFolder, lessonTitle } from "../src/jobs/video-folder";
+import { makeEnv } from "./harness";
+import { makePdf, makeVideo, tmp } from "./fixtures";
+
+test("lessonTitle strips ordering prefixes", () => {
+  expect(lessonTitle("01 - Intro to Nix.mp4")).toBe("Intro to Nix");
+  expect(lessonTitle("2_setup_env.mkv")).toBe("setup env");
+  expect(lessonTitle("12.mp4")).toBe("12");
+});
+
+async function folderCourse() {
+  const env = await makeEnv();
+  const dir = await tmp();
+  const src = join(dir, "My Homelab Videos");
+  await makeVideo(join(src, "10 - Last.mp4"), 1);
+  await makeVideo(join(src, "2 - Second Clip.mp4"), 1);
+  await makeVideo(join(src, "1 - Ünïcode first.mp4"), 1);
+  const before = (await readdir(src)).sort();
+  const r = await importVideoFolder(env.deps, { kind: "video-folder", path: src }, env.jc);
+  return { env, src, before, r };
+}
+
+test("a folder of videos becomes one lesson per file in natural order", async () => {
+  const { env, src, before, r } = await folderCourse();
+  expect(r.courseId).toBe("my-homelab-videos");
+  const m = env.deps.ctx.courses.get(r.courseId)!;
+  expect(m.status).toBe("ok");
+  expect(m.manifest!.modules[0]!.lessons.map((l) => l.title)).toEqual(["Ünïcode first", "Second Clip", "Last"]);
+  expect(m.manifest!.visibility).toBe("restricted");
+  expect(m.manifest!.source?.kind).toBe("video-folder");
+  expect(env.deps.ctx.courses.resolvePath(r.courseId, "media/1 - Ünïcode first.mp4")).toBeTruthy();
+  expect((await readdir(src)).sort()).toEqual(before);
+});
+
+test("a deck attaches to a lesson, builds page images and text, and leaves the PDF original alone", async () => {
+  const { env, r } = await folderCourse();
+  const d = await tmp();
+  await makePdf(join(d, "Slides Deck.pdf"), ["Alpha slide", "Beta slide", "Gamma slide"]);
+  const before = await readFile(join(d, "Slides Deck.pdf"));
+  const res = await importPdfDeck(env.deps, { kind: "pdf-deck", path: join(d, "Slides Deck.pdf"), options: { courseId: r.courseId, attachTo: "l2", title: "My deck" } }, env.jc);
+  expect(res.warnings).toEqual([]);
+  const c = env.deps.ctx.courses.get(r.courseId)!;
+  expect(c.status).toBe("ok");
+  expect(c.manifest!.decks).toEqual([{ id: "d-slides-deck", file: "media/decks/Slides Deck.pdf", title: "My deck", attachTo: "l2" }]);
+  const cache = join(env.dataDir, "cache/decks", r.courseId, "d-slides-deck");
+  expect((await readdir(cache)).sort()).toEqual(["p-001.jpg", "p-002.jpg", "p-003.jpg", "text.json"]);
+  expect(JSON.parse(await readFile(join(cache, "text.json"), "utf8"))[2].text).toContain("Gamma slide");
+  expect((await stat(join(env.dataDir, "cache/covers", r.courseId, "deck-d-slides-deck.jpg"))).size).toBeGreaterThan(100);
+  expect(Buffer.compare(await readFile(join(d, "Slides Deck.pdf")), before)).toBe(0);
+});
+
+test("deck attach validates the target and refuses courses the importer did not create", async () => {
+  const { env, r } = await folderCourse();
+  const d = await tmp();
+  await makePdf(join(d, "a.pdf"));
+  await expect(importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: r.courseId, attachTo: "nope" } }, env.jc)).rejects.toThrow(/not a lesson or module/);
+  await expect(importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: r.courseId } }, env.jc)).rejects.toThrow(/attachTo is required/);
+  await expect(importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: "sample", attachTo: "x" } }, env.jc)).rejects.toThrow(/only edits courses it created/);
+  await expect(importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: "../x", attachTo: "x" } }, env.jc)).rejects.toThrow(/only edits courses it created/);
+  await expect(importPdfDeck(env.deps, { path: join(d, "missing.pdf"), options: {} }, env.jc)).rejects.toThrow(/not a PDF/);
+});
+
+test("without PyMuPDF the deck is still attached, with a warning", async () => {
+  const { env, r } = await folderCourse();
+  env.deps.python = "/nonexistent/python";
+  const d = await tmp();
+  await makePdf(join(d, "a.pdf"));
+  const res = await importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: r.courseId, attachTo: "m1" } }, env.jc);
+  expect(res.warnings.join()).toContain("page images and text could not be built");
+  expect(env.deps.ctx.courses.get(r.courseId)!.manifest!.decks).toHaveLength(1);
+});