diff --git a/packages/feature-import/src/deck.ts b/packages/feature-import/src/deck.ts new file mode 100644 index 0000000..9844060 --- /dev/null +++ b/packages/feature-import/src/deck.ts @@ -0,0 +1,49 @@ +import { basename, extname, join } from "node:path"; +import { copyFile, mkdir } from "node:fs/promises"; +import { coverCacheDir, deckCacheDir, exists, type Deps, type ManagedCourse } from "./context"; +import { putFile } from "./media"; +import { renderPdf } from "./pdf"; +import { slugify, uniqueId } from "./util/text"; + +export interface AttachedDeck { deckId: string; file: string; pages?: number; warning?: string } + +/** All lesson and module ids of a raw manifest, for validating "attach to". */ +export function attachTargets(manifest: Record): Set { + const ids = new Set(); + for (const m of manifest.modules ?? []) { + ids.add(m.id); + for (const l of m.lessons ?? []) ids.add(l.id); + } + return ids; +} + +/** + * Copies a PDF into the course (media/decks/), registers it in the (in-memory) manifest, and builds the derived cache: + * page images + text.json under /cache/decks/// and the first page as a cover candidate. + * A failure of the PDF tool only degrades to a warning: the deck stays attached and the cache can be rebuilt. + * The caller writes the manifest. + */ +export async function attachDeck(d: Deps, course: ManagedCourse, pdfPath: string, attachTo: string, opts: { title?: string; mode?: "copy" | "link"; onProgress?: (fraction: number) => void } = {}): Promise { + if (!attachTarget(course, attachTo)) throw new Error(`attachTo "${attachTo}" is not a lesson or module of course "${course.id}"`); + const decks: any[] = (course.manifest.decks ??= []); + const baseName = basename(pdfPath, extname(pdfPath)); + const deckId = uniqueId(`d-${slugify(baseName)}`, (x) => decks.some((k) => k.id === x)); + let file = `media/decks/${basename(pdfPath)}`; + for (let n = 2; await exists(join(course.dir, file)); n++) file = `media/decks/${baseName}-${n}${extname(pdfPath)}`; + await putFile(pdfPath, join(course.dir, file), opts.mode ?? "copy"); + decks.push({ id: deckId, file, title: opts.title ?? baseName.replace(/[_]+/g, " ").trim(), attachTo }); + + const out: AttachedDeck = { deckId, file }; + try { + const r = await renderPdf(d.python, join(course.dir, file), deckCacheDir(d, course.id, deckId), { onPage: (p, t) => opts.onProgress?.(p / t) }); + out.pages = r.pages; + await mkdir(coverCacheDir(d, course.id), { recursive: true }); + await copyFile(join(deckCacheDir(d, course.id, deckId), "p-001.jpg"), join(coverCacheDir(d, course.id), `deck-${deckId}.jpg`)); + } catch (e) { + out.warning = `deck "${basename(pdfPath)}" is attached, but page images and text could not be built (${e instanceof Error ? e.message.split("\n")[0] : e}); STC_PYTHON must point to Python with PyMuPDF`; + } + return out; +} + +const attachTarget = (c: ManagedCourse, id: string) => attachTargets(c.manifest).has(id); + diff --git a/packages/feature-import/src/jobs/pdf-deck.ts b/packages/feature-import/src/jobs/pdf-deck.ts new file mode 100644 index 0000000..7cfe3e4 --- /dev/null +++ b/packages/feature-import/src/jobs/pdf-deck.ts @@ -0,0 +1,26 @@ +import { stat } from "node:fs/promises"; +import { join } from "node:path"; +import type { ImportResult, JobContext } from "@stc/contracts"; +import { finish, managedCourse, saveManifest, type Deps } from "../context"; +import { attachDeck } from "../deck"; +import { optStr, parsePayload } from "./common"; + +export async function importPdfDeck(d: Deps, payload: unknown, jc: JobContext): Promise { + const { path, options } = parsePayload(payload, "pdf-deck"); + const warnings: string[] = []; + if (!(await stat(path).catch(() => null))?.isFile() || !/\.pdf$/i.test(path)) throw new Error(`not a PDF file: ${path}`); + const course = await managedCourse(d, optStr(options, "courseId")); + const attachTo = optStr(options, "attachTo"); + if (!attachTo) throw new Error("options.attachTo is required: the lesson or module id the deck belongs to"); + await jc.reportProgress(0.05, "Copying the deck"); + const r = await attachDeck(d, course, path, attachTo, { + title: optStr(options, "title"), + mode: options.media === "link" ? "link" : "copy", + onProgress: (f) => void jc.reportProgress(0.1 + 0.8 * f, "Rendering slides"), + }); + if (r.warning) warnings.push(r.warning); + await saveManifest(course.dir, course.manifest); + await finish(d, course.id, warnings); + await jc.reportProgress(1, r.pages ? `Deck attached (${r.pages} slides)` : "Deck attached"); + return { courseId: course.id, manifestPath: join(course.dir, "stcourse.yaml"), warnings }; +} diff --git a/packages/feature-import/src/jobs/video-folder.ts b/packages/feature-import/src/jobs/video-folder.ts new file mode 100644 index 0000000..b4ff4f4 --- /dev/null +++ b/packages/feature-import/src/jobs/video-folder.ts @@ -0,0 +1,62 @@ +import { stat } from "node:fs/promises"; +import { basename, extname, join } from "node:path"; +import type { ImportResult, JobContext } from "@stc/contracts"; +import { finish, frameCandidates, freshCourseId, publishCourse, saveManifest, sourceBlock, stageDir, type Deps } from "../context"; +import { probeDuration, putFile } from "../media"; +import { naturalSort } from "../util/text"; +import { openSource } from "../util/source"; +import { adoptCover, maybeQueueTranscription, optBool, optStr, parsePayload, withStage } from "./common"; + +const VIDEO = /\.(mp4|m4v|webm|mov|mkv|avi)$/i; +const BROWSER_OK = /\.(mp4|m4v|webm|mov)$/i; + +export function lessonTitle(file: string): string { + const base = basename(file, extname(file)); + const t = base.replace(/^\s*\d+\s*[-_.)\s]+(?=\S)/, "").replace(/_/g, " ").replace(/\s+/g, " ").trim(); + return t || base; +} + +export async function importVideoFolder(d: Deps, payload: unknown, jc: JobContext): Promise { + const { path, options } = parsePayload(payload, "video-folder"); + const warnings: string[] = []; + if (!(await stat(path).catch(() => null))?.isDirectory()) throw new Error(`not a folder: ${path}`); + const src = await openSource(path); + const files = naturalSort(src.entries.filter((e) => VIDEO.test(e) && !e.split("/").some((p) => p.startsWith(".")))); + if (files.length === 0) throw new Error(`no video files found in ${path}`); + const title = optStr(options, "title") ?? basename(path).replace(/[_]+/g, " "); + const id = await freshCourseId(d, optStr(options, "courseId"), title); + const mode = options.media === "link" ? "link" : "copy"; + const stage = await stageDir(d, "videos"); + + return withStage(stage, async () => { + const lessons: any[] = []; + const placed: string[] = []; + const used = new Set(); + for (const [i, f] of files.entries()) { + await jc.reportProgress(0.05 + 0.7 * (i / files.length), `Copying ${basename(f)}`); + let rel = `media/${f}`; + if (used.has(rel.toLowerCase())) rel = `media/${i + 1}-${basename(f)}`; + used.add(rel.toLowerCase()); + await putFile(join(path, f), join(stage, rel), mode); + placed.push(join(stage, rel)); + if (!BROWSER_OK.test(f)) warnings.push(`${f} may not play in browsers (${extname(f)}); consider converting it to MP4`); + const duration = await probeDuration(d.tools, join(stage, rel)); + lessons.push({ id: `l${i + 1}`, title: lessonTitle(f), type: "video", file: rel, ...(duration ? { duration } : {}) }); + } + await jc.reportProgress(0.8, "Making cover candidates"); + const covers = await frameCandidates(d, id, placed); + if (covers.length === 0) warnings.push("could not extract cover frames (is ffmpeg available?)"); + const cover = await adoptCover(d, id, stage, covers[0]); + await saveManifest(stage, { + schema: 1, id, title, visibility: "restricted", ...(cover ? { cover } : {}), + source: sourceBlock("video-folder", path), + modules: [{ id: "m1", title: "Videos", lessons }], + }); + await jc.reportProgress(0.92, "Publishing the course"); + const dir = await publishCourse(d, stage, id); + await finish(d, id, warnings); + await maybeQueueTranscription(d, id, optBool(options, "transcribe"), warnings); + await jc.reportProgress(1, `Imported ${files.length} videos`); + return { courseId: id, manifestPath: join(dir, "stcourse.yaml"), warnings }; + }); +} diff --git a/packages/feature-import/test/video-folder-pdf.test.ts b/packages/feature-import/test/video-folder-pdf.test.ts new file mode 100644 index 0000000..1df5d4b --- /dev/null +++ b/packages/feature-import/test/video-folder-pdf.test.ts @@ -0,0 +1,75 @@ +import { expect, test } from "bun:test"; +import { readdir, readFile, stat } from "node:fs/promises"; +import { join } from "node:path"; +import { importPdfDeck } from "../src/jobs/pdf-deck"; +import { importVideoFolder, lessonTitle } from "../src/jobs/video-folder"; +import { makeEnv } from "./harness"; +import { makePdf, makeVideo, tmp } from "./fixtures"; + +test("lessonTitle strips ordering prefixes", () => { + expect(lessonTitle("01 - Intro to Nix.mp4")).toBe("Intro to Nix"); + expect(lessonTitle("2_setup_env.mkv")).toBe("setup env"); + expect(lessonTitle("12.mp4")).toBe("12"); +}); + +async function folderCourse() { + const env = await makeEnv(); + const dir = await tmp(); + const src = join(dir, "My Homelab Videos"); + await makeVideo(join(src, "10 - Last.mp4"), 1); + await makeVideo(join(src, "2 - Second Clip.mp4"), 1); + await makeVideo(join(src, "1 - Ünïcode first.mp4"), 1); + const before = (await readdir(src)).sort(); + const r = await importVideoFolder(env.deps, { kind: "video-folder", path: src }, env.jc); + return { env, src, before, r }; +} + +test("a folder of videos becomes one lesson per file in natural order", async () => { + const { env, src, before, r } = await folderCourse(); + expect(r.courseId).toBe("my-homelab-videos"); + const m = env.deps.ctx.courses.get(r.courseId)!; + expect(m.status).toBe("ok"); + expect(m.manifest!.modules[0]!.lessons.map((l) => l.title)).toEqual(["Ünïcode first", "Second Clip", "Last"]); + expect(m.manifest!.visibility).toBe("restricted"); + expect(m.manifest!.source?.kind).toBe("video-folder"); + expect(env.deps.ctx.courses.resolvePath(r.courseId, "media/1 - Ünïcode first.mp4")).toBeTruthy(); + expect((await readdir(src)).sort()).toEqual(before); +}); + +test("a deck attaches to a lesson, builds page images and text, and leaves the PDF original alone", async () => { + const { env, r } = await folderCourse(); + const d = await tmp(); + await makePdf(join(d, "Slides Deck.pdf"), ["Alpha slide", "Beta slide", "Gamma slide"]); + const before = await readFile(join(d, "Slides Deck.pdf")); + const res = await importPdfDeck(env.deps, { kind: "pdf-deck", path: join(d, "Slides Deck.pdf"), options: { courseId: r.courseId, attachTo: "l2", title: "My deck" } }, env.jc); + expect(res.warnings).toEqual([]); + const c = env.deps.ctx.courses.get(r.courseId)!; + expect(c.status).toBe("ok"); + expect(c.manifest!.decks).toEqual([{ id: "d-slides-deck", file: "media/decks/Slides Deck.pdf", title: "My deck", attachTo: "l2" }]); + const cache = join(env.dataDir, "cache/decks", r.courseId, "d-slides-deck"); + expect((await readdir(cache)).sort()).toEqual(["p-001.jpg", "p-002.jpg", "p-003.jpg", "text.json"]); + expect(JSON.parse(await readFile(join(cache, "text.json"), "utf8"))[2].text).toContain("Gamma slide"); + expect((await stat(join(env.dataDir, "cache/covers", r.courseId, "deck-d-slides-deck.jpg"))).size).toBeGreaterThan(100); + expect(Buffer.compare(await readFile(join(d, "Slides Deck.pdf")), before)).toBe(0); +}); + +test("deck attach validates the target and refuses courses the importer did not create", async () => { + const { env, r } = await folderCourse(); + const d = await tmp(); + await makePdf(join(d, "a.pdf")); + await expect(importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: r.courseId, attachTo: "nope" } }, env.jc)).rejects.toThrow(/not a lesson or module/); + await expect(importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: r.courseId } }, env.jc)).rejects.toThrow(/attachTo is required/); + await expect(importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: "sample", attachTo: "x" } }, env.jc)).rejects.toThrow(/only edits courses it created/); + await expect(importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: "../x", attachTo: "x" } }, env.jc)).rejects.toThrow(/only edits courses it created/); + await expect(importPdfDeck(env.deps, { path: join(d, "missing.pdf"), options: {} }, env.jc)).rejects.toThrow(/not a PDF/); +}); + +test("without PyMuPDF the deck is still attached, with a warning", async () => { + const { env, r } = await folderCourse(); + env.deps.python = "/nonexistent/python"; + const d = await tmp(); + await makePdf(join(d, "a.pdf")); + const res = await importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: r.courseId, attachTo: "m1" } }, env.jc); + expect(res.warnings.join()).toContain("page images and text could not be built"); + expect(env.deps.ctx.courses.get(r.courseId)!.manifest!.decks).toHaveLength(1); +});