From 34f27d05c9482428bb4ebb9ea0b1b9eae739333e Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 03:13:22 +0100 Subject: [PATCH] feat(import): utilities, media tooling, PyMuPDF helper, synthetic fixtures Co-Authored-By: Claude Opus 5.5 --- packages/feature-import/package.json | 3 +- packages/feature-import/py/pdf_tool.py | 50 ++++++++ packages/feature-import/src/media.ts | 40 ++++++ packages/feature-import/src/pdf.ts | 23 ++++ packages/feature-import/src/util/exec.ts | 30 +++++ packages/feature-import/src/util/manifest.ts | 21 ++++ packages/feature-import/src/util/text.ts | 48 +++++++ packages/feature-import/test/fixtures.ts | 125 +++++++++++++++++++ packages/feature-import/test/media.test.ts | 38 ++++++ packages/feature-import/test/util.test.ts | 41 ++++++ 10 files changed, 418 insertions(+), 1 deletion(-) create mode 100644 packages/feature-import/py/pdf_tool.py create mode 100644 packages/feature-import/src/media.ts create mode 100644 packages/feature-import/src/pdf.ts create mode 100644 packages/feature-import/src/util/exec.ts create mode 100644 packages/feature-import/src/util/manifest.ts create mode 100644 packages/feature-import/src/util/text.ts create mode 100644 packages/feature-import/test/fixtures.ts create mode 100644 packages/feature-import/test/media.test.ts create mode 100644 packages/feature-import/test/util.test.ts diff --git a/packages/feature-import/package.json b/packages/feature-import/package.json index cab5a10..0b65795 100644 --- a/packages/feature-import/package.json +++ b/packages/feature-import/package.json @@ -3,5 +3,6 @@ "version": "0.0.0", "private": true, "type": "module", - "license": "AGPL-3.0-only" + "license": "AGPL-3.0-only", + "main": "src/index.ts" } diff --git a/packages/feature-import/py/pdf_tool.py b/packages/feature-import/py/pdf_tool.py new file mode 100644 index 0000000..0bff2b7 --- /dev/null +++ b/packages/feature-import/py/pdf_tool.py @@ -0,0 +1,50 @@ +"""PDF helper for the STCourses importer. Run as: python -I pdf_tool.py render [--width 1280] [--quality 80] + +Writes outdir/p-001.jpg ... and outdir/text.json ([{"page": 1, "text": "..."}]). +Prints one "page N/M" line per page on stdout and a final JSON line {"pages": M, "title": "..."}. +""" +import json +import os +import sys + +try: + import pymupdf as fitz +except ImportError: # older PyMuPDF + import fitz + + +def render(pdf, outdir, width, quality): + os.makedirs(outdir, exist_ok=True) + doc = fitz.open(pdf) + n = len(doc) + texts = [] + for i, page in enumerate(doc, start=1): + zoom = width / max(page.rect.width, 1) + pix = page.get_pixmap(matrix=fitz.Matrix(zoom, zoom), alpha=False) + pix.save(os.path.join(outdir, "p-%03d.jpg" % i), jpg_quality=quality) + texts.append({"page": i, "text": page.get_text().strip()}) + print("page %d/%d" % (i, n), flush=True) + with open(os.path.join(outdir, "text.json"), "w", encoding="utf-8") as f: + json.dump(texts, f, ensure_ascii=False) + print(json.dumps({"pages": n, "title": (doc.metadata or {}).get("title") or ""}), flush=True) + + +def main(argv): + if len(argv) < 4 or argv[1] != "render": + print("usage: pdf_tool.py render [--width N] [--quality N]", file=sys.stderr) + return 2 + width, quality = 1280, 80 + rest = argv[4:] + while rest: + flag, val = rest[0], rest[1] if len(rest) > 1 else "" + if flag == "--width": + width = int(val) + elif flag == "--quality": + quality = int(val) + rest = rest[2:] + render(argv[2], argv[3], width, quality) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/packages/feature-import/src/media.ts b/packages/feature-import/src/media.ts new file mode 100644 index 0000000..0fab184 --- /dev/null +++ b/packages/feature-import/src/media.ts @@ -0,0 +1,40 @@ +import { link, mkdir, copyFile } from "node:fs/promises"; +import { dirname } from "node:path"; +import { run } from "./util/exec"; + +export interface Tools { ffmpeg: string; ffprobe: string } + +export async function probeDuration(t: Tools, file: string): Promise { + try { + const out = await run([t.ffprobe, "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", file]); + const s = Number(out.trim()); + return Number.isFinite(s) && s > 0 ? Math.round(s) : undefined; + } catch { + return undefined; + } +} + +/** One JPEG frame at `fraction` of the clip, 640px wide. */ +export async function extractFrame(t: Tools, file: string, out: string, fraction = 0.2): Promise { + await mkdir(dirname(out), { recursive: true }); + const dur = (await probeDuration(t, file)) ?? 0; + const at = Math.max(0, dur * fraction); + await run([t.ffmpeg, "-v", "error", "-y", "-ss", String(at), "-i", file, "-frames:v", "1", "-vf", "scale=640:-2", "-q:v", "4", out]); +} + +/** + * Puts a large file into the course. "copy" uses `cp --reflink=auto` (a CoW clone where the filesystem supports it, + * a plain copy elsewhere); "link" makes a hard link and falls back to copying across filesystems. + * Either way the original is only read, and the course folder stays self-contained (the media route refuses symlinks that leave it). + */ +export async function putFile(src: string, dest: string, mode: "copy" | "link" = "copy"): Promise { + await mkdir(dirname(dest), { recursive: true }); + if (mode === "link") { + try { await link(src, dest); return; } catch { /* cross-device or unsupported: copy */ } + } + try { + await run(["cp", "--reflink=auto", "--", src, dest]); + } catch { + await copyFile(src, dest); + } +} diff --git a/packages/feature-import/src/pdf.ts b/packages/feature-import/src/pdf.ts new file mode 100644 index 0000000..c1b4366 --- /dev/null +++ b/packages/feature-import/src/pdf.ts @@ -0,0 +1,23 @@ +import { join } from "node:path"; +import { run } from "./util/exec"; + +export interface DeckRender { pages: number; title: string } + +const TOOL = join(import.meta.dir, "..", "py", "pdf_tool.py"); + +/** + * Renders page images (p-001.jpg ...) and text.json into `outDir` with PyMuPDF in a subprocess, + * so the server process never holds a PDF engine. `python` is STC_PYTHON (default python3). + */ +export async function renderPdf(python: string, pdf: string, outDir: string, opts: { onPage?: (page: number, total: number) => void; signal?: AbortSignal } = {}): Promise { + let last = ""; + await run([python, "-I", TOOL, "render", pdf, outDir], { + signal: opts.signal, + onLine(l) { + const m = /^page (\d+)\/(\d+)$/.exec(l); + if (m) opts.onPage?.(Number(m[1]), Number(m[2])); + else if (l.trim()) last = l; + }, + }); + return JSON.parse(last) as DeckRender; +} diff --git a/packages/feature-import/src/util/exec.ts b/packages/feature-import/src/util/exec.ts new file mode 100644 index 0000000..78b3c2f --- /dev/null +++ b/packages/feature-import/src/util/exec.ts @@ -0,0 +1,30 @@ +/** Runs a command (no shell). Resolves with stdout; rejects with the stderr tail on a non-zero exit. */ +export async function run(cmd: string[], opts: { signal?: AbortSignal; env?: Record; onLine?: (line: string) => void } = {}): Promise { + const proc = Bun.spawn(cmd, { + stdout: "pipe", stderr: "pipe", stdin: "ignore", + env: { ...process.env, LC_ALL: "C.UTF-8", ...opts.env } as Record, + signal: opts.signal, + }); + const out = (async () => { + let text = ""; + let pending = ""; + const dec = new TextDecoder(); + for await (const chunk of proc.stdout) { + const s = dec.decode(chunk, { stream: true }); + text += s; + if (opts.onLine) { + pending += s; + const lines = pending.split("\n"); + pending = lines.pop()!; + for (const l of lines) opts.onLine(l); + } + } + return text; + })(); + const [stdout, stderr, code] = await Promise.all([out, new Response(proc.stderr).text(), proc.exited]); + if (code !== 0) { + const tail = stderr.trim().split("\n").slice(-6).join("\n"); + throw new Error(`${cmd[0]} exited with ${code}${tail ? `: ${tail}` : ""}`); + } + return stdout; +} diff --git a/packages/feature-import/src/util/manifest.ts b/packages/feature-import/src/util/manifest.ts new file mode 100644 index 0000000..80b4ab3 --- /dev/null +++ b/packages/feature-import/src/util/manifest.ts @@ -0,0 +1,21 @@ +import { readFile, rename, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { ManifestSchema } from "@stc/course-format"; + +export type RawManifest = Record; + +export async function readRawManifest(courseDir: string): Promise { + return Bun.YAML.parse(await readFile(join(courseDir, "stcourse.yaml"), "utf8")) as RawManifest; +} + +/** Validates against the course-format schema, then writes stcourse.yaml via temp file + rename. */ +export async function writeManifest(courseDir: string, manifest: RawManifest): Promise { + const r = ManifestSchema.safeParse(manifest); + if (!r.success) { + throw new Error(`generated manifest is invalid: ${r.error.issues.map((i) => `${i.path.join(".") || "(root)"}: ${i.message}`).join("; ")}`); + } + const target = join(courseDir, "stcourse.yaml"); + const tmp = `${target}.${process.pid}.tmp`; + await writeFile(tmp, Bun.YAML.stringify(manifest, null, 2) + "\n"); + await rename(tmp, target); +} diff --git a/packages/feature-import/src/util/text.ts b/packages/feature-import/src/util/text.ts new file mode 100644 index 0000000..6fffbab --- /dev/null +++ b/packages/feature-import/src/util/text.ts @@ -0,0 +1,48 @@ +export function slugify(s: string): string { + const slug = s + .normalize("NFKD") + .replace(/[̀-ͯ]/g, "") + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 48) + .replace(/-+$/g, ""); + return slug || "course"; +} + +export function uniqueId(base: string, taken: (id: string) => boolean): string { + if (!taken(base)) return base; + for (let i = 2; ; i++) if (!taken(`${base}-${i}`)) return `${base}-${i}`; +} + +const NAMED: Record = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " " }; +export function decodeEntities(s: string): string { + return s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e: string) => { + if (e[0] === "#") { + const code = e[1]!.toLowerCase() === "x" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10); + return Number.isFinite(code) ? String.fromCodePoint(code) : m; + } + return NAMED[e.toLowerCase()] ?? m; + }); +} + +export function stripTags(s: string): string { + return decodeEntities(s.replace(/<[^>]*>/g, "")).replace(/\s+/g, " ").trim(); +} + +const collator = new Intl.Collator("en", { numeric: true, sensitivity: "base" }); +export function naturalSort(list: string[]): string[] { + return [...list].sort((a, b) => collator.compare(a, b) || (a < b ? -1 : a > b ? 1 : 0)); +} + +/** Lowercase alphanumeric words, for loose name matching. */ +export function words(s: string): string[] { + return s.normalize("NFKD").replace(/[̀-ͯ]/g, "").toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length > 1); +} +/** Share of the words of `a` that also occur in `b`. */ +export function containment(a: string, b: string): number { + const wa = words(a); + if (wa.length === 0) return 0; + const wb = new Set(words(b)); + return wa.filter((w) => wb.has(w)).length / wa.length; +} diff --git a/packages/feature-import/test/fixtures.ts b/packages/feature-import/test/fixtures.ts new file mode 100644 index 0000000..6e78abf --- /dev/null +++ b/packages/feature-import/test/fixtures.ts @@ -0,0 +1,125 @@ +// Tiny synthetic inputs for the importer tests. Nothing here is real course material. +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { run } from "../src/util/exec"; + +// PyMuPDF is needed for PDF tests: set STC_PYTHON, or this machine's venv is used when present. +const VENV_PY = "/home/user/.venvs/pdf/bin/python"; +export const PYTHON = process.env.STC_PYTHON ?? (existsSync(VENV_PY) ? VENV_PY : "python3"); + +export async function tmp(prefix = "stc-imp-"): Promise { + return mkdtemp(join(tmpdir(), prefix)); +} + +export async function makeVideo(path: string, seconds = 2): Promise { + await mkdir(dirname(path), { recursive: true }); + await run(["ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", `testsrc=duration=${seconds}:size=160x90:rate=10`, "-f", "lavfi", "-i", `sine=frequency=440:duration=${seconds}`, + "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-shortest", path]); +} + +/** Minimal valid PDF with one line of text per page. */ +export function pdfBytes(pages: string[]): Uint8Array { + const objs: string[] = []; + const kids = pages.map((_, i) => `${4 + i * 2} 0 R`).join(" "); + objs.push("<< /Type /Catalog /Pages 2 0 R >>"); + objs.push(`<< /Type /Pages /Kids [${kids}] /Count ${pages.length} >>`); + objs.push("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"); + pages.forEach((text, i) => { + const content = `BT /F1 24 Tf 40 150 Td (${text.replace(/[()\\]/g, "")}) Tj ET`; + objs.push(`<< /Type /Page /Parent 2 0 R /MediaBox [0 0 320 200] /Resources << /Font << /F1 3 0 R >> >> /Contents ${5 + i * 2} 0 R >>`); + objs.push(`<< /Length ${content.length} >>\nstream\n${content}\nendstream`); + }); + let out = "%PDF-1.4\n"; + const offsets: number[] = []; + objs.forEach((o, i) => { offsets.push(out.length); out += `${i + 1} 0 obj\n${o}\nendobj\n`; }); + const xref = out.length; + out += `xref\n0 ${objs.length + 1}\n0000000000 65535 f \n` + offsets.map((o) => `${String(o).padStart(10, "0")} 00000 n \n`).join(""); + out += `trailer\n<< /Size ${objs.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`; + return new TextEncoder().encode(out); +} + +export async function makePdf(path: string, pages = ["First slide alpha", "Second slide beta"]): Promise { + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, pdfBytes(pages)); +} + +export async function zipDir(srcDir: string, zipPath: string): Promise { + await run([PYTHON, "-I", "-c", + "import sys,os,zipfile\nsrc,out=sys.argv[1:3]\nz=zipfile.ZipFile(out,'w',zipfile.ZIP_STORED)\nfor r,d,f in os.walk(src):\n for n in f:\n p=os.path.join(r,n); z.write(p, os.path.relpath(p,src))\nz.close()", + srcDir, zipPath]); +} + +const li = (t: string) => `
  • ${t}
    • `; +const clipHtml = (s: number, m: number, title: string) => ` +
  • + +

    ${s}.${m} ${title}

    + In this video, ${title}, you'll learn it. +
    Watch Video
    + Download +
    • `; + +export const PACKT_SECTIONS = [ + { title: "Intro to Docker", clips: ["Course Introduction", "Docker & Friends"] }, + { title: "Images (Sample)", clips: ["Topic Introduction - Images"] }, +]; + +/** Mimics the real package layout: summary lists without "Video N.M:" prefixes, then chapter blocks. */ +export function packtIndexHtml(opts: { dropChapter?: number } = {}): string { + const summary = PACKT_SECTIONS.map((s, i) => ` +
  • + +

    Section ${i + 1}: ${s.title}

    +

    Video ${i + 1}.1: ${s.clips[0]}

    +
      ${s.clips.map(li).join("\n")}
    +
    Browse Videos
    +
    • `).join(""); + const chapters = PACKT_SECTIONS.map((s, i) => i + 1 === opts.dropChapter ? "" : ` + `).join(""); + return `Fake +

    Fake Docker Course

    +
      ${summary}
    ${chapters}
    `; +} + +export async function makePacktTree(root: string): Promise { + await mkdir(join(root, "package", "videos"), { recursive: true }); + await writeFile(join(root, "package", "index.html"), packtIndexHtml()); + await writeFile(join(root, "Video Package_User Guide.pdf"), "not used"); + await makeVideo(join(root, "package/videos/video1_1.mp4"), 2); + for (const n of ["video1_2", "video2_1"]) await Bun.write(join(root, "package/videos", `${n}.mp4`), Bun.file(join(root, "package/videos/video1_1.mp4"))); +} + +export async function makePacktZip(dir: string): Promise { + const tree = join(dir, "packt-tree"); + await makePacktTree(tree); + const zip = join(dir, "video_zip.zip"); + await zipDir(tree, zip); + return zip; +} + +/** Companion repo with folders that have spaces; S01 has a deck and a cheat sheet, S02 has code. */ +export async function makeRepoTree(root: string): Promise { + const top = join(root, "Course-Repo-main"); + await makePdf(join(top, "S01 - Intro to Docker", "Slides_Intro.pdf"), ["Repo deck one", "Repo deck two", "Repo deck three"]); + await makePdf(join(top, "S01 - Intro to Docker", "Docker_CheatSheet.pdf"), ["cheats"]); + await mkdir(join(top, "S02 - Images (Sample)", "app"), { recursive: true }); + await writeFile(join(top, "S02 - Images (Sample)", "app", "main.py"), "print('hi')\n"); + await writeFile(join(top, "S02 - Images (Sample)", "Ünïcode notes.txt"), "notes\n"); + await mkdir(join(top, "Bonus Extras"), { recursive: true }); + await writeFile(join(top, "Bonus Extras", "x.txt"), "x"); + await writeFile(join(top, "README.md"), "# repo\n"); + return top; +} + +export async function makeRepoZip(dir: string): Promise { + const tree = join(dir, "repo-tree"); + await makeRepoTree(tree); + const zip = join(dir, "repo.zip"); + await zipDir(tree, zip); + return zip; +} diff --git a/packages/feature-import/test/media.test.ts b/packages/feature-import/test/media.test.ts new file mode 100644 index 0000000..7704dc3 --- /dev/null +++ b/packages/feature-import/test/media.test.ts @@ -0,0 +1,38 @@ +import { expect, test } from "bun:test"; +import { join } from "node:path"; +import { readFile, stat } from "node:fs/promises"; +import { extractFrame, probeDuration, putFile } from "../src/media"; +import { renderPdf } from "../src/pdf"; +import { makePdf, makeVideo, PYTHON, tmp } from "./fixtures"; + +const tools = { ffmpeg: "ffmpeg", ffprobe: "ffprobe" }; + +test("duration and frame extraction on a tiny clip", async () => { + const d = await tmp(); + await makeVideo(join(d, "a.mp4"), 2); + expect(await probeDuration(tools, join(d, "a.mp4"))).toBe(2); + await extractFrame(tools, join(d, "a.mp4"), join(d, "out", "f.jpg")); + expect((await stat(join(d, "out", "f.jpg"))).size).toBeGreaterThan(100); +}); + +test("putFile copies and links without touching the original", async () => { + const d = await tmp(); + await makeVideo(join(d, "sp ace", "ö.mp4"), 1); + const before = await readFile(join(d, "sp ace", "ö.mp4")); + await putFile(join(d, "sp ace", "ö.mp4"), join(d, "c", "x.mp4"), "copy"); + await putFile(join(d, "sp ace", "ö.mp4"), join(d, "l", "x.mp4"), "link"); + expect(Buffer.compare(await readFile(join(d, "c", "x.mp4")), before)).toBe(0); + expect((await stat(join(d, "l", "x.mp4"))).ino).toBe((await stat(join(d, "sp ace", "ö.mp4"))).ino); +}); + +test("renderPdf writes page images and text", async () => { + const d = await tmp(); + await makePdf(join(d, "deck.pdf"), ["Hello alpha", "Second beta"]); + const seen: number[] = []; + const r = await renderPdf(PYTHON, join(d, "deck.pdf"), join(d, "out"), { onPage: (p) => seen.push(p) }); + expect(r.pages).toBe(2); + expect(seen).toEqual([1, 2]); + expect((await stat(join(d, "out", "p-002.jpg"))).size).toBeGreaterThan(100); + const text = JSON.parse(await readFile(join(d, "out", "text.json"), "utf8")); + expect(text[0].text).toContain("Hello alpha"); +}); diff --git a/packages/feature-import/test/util.test.ts b/packages/feature-import/test/util.test.ts new file mode 100644 index 0000000..7f04d61 --- /dev/null +++ b/packages/feature-import/test/util.test.ts @@ -0,0 +1,41 @@ +import { expect, test } from "bun:test"; +import { mkdtemp, mkdir, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { decodeEntities, naturalSort, slugify, uniqueId } from "../src/util/text"; +import { run } from "../src/util/exec"; +import { readRawManifest, writeManifest } from "../src/util/manifest"; + +test("slugify folds accents and falls back", () => { + expect(slugify("Mastering Docker Essentials - Hands-on DevOps!")).toBe("mastering-docker-essentials-hands-on-devops"); + expect(slugify("Café Ünïcode")).toBe("cafe-unicode"); + expect(slugify("日本語")).toBe("course"); +}); + +test("uniqueId appends a counter", () => { + expect(uniqueId("a", (x) => x === "a" || x === "a-2")).toBe("a-3"); + expect(uniqueId("a", () => false)).toBe("a"); +}); + +test("decodeEntities handles named and numeric entities", () => { + expect(decodeEntities("you'll & <b> "x" A")).toBe(`you'll & "x" A`); +}); + +test("naturalSort orders numbers numerically", () => { + expect(naturalSort(["10 b.mp4", "2 a.mp4", "1 z.mp4"])).toEqual(["1 z.mp4", "2 a.mp4", "10 b.mp4"]); +}); + +test("run reports stderr on failure", async () => { + expect(await run(["echo", "hi"])).toContain("hi"); + await expect(run(["sh", "-c", "echo oops >&2; exit 3"])).rejects.toThrow(/oops/); +}); + +test("writeManifest validates and round-trips", async () => { + const dir = await mkdtemp(join(tmpdir(), "stc-m-")); + await mkdir(join(dir, "c")); + const m = { schema: 1, id: "x", title: "T: with colon", visibility: "restricted", modules: [{ id: "m1", title: "S08 - Foo", lessons: [] }] }; + await writeManifest(join(dir, "c"), m); + expect((await readRawManifest(join(dir, "c")) as any).modules[0].title).toBe("S08 - Foo"); + await expect(writeManifest(join(dir, "c"), { ...m, id: "BAD ID" })).rejects.toThrow(/id/); + await writeFile(join(dir, "c", "marker"), ""); +});