feat(import): utilities, media tooling, PyMuPDF helper, synthetic fixtures

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

packages/feature-import/src/media.ts

@@ -0,0 +1,40 @@
+import { link, mkdir, copyFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import { run } from "./util/exec";
+
+export interface Tools { ffmpeg: string; ffprobe: string }
+
+export async function probeDuration(t: Tools, file: string): Promise<number | undefined> {
+  try {
+    const out = await run([t.ffprobe, "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", file]);
+    const s = Number(out.trim());
+    return Number.isFinite(s) && s > 0 ? Math.round(s) : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+/** One JPEG frame at `fraction` of the clip, 640px wide. */
+export async function extractFrame(t: Tools, file: string, out: string, fraction = 0.2): Promise<void> {
+  await mkdir(dirname(out), { recursive: true });
+  const dur = (await probeDuration(t, file)) ?? 0;
+  const at = Math.max(0, dur * fraction);
+  await run([t.ffmpeg, "-v", "error", "-y", "-ss", String(at), "-i", file, "-frames:v", "1", "-vf", "scale=640:-2", "-q:v", "4", out]);
+}
+
+/**
+ * Puts a large file into the course. "copy" uses `cp --reflink=auto` (a CoW clone where the filesystem supports it,
+ * a plain copy elsewhere); "link" makes a hard link and falls back to copying across filesystems.
+ * Either way the original is only read, and the course folder stays self-contained (the media route refuses symlinks that leave it).
+ */
+export async function putFile(src: string, dest: string, mode: "copy" | "link" = "copy"): Promise<void> {
+  await mkdir(dirname(dest), { recursive: true });
+  if (mode === "link") {
+    try { await link(src, dest); return; } catch { /* cross-device or unsupported: copy */ }
+  }
+  try {
+    await run(["cp", "--reflink=auto", "--", src, dest]);
+  } catch {
+    await copyFile(src, dest);
+  }
+}

packages/feature-import/src/pdf.ts

@@ -0,0 +1,23 @@
+import { join } from "node:path";
+import { run } from "./util/exec";
+
+export interface DeckRender { pages: number; title: string }
+
+const TOOL = join(import.meta.dir, "..", "py", "pdf_tool.py");
+
+/**
+ * Renders page images (p-001.jpg ...) and text.json into `outDir` with PyMuPDF in a subprocess,
+ * so the server process never holds a PDF engine. `python` is STC_PYTHON (default python3).
+ */
+export async function renderPdf(python: string, pdf: string, outDir: string, opts: { onPage?: (page: number, total: number) => void; signal?: AbortSignal } = {}): Promise<DeckRender> {
+  let last = "";
+  await run([python, "-I", TOOL, "render", pdf, outDir], {
+    signal: opts.signal,
+    onLine(l) {
+      const m = /^page (\d+)\/(\d+)$/.exec(l);
+      if (m) opts.onPage?.(Number(m[1]), Number(m[2]));
+      else if (l.trim()) last = l;
+    },
+  });
+  return JSON.parse(last) as DeckRender;
+}

packages/feature-import/src/util/exec.ts

@@ -0,0 +1,30 @@
+/** Runs a command (no shell). Resolves with stdout; rejects with the stderr tail on a non-zero exit. */
+export async function run(cmd: string[], opts: { signal?: AbortSignal; env?: Record<string, string | undefined>; onLine?: (line: string) => void } = {}): Promise<string> {
+  const proc = Bun.spawn(cmd, {
+    stdout: "pipe", stderr: "pipe", stdin: "ignore",
+    env: { ...process.env, LC_ALL: "C.UTF-8", ...opts.env } as Record<string, string>,
+    signal: opts.signal,
+  });
+  const out = (async () => {
+    let text = "";
+    let pending = "";
+    const dec = new TextDecoder();
+    for await (const chunk of proc.stdout) {
+      const s = dec.decode(chunk, { stream: true });
+      text += s;
+      if (opts.onLine) {
+        pending += s;
+        const lines = pending.split("\n");
+        pending = lines.pop()!;
+        for (const l of lines) opts.onLine(l);
+      }
+    }
+    return text;
+  })();
+  const [stdout, stderr, code] = await Promise.all([out, new Response(proc.stderr).text(), proc.exited]);
+  if (code !== 0) {
+    const tail = stderr.trim().split("\n").slice(-6).join("\n");
+    throw new Error(`${cmd[0]} exited with ${code}${tail ? `: ${tail}` : ""}`);
+  }
+  return stdout;
+}

packages/feature-import/src/util/manifest.ts

@@ -0,0 +1,21 @@
+import { readFile, rename, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { ManifestSchema } from "@stc/course-format";
+
+export type RawManifest = Record<string, any>;
+
+export async function readRawManifest(courseDir: string): Promise<RawManifest> {
+  return Bun.YAML.parse(await readFile(join(courseDir, "stcourse.yaml"), "utf8")) as RawManifest;
+}
+
+/** Validates against the course-format schema, then writes stcourse.yaml via temp file + rename. */
+export async function writeManifest(courseDir: string, manifest: RawManifest): Promise<void> {
+  const r = ManifestSchema.safeParse(manifest);
+  if (!r.success) {
+    throw new Error(`generated manifest is invalid: ${r.error.issues.map((i) => `${i.path.join(".") || "(root)"}: ${i.message}`).join("; ")}`);
+  }
+  const target = join(courseDir, "stcourse.yaml");
+  const tmp = `${target}.${process.pid}.tmp`;
+  await writeFile(tmp, Bun.YAML.stringify(manifest, null, 2) + "\n");
+  await rename(tmp, target);
+}

packages/feature-import/src/util/text.ts

@@ -0,0 +1,48 @@
+export function slugify(s: string): string {
+  const slug = s
+    .normalize("NFKD")
+    .replace(/[̀-ͯ]/g, "")
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .slice(0, 48)
+    .replace(/-+$/g, "");
+  return slug || "course";
+}
+
+export function uniqueId(base: string, taken: (id: string) => boolean): string {
+  if (!taken(base)) return base;
+  for (let i = 2; ; i++) if (!taken(`${base}-${i}`)) return `${base}-${i}`;
+}
+
+const NAMED: Record<string, string> = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " " };
+export function decodeEntities(s: string): string {
+  return s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e: string) => {
+    if (e[0] === "#") {
+      const code = e[1]!.toLowerCase() === "x" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);
+      return Number.isFinite(code) ? String.fromCodePoint(code) : m;
+    }
+    return NAMED[e.toLowerCase()] ?? m;
+  });
+}
+
+export function stripTags(s: string): string {
+  return decodeEntities(s.replace(/<[^>]*>/g, "")).replace(/\s+/g, " ").trim();
+}
+
+const collator = new Intl.Collator("en", { numeric: true, sensitivity: "base" });
+export function naturalSort(list: string[]): string[] {
+  return [...list].sort((a, b) => collator.compare(a, b) || (a < b ? -1 : a > b ? 1 : 0));
+}
+
+/** Lowercase alphanumeric words, for loose name matching. */
+export function words(s: string): string[] {
+  return s.normalize("NFKD").replace(/[̀-ͯ]/g, "").toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length > 1);
+}
+/** Share of the words of `a` that also occur in `b`. */
+export function containment(a: string, b: string): number {
+  const wa = words(a);
+  if (wa.length === 0) return 0;
+  const wb = new Set(words(b));
+  return wa.filter((w) => wb.has(w)).length / wa.length;
+}