feat(import): code-repo job

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-30 03:16:00 +01:00
parent 9738da458d
commit 524b1b4645
2 changed files with 191 additions and 0 deletions
@@ -0,0 +1,134 @@
+import { cp, lstat, mkdir, readdir } from "node:fs/promises";
+import { basename, extname, join } from "node:path";
+import type { ImportResult, JobContext } from "@stc/contracts";
+import { finish, managedCourse, saveManifest, stageDir, type Deps } from "../context";
+import { attachDeck } from "../deck";
+import { putFile } from "../media";
+import { run } from "../util/exec";
+import { openSource } from "../util/source";
+import { containment } from "../util/text";
+import { optStr, parsePayload, withStage } from "./common";
+
+const GIT_URL = /^(https?:\/\/|ssh:\/\/|git@)[^\s]+$/;
+const SKIP_DIRS = new Set([".git", "node_modules", "__pycache__", ".venv"]);
+const NUMBER = /^\s*(?:s|sec(?:tion)?|mod(?:ule)?)?\s*0*(\d+)\b\s*[-–—:.)_]*\s*/i;
+
+export function folderNumber(name: string): number | null {
+  const m = NUMBER.exec(name);
+  return m ? Number(m[1]) : null;
+}
+export function folderName(name: string): string {
+  return name.replace(NUMBER, "").trim();
+}
+
+function moduleNumber(m: { id: string; title: string }, index: number): number {
+  const byId = /^m0*(\d+)$/i.exec(m.id);
+  if (byId) return Number(byId[1]);
+  return folderNumber(m.title) ?? index + 1;
+}
+
+/** One level of single-directory wrappers ("Repo-main/Repo-main/…") is skipped. */
+async function effectiveRoot(dir: string): Promise<string> {
+  for (let i = 0; i < 3; i++) {
+    const entries = (await readdir(dir, { withFileTypes: true })).filter((e) => !e.name.startsWith(".") && e.name !== "__MACOSX");
+    if (entries.length === 1 && entries[0]!.isDirectory()) dir = join(dir, entries[0]!.name);
+    else break;
+  }
+  return dir;
+}
+
+async function findPdfs(dir: string, depth = 0, out: string[] = []): Promise<string[]> {
+  for (const e of await readdir(dir, { withFileTypes: true })) {
+    if (SKIP_DIRS.has(e.name)) continue;
+    if (e.isDirectory() && depth < 2) await findPdfs(join(dir, e.name), depth + 1, out);
+    else if (e.isFile() && extname(e.name).toLowerCase() === ".pdf") out.push(join(dir, e.name));
+  }
+  return out;
+}
+
+export async function importCodeRepo(d: Deps, payload: unknown, jc: JobContext): Promise<ImportResult> {
+  const { path, options } = parsePayload(payload, "code-repo");
+  const warnings: string[] = [];
+  const course = await managedCourse(d, optStr(options, "courseId"));
+  const stage = await stageDir(d, "repo");
+
+  return withStage(stage, async () => {
+    let root: string;
+    if (GIT_URL.test(path)) {
+      await jc.reportProgress(0.05, "Cloning the repository");
+      await run(["git", "clone", "--depth", "1", "--", path, join(stage, "repo")], { signal: jc.signal, env: { GIT_TERMINAL_PROMPT: "0" } });
+      root = join(stage, "repo");
+    } else if (/\.zip$/i.test(path)) {
+      await jc.reportProgress(0.05, "Unpacking the repository zip");
+      const src = await openSource(path);
+      await src.extract(src.entries.filter((e) => !e.startsWith("__MACOSX/")), join(stage, "repo"), jc.signal);
+      root = join(stage, "repo");
+    } else {
+      root = path; // an extracted folder is only read
+    }
+    root = await effectiveRoot(root);
+
+    const modules: any[] = course.manifest.modules ?? [];
+    const folders = (await readdir(root, { withFileTypes: true })).filter((e) => e.isDirectory() && !SKIP_DIRS.has(e.name) && !e.name.startsWith(".")).map((e) => e.name).sort();
+    const attachments: any[] = (course.manifest.attachments ??= []);
+    const claimed = new Set<string>();
+    let matched = 0;
+
+    for (const [i, folder] of folders.entries()) {
+      await jc.reportProgress(0.1 + 0.8 * (i / Math.max(1, folders.length)), `Matching ${folder}`);
+      const num = folderNumber(folder);
+      const name = folderName(folder);
+      let mod = num !== null ? modules.find((m, idx) => moduleNumber(m, idx) === num) : undefined;
+      if (mod && name && containment(name, mod.title) < 0.3 && containment(mod.title, name) < 0.3) {
+        warnings.push(`"${folder}" was matched to module ${num} ("${mod.title}") by number only; the names differ`);
+      }
+      if (!mod && name) {
+        let best = 0;
+        for (const m of modules) {
+          const score = containment(name, m.title);
+          if (score > best) { best = score; mod = m; }
+        }
+        if (best < 0.6) mod = undefined;
+      }
+      if (!mod) { warnings.push(`folder "${folder}" matches no module and was skipped`); continue; }
+      if (claimed.has(mod.id)) { warnings.push(`folder "${folder}" also matches module "${mod.id}", which already got code; skipped`); continue; }
+      if (mod.code) { warnings.push(`module "${mod.id}" already has a code folder; "${folder}" skipped`); continue; }
+      claimed.add(mod.id);
+      matched++;
+
+      const srcDir = join(root, folder);
+      const dest = `code/${folder}`;
+      await mkdir(join(course.dir, "code"), { recursive: true });
+      await cp(srcDir, join(course.dir, dest), {
+        recursive: true,
+        filter: async (s) => {
+          const base = basename(s);
+          if (SKIP_DIRS.has(base)) return false;
+          const st = await lstat(s);
+          if (st.isSymbolicLink()) return false; // a link could point outside the course
+          return !(st.isFile() && extname(base).toLowerCase() === ".pdf");
+        },
+      });
+      mod.code = { dir: dest };
+
+      for (const pdf of await findPdfs(srcDir)) {
+        const title = basename(pdf, ".pdf").replace(/_/g, " ").trim();
+        if (/cheat|quick.?ref|reference|handout/i.test(basename(pdf))) {
+          let rel = `media/files/${basename(pdf)}`;
+          await putFile(pdf, join(course.dir, rel));
+          attachments.push({ file: rel, title, module: mod.id });
+        } else {
+          const r = await attachDeck(d, course, pdf, mod.id, { title });
+          if (r.warning) warnings.push(r.warning);
+        }
+      }
+    }
+    if (attachments.length === 0) delete course.manifest.attachments;
+    if (matched === 0) warnings.push("no folder of the repository matched a module; nothing was attached");
+
+    await saveManifest(course.dir, course.manifest);
+    await finish(d, course.id, warnings);
+    await jc.reportProgress(1, `Attached code to ${matched} module(s)`);
+    return { courseId: course.id, manifestPath: join(course.dir, "stcourse.yaml"), warnings };
+  });
+}
@@ -0,0 +1,57 @@
+import { expect, test } from "bun:test";
+import { mkdir, readdir, symlink, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { folderName, folderNumber, importCodeRepo } from "../src/jobs/code-repo";
+import { importPacktVideo } from "../src/jobs/packt-video";
+import { makeEnv } from "./harness";
+import { makePacktZip, makeRepoZip, tmp } from "./fixtures";
+
+test("folder names split into number and name", () => {
+  expect(folderNumber("S08 - Containerizing Applications")).toBe(8);
+  expect(folderName("S08 - Containerizing Applications")).toBe("Containerizing Applications");
+  expect(folderNumber("Section 3: Images")).toBe(3);
+  expect(folderNumber("Bonus Extras")).toBeNull();
+});
+
+async function target() {
+  const env = await makeEnv();
+  const r = await importPacktVideo(env.deps, { path: await makePacktZip(await tmp()), options: { title: "Docker Test" } }, env.jc);
+  return { env, id: r.courseId };
+}
+
+test("zip repo: folders attach to modules by number, PDFs become deck and attachment", async () => {
+  const { env, id } = await target();
+  const zip = await makeRepoZip(await tmp());
+  const res = await importCodeRepo(env.deps, { kind: "code-repo", path: zip, options: { courseId: id } }, env.jc);
+  const c = env.deps.ctx.courses.get(id)!;
+  expect(c.status).toBe("ok");
+  expect(c.manifest!.modules[0]!.code).toEqual({ dir: "code/S01 - Intro to Docker" });
+  expect(c.manifest!.modules[1]!.code).toEqual({ dir: "code/S02 - Images (Sample)" });
+  expect(env.deps.ctx.courses.resolvePath(id, "code/S02 - Images (Sample)/app/main.py")).toBeTruthy();
+  expect(env.deps.ctx.courses.resolvePath(id, "code/S02 - Images (Sample)/Ünïcode notes.txt")).toBeTruthy();
+  expect(c.manifest!.decks).toEqual([{ id: "d-slides-intro", file: "media/decks/Slides_Intro.pdf", title: "Slides Intro", attachTo: "m1" }]);
+  expect(c.manifest!.attachments).toEqual([{ file: "media/files/Docker_CheatSheet.pdf", title: "Docker CheatSheet", module: "m1" }]);
+  expect(await readdir(join(env.dataDir, "cache/decks", id, "d-slides-intro"))).toContain("p-003.jpg");
+  expect(await readdir(join(c.root, "code/S01 - Intro to Docker"))).toEqual([]); // PDFs are not duplicated into code
+  expect(res.warnings).toEqual(['folder "Bonus Extras" matches no module and was skipped']);
+});
+
+test("an unnumbered folder matches a module by name; symlinks are not copied; the input is untouched", async () => {
+  const { env, id } = await target();
+  const top = join(await tmp(), "repo");
+  await mkdir(join(top, "Images Sample"), { recursive: true });
+  await writeFile(join(top, "Images Sample", "a.txt"), "a");
+  await symlink("/etc/passwd", join(top, "Images Sample", "link"));
+  const res = await importCodeRepo(env.deps, { path: top, options: { courseId: id } }, env.jc);
+  expect(res.warnings).toEqual([]);
+  const c = env.deps.ctx.courses.get(id)!;
+  expect(c.manifest!.modules[1]!.code).toEqual({ dir: "code/Images Sample" });
+  expect(await readdir(join(c.root, "code/Images Sample"))).toEqual(["a.txt"]);
+  expect(await readdir(join(top, "Images Sample"))).toContain("link");
+});
+
+test("rejects unmanaged courses and bad git URLs without running git", async () => {
+  const { env, id } = await target();
+  await expect(importCodeRepo(env.deps, { path: "/x", options: { courseId: "sample" } }, env.jc)).rejects.toThrow(/only edits courses it created/);
+  await expect(importCodeRepo(env.deps, { path: "--upload-pack=touch /tmp/pwn", options: { courseId: id } }, env.jc)).rejects.toThrow();
+});