diff --git a/packages/feature-import/src/jobs/code-repo.ts b/packages/feature-import/src/jobs/code-repo.ts new file mode 100644 index 0000000..78db8f6 --- /dev/null +++ b/packages/feature-import/src/jobs/code-repo.ts @@ -0,0 +1,134 @@ +import { cp, lstat, mkdir, readdir } from "node:fs/promises"; +import { basename, extname, join } from "node:path"; +import type { ImportResult, JobContext } from "@stc/contracts"; +import { finish, managedCourse, saveManifest, stageDir, type Deps } from "../context"; +import { attachDeck } from "../deck"; +import { putFile } from "../media"; +import { run } from "../util/exec"; +import { openSource } from "../util/source"; +import { containment } from "../util/text"; +import { optStr, parsePayload, withStage } from "./common"; + +const GIT_URL = /^(https?:\/\/|ssh:\/\/|git@)[^\s]+$/; +const SKIP_DIRS = new Set([".git", "node_modules", "__pycache__", ".venv"]); +const NUMBER = /^\s*(?:s|sec(?:tion)?|mod(?:ule)?)?\s*0*(\d+)\b\s*[-–—:.)_]*\s*/i; + +export function folderNumber(name: string): number | null { + const m = NUMBER.exec(name); + return m ? Number(m[1]) : null; +} +export function folderName(name: string): string { + return name.replace(NUMBER, "").trim(); +} + +function moduleNumber(m: { id: string; title: string }, index: number): number { + const byId = /^m0*(\d+)$/i.exec(m.id); + if (byId) return Number(byId[1]); + return folderNumber(m.title) ?? index + 1; +} + +/** One level of single-directory wrappers ("Repo-main/Repo-main/…") is skipped. */ +async function effectiveRoot(dir: string): Promise { + for (let i = 0; i < 3; i++) { + const entries = (await readdir(dir, { withFileTypes: true })).filter((e) => !e.name.startsWith(".") && e.name !== "__MACOSX"); + if (entries.length === 1 && entries[0]!.isDirectory()) dir = join(dir, entries[0]!.name); + else break; + } + return dir; +} + +async function findPdfs(dir: string, depth = 0, out: string[] = []): Promise { + for (const e of await readdir(dir, { withFileTypes: true })) { + if (SKIP_DIRS.has(e.name)) continue; + if (e.isDirectory() && depth < 2) await findPdfs(join(dir, e.name), depth + 1, out); + else if (e.isFile() && extname(e.name).toLowerCase() === ".pdf") out.push(join(dir, e.name)); + } + return out; +} + +export async function importCodeRepo(d: Deps, payload: unknown, jc: JobContext): Promise { + const { path, options } = parsePayload(payload, "code-repo"); + const warnings: string[] = []; + const course = await managedCourse(d, optStr(options, "courseId")); + const stage = await stageDir(d, "repo"); + + return withStage(stage, async () => { + let root: string; + if (GIT_URL.test(path)) { + await jc.reportProgress(0.05, "Cloning the repository"); + await run(["git", "clone", "--depth", "1", "--", path, join(stage, "repo")], { signal: jc.signal, env: { GIT_TERMINAL_PROMPT: "0" } }); + root = join(stage, "repo"); + } else if (/\.zip$/i.test(path)) { + await jc.reportProgress(0.05, "Unpacking the repository zip"); + const src = await openSource(path); + await src.extract(src.entries.filter((e) => !e.startsWith("__MACOSX/")), join(stage, "repo"), jc.signal); + root = join(stage, "repo"); + } else { + root = path; // an extracted folder is only read + } + root = await effectiveRoot(root); + + const modules: any[] = course.manifest.modules ?? []; + const folders = (await readdir(root, { withFileTypes: true })).filter((e) => e.isDirectory() && !SKIP_DIRS.has(e.name) && !e.name.startsWith(".")).map((e) => e.name).sort(); + const attachments: any[] = (course.manifest.attachments ??= []); + const claimed = new Set(); + let matched = 0; + + for (const [i, folder] of folders.entries()) { + await jc.reportProgress(0.1 + 0.8 * (i / Math.max(1, folders.length)), `Matching ${folder}`); + const num = folderNumber(folder); + const name = folderName(folder); + let mod = num !== null ? modules.find((m, idx) => moduleNumber(m, idx) === num) : undefined; + if (mod && name && containment(name, mod.title) < 0.3 && containment(mod.title, name) < 0.3) { + warnings.push(`"${folder}" was matched to module ${num} ("${mod.title}") by number only; the names differ`); + } + if (!mod && name) { + let best = 0; + for (const m of modules) { + const score = containment(name, m.title); + if (score > best) { best = score; mod = m; } + } + if (best < 0.6) mod = undefined; + } + if (!mod) { warnings.push(`folder "${folder}" matches no module and was skipped`); continue; } + if (claimed.has(mod.id)) { warnings.push(`folder "${folder}" also matches module "${mod.id}", which already got code; skipped`); continue; } + if (mod.code) { warnings.push(`module "${mod.id}" already has a code folder; "${folder}" skipped`); continue; } + claimed.add(mod.id); + matched++; + + const srcDir = join(root, folder); + const dest = `code/${folder}`; + await mkdir(join(course.dir, "code"), { recursive: true }); + await cp(srcDir, join(course.dir, dest), { + recursive: true, + filter: async (s) => { + const base = basename(s); + if (SKIP_DIRS.has(base)) return false; + const st = await lstat(s); + if (st.isSymbolicLink()) return false; // a link could point outside the course + return !(st.isFile() && extname(base).toLowerCase() === ".pdf"); + }, + }); + mod.code = { dir: dest }; + + for (const pdf of await findPdfs(srcDir)) { + const title = basename(pdf, ".pdf").replace(/_/g, " ").trim(); + if (/cheat|quick.?ref|reference|handout/i.test(basename(pdf))) { + let rel = `media/files/${basename(pdf)}`; + await putFile(pdf, join(course.dir, rel)); + attachments.push({ file: rel, title, module: mod.id }); + } else { + const r = await attachDeck(d, course, pdf, mod.id, { title }); + if (r.warning) warnings.push(r.warning); + } + } + } + if (attachments.length === 0) delete course.manifest.attachments; + if (matched === 0) warnings.push("no folder of the repository matched a module; nothing was attached"); + + await saveManifest(course.dir, course.manifest); + await finish(d, course.id, warnings); + await jc.reportProgress(1, `Attached code to ${matched} module(s)`); + return { courseId: course.id, manifestPath: join(course.dir, "stcourse.yaml"), warnings }; + }); +} diff --git a/packages/feature-import/test/code-repo.test.ts b/packages/feature-import/test/code-repo.test.ts new file mode 100644 index 0000000..b0112bd --- /dev/null +++ b/packages/feature-import/test/code-repo.test.ts @@ -0,0 +1,57 @@ +import { expect, test } from "bun:test"; +import { mkdir, readdir, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { folderName, folderNumber, importCodeRepo } from "../src/jobs/code-repo"; +import { importPacktVideo } from "../src/jobs/packt-video"; +import { makeEnv } from "./harness"; +import { makePacktZip, makeRepoZip, tmp } from "./fixtures"; + +test("folder names split into number and name", () => { + expect(folderNumber("S08 - Containerizing Applications")).toBe(8); + expect(folderName("S08 - Containerizing Applications")).toBe("Containerizing Applications"); + expect(folderNumber("Section 3: Images")).toBe(3); + expect(folderNumber("Bonus Extras")).toBeNull(); +}); + +async function target() { + const env = await makeEnv(); + const r = await importPacktVideo(env.deps, { path: await makePacktZip(await tmp()), options: { title: "Docker Test" } }, env.jc); + return { env, id: r.courseId }; +} + +test("zip repo: folders attach to modules by number, PDFs become deck and attachment", async () => { + const { env, id } = await target(); + const zip = await makeRepoZip(await tmp()); + const res = await importCodeRepo(env.deps, { kind: "code-repo", path: zip, options: { courseId: id } }, env.jc); + const c = env.deps.ctx.courses.get(id)!; + expect(c.status).toBe("ok"); + expect(c.manifest!.modules[0]!.code).toEqual({ dir: "code/S01 - Intro to Docker" }); + expect(c.manifest!.modules[1]!.code).toEqual({ dir: "code/S02 - Images (Sample)" }); + expect(env.deps.ctx.courses.resolvePath(id, "code/S02 - Images (Sample)/app/main.py")).toBeTruthy(); + expect(env.deps.ctx.courses.resolvePath(id, "code/S02 - Images (Sample)/Ünïcode notes.txt")).toBeTruthy(); + expect(c.manifest!.decks).toEqual([{ id: "d-slides-intro", file: "media/decks/Slides_Intro.pdf", title: "Slides Intro", attachTo: "m1" }]); + expect(c.manifest!.attachments).toEqual([{ file: "media/files/Docker_CheatSheet.pdf", title: "Docker CheatSheet", module: "m1" }]); + expect(await readdir(join(env.dataDir, "cache/decks", id, "d-slides-intro"))).toContain("p-003.jpg"); + expect(await readdir(join(c.root, "code/S01 - Intro to Docker"))).toEqual([]); // PDFs are not duplicated into code + expect(res.warnings).toEqual(['folder "Bonus Extras" matches no module and was skipped']); +}); + +test("an unnumbered folder matches a module by name; symlinks are not copied; the input is untouched", async () => { + const { env, id } = await target(); + const top = join(await tmp(), "repo"); + await mkdir(join(top, "Images Sample"), { recursive: true }); + await writeFile(join(top, "Images Sample", "a.txt"), "a"); + await symlink("/etc/passwd", join(top, "Images Sample", "link")); + const res = await importCodeRepo(env.deps, { path: top, options: { courseId: id } }, env.jc); + expect(res.warnings).toEqual([]); + const c = env.deps.ctx.courses.get(id)!; + expect(c.manifest!.modules[1]!.code).toEqual({ dir: "code/Images Sample" }); + expect(await readdir(join(c.root, "code/Images Sample"))).toEqual(["a.txt"]); + expect(await readdir(join(top, "Images Sample"))).toContain("link"); +}); + +test("rejects unmanaged courses and bad git URLs without running git", async () => { + const { env, id } = await target(); + await expect(importCodeRepo(env.deps, { path: "/x", options: { courseId: "sample" } }, env.jc)).rejects.toThrow(/only edits courses it created/); + await expect(importCodeRepo(env.deps, { path: "--upload-pack=touch /tmp/pwn", options: { courseId: id } }, env.jc)).rejects.toThrow(); +});