feat(import): code-repo job

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

packages/feature-import/src/jobs/code-repo.ts

@@ -0,0 +1,134 @@
+import { cp, lstat, mkdir, readdir } from "node:fs/promises";
+import { basename, extname, join } from "node:path";
+import type { ImportResult, JobContext } from "@stc/contracts";
+import { finish, managedCourse, saveManifest, stageDir, type Deps } from "../context";
+import { attachDeck } from "../deck";
+import { putFile } from "../media";
+import { run } from "../util/exec";
+import { openSource } from "../util/source";
+import { containment } from "../util/text";
+import { optStr, parsePayload, withStage } from "./common";
+
+const GIT_URL = /^(https?:\/\/|ssh:\/\/|git@)[^\s]+$/;
+const SKIP_DIRS = new Set([".git", "node_modules", "__pycache__", ".venv"]);
+const NUMBER = /^\s*(?:s|sec(?:tion)?|mod(?:ule)?)?\s*0*(\d+)\b\s*[-–—:.)_]*\s*/i;
+
+export function folderNumber(name: string): number | null {
+  const m = NUMBER.exec(name);
+  return m ? Number(m[1]) : null;
+}
+export function folderName(name: string): string {
+  return name.replace(NUMBER, "").trim();
+}
+
+function moduleNumber(m: { id: string; title: string }, index: number): number {
+  const byId = /^m0*(\d+)$/i.exec(m.id);
+  if (byId) return Number(byId[1]);
+  return folderNumber(m.title) ?? index + 1;
+}
+
+/** One level of single-directory wrappers ("Repo-main/Repo-main/…") is skipped. */
+async function effectiveRoot(dir: string): Promise<string> {
+  for (let i = 0; i < 3; i++) {
+    const entries = (await readdir(dir, { withFileTypes: true })).filter((e) => !e.name.startsWith(".") && e.name !== "__MACOSX");
+    if (entries.length === 1 && entries[0]!.isDirectory()) dir = join(dir, entries[0]!.name);
+    else break;
+  }
+  return dir;
+}
+
+async function findPdfs(dir: string, depth = 0, out: string[] = []): Promise<string[]> {
+  for (const e of await readdir(dir, { withFileTypes: true })) {
+    if (SKIP_DIRS.has(e.name)) continue;
+    if (e.isDirectory() && depth < 2) await findPdfs(join(dir, e.name), depth + 1, out);
+    else if (e.isFile() && extname(e.name).toLowerCase() === ".pdf") out.push(join(dir, e.name));
+  }
+  return out;
+}
+
+export async function importCodeRepo(d: Deps, payload: unknown, jc: JobContext): Promise<ImportResult> {
+  const { path, options } = parsePayload(payload, "code-repo");
+  const warnings: string[] = [];
+  const course = await managedCourse(d, optStr(options, "courseId"));
+  const stage = await stageDir(d, "repo");
+
+  return withStage(stage, async () => {
+    let root: string;
+    if (GIT_URL.test(path)) {
+      await jc.reportProgress(0.05, "Cloning the repository");
+      await run(["git", "clone", "--depth", "1", "--", path, join(stage, "repo")], { signal: jc.signal, env: { GIT_TERMINAL_PROMPT: "0" } });
+      root = join(stage, "repo");
+    } else if (/\.zip$/i.test(path)) {
+      await jc.reportProgress(0.05, "Unpacking the repository zip");
+      const src = await openSource(path);
+      await src.extract(src.entries.filter((e) => !e.startsWith("__MACOSX/")), join(stage, "repo"), jc.signal);
+      root = join(stage, "repo");
+    } else {
+      root = path; // an extracted folder is only read
+    }
+    root = await effectiveRoot(root);
+
+    const modules: any[] = course.manifest.modules ?? [];
+    const folders = (await readdir(root, { withFileTypes: true })).filter((e) => e.isDirectory() && !SKIP_DIRS.has(e.name) && !e.name.startsWith(".")).map((e) => e.name).sort();
+    const attachments: any[] = (course.manifest.attachments ??= []);
+    const claimed = new Set<string>();
+    let matched = 0;
+
+    for (const [i, folder] of folders.entries()) {
+      await jc.reportProgress(0.1 + 0.8 * (i / Math.max(1, folders.length)), `Matching ${folder}`);
+      const num = folderNumber(folder);
+      const name = folderName(folder);
+      let mod = num !== null ? modules.find((m, idx) => moduleNumber(m, idx) === num) : undefined;
+      if (mod && name && containment(name, mod.title) < 0.3 && containment(mod.title, name) < 0.3) {
+        warnings.push(`"${folder}" was matched to module ${num} ("${mod.title}") by number only; the names differ`);
+      }
+      if (!mod && name) {
+        let best = 0;
+        for (const m of modules) {
+          const score = containment(name, m.title);
+          if (score > best) { best = score; mod = m; }
+        }
+        if (best < 0.6) mod = undefined;
+      }
+      if (!mod) { warnings.push(`folder "${folder}" matches no module and was skipped`); continue; }
+      if (claimed.has(mod.id)) { warnings.push(`folder "${folder}" also matches module "${mod.id}", which already got code; skipped`); continue; }
+      if (mod.code) { warnings.push(`module "${mod.id}" already has a code folder; "${folder}" skipped`); continue; }
+      claimed.add(mod.id);
+      matched++;
+
+      const srcDir = join(root, folder);
+      const dest = `code/${folder}`;
+      await mkdir(join(course.dir, "code"), { recursive: true });
+      await cp(srcDir, join(course.dir, dest), {
+        recursive: true,
+        filter: async (s) => {
+          const base = basename(s);
+          if (SKIP_DIRS.has(base)) return false;
+          const st = await lstat(s);
+          if (st.isSymbolicLink()) return false; // a link could point outside the course
+          return !(st.isFile() && extname(base).toLowerCase() === ".pdf");
+        },
+      });
+      mod.code = { dir: dest };
+
+      for (const pdf of await findPdfs(srcDir)) {
+        const title = basename(pdf, ".pdf").replace(/_/g, " ").trim();
+        if (/cheat|quick.?ref|reference|handout/i.test(basename(pdf))) {
+          let rel = `media/files/${basename(pdf)}`;
+          await putFile(pdf, join(course.dir, rel));
+          attachments.push({ file: rel, title, module: mod.id });
+        } else {
+          const r = await attachDeck(d, course, pdf, mod.id, { title });
+          if (r.warning) warnings.push(r.warning);
+        }
+      }
+    }
+    if (attachments.length === 0) delete course.manifest.attachments;
+    if (matched === 0) warnings.push("no folder of the repository matched a module; nothing was attached");
+
+    await saveManifest(course.dir, course.manifest);
+    await finish(d, course.id, warnings);
+    await jc.reportProgress(1, `Attached code to ${matched} module(s)`);
+    return { courseId: course.id, manifestPath: join(course.dir, "stcourse.yaml"), warnings };
+  });
+}