feat(course-format): markdown extraction, discovery, loading, repository

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Lucas Winther
2026-10-30 02:51:26 +01:00
co-authored by Claude Opus 5.5
parent 7a68800f6f
commit c78c7ec8d8
7 changed files with 558 additions and 2 deletions
+2 -2
View File
@@ -1,3 +1,4 @@
import type { CourseRepository } from "@stc/course-format";
export type Role = "owner" | "author" | "learner";
export interface AuthUser { id: string; email: string; name: string; roles: Role[]; groups: string[] }
export interface RequestContext { user: AuthUser | null; params: Record<string, string>; url: URL }
@@ -13,9 +14,8 @@ export type JobStatus = "queued" | "running" | "done" | "failed";
export interface JobRecord { id: string; kind: string; status: JobStatus; progress: number; message: string | null; payload: unknown; result: unknown; error: string | null; createdAt: string; updatedAt: string }
export interface JobQueue { enqueue(kind: string, payload: unknown): Promise<string>; get(id: string): Promise<JobRecord | null>; list(kind?: string): Promise<JobRecord[]> }
export interface AuthService { userFromRequest(req: Request): Promise<AuthUser | null> }
// Placeholders until Task 4 (@stc/db) and Task 7 (@stc/course-format) exist; switch to the real imports in Task 7.
// Placeholder until Task 7 (@stc/db) exists.
type Db = unknown;
type CourseRepository = unknown;
export interface ServerContext {
config: AppConfig; db: Db; courses: CourseRepository;
jobs: JobQueue; auth: AuthService;
+4
View File
@@ -0,0 +1,4 @@
export * from "./schema";
export * from "./markdown";
export * from "./load";
export * from "./repository";
+208
View File
@@ -0,0 +1,208 @@
import { readdir, readFile, realpath } from "node:fs/promises";
import { realpathSync } from "node:fs";
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
import { extractHints, extractSection } from "./markdown";
import { parseManifest, type Lesson, type Manifest } from "./schema";
export type Diagnostic = {
level: "error" | "warning";
courseId?: string;
lessonId?: string;
path?: string;
message: string;
};
export interface LoadedCourse {
id: string;
manifestPath: string;
root: string;
manifest: Manifest | null;
status: "ok" | "error";
diagnostics: Diagnostic[];
lessonIndex: Map<string, { moduleId: string; lesson: Lesson }>;
}
const MANIFEST_NAME = "stcourse.yaml";
const SKIP_DIRS = new Set(["node_modules", ".git"]);
/**
* Absolute path of `rel` under `root` if its real path (symlinks followed) is inside the real root,
* else null. A nonexistent target is also null.
*/
export function resolveInside(root: string, rel: string): string | null {
try {
const realRoot = realpathSync(root);
const real = realpathSync(resolve(realRoot, rel));
return real === realRoot || real.startsWith(realRoot + sep) ? real : null;
} catch {
return null;
}
}
/** Finds stcourse.yaml at depth 0-2 under each dir. */
export async function discoverManifests(courseDirs: string[]): Promise<string[]> {
const found: string[] = [];
const seen = new Set<string>();
async function walk(dir: string, depth: number) {
const candidate = join(dir, MANIFEST_NAME);
if (await Bun.file(candidate).exists()) {
if (!seen.has(candidate)) {
seen.add(candidate);
found.push(candidate);
}
}
if (depth >= 2) return;
let entries;
try {
entries = await readdir(dir, { withFileTypes: true });
} catch {
return;
}
entries.sort((a, b) => a.name.localeCompare(b.name));
for (const e of entries) {
if (e.isDirectory() && !SKIP_DIRS.has(e.name)) await walk(join(dir, e.name), depth + 1);
}
}
for (const d of courseDirs) await walk(resolve(d), 0);
return found;
}
export async function loadCourse(manifestPath: string): Promise<LoadedCourse> {
manifestPath = resolve(manifestPath);
const dir = dirname(manifestPath);
const diagnostics: Diagnostic[] = [];
const course: LoadedCourse = {
id: basename(dir),
manifestPath,
root: dir,
manifest: null,
status: "error",
diagnostics,
lessonIndex: new Map(),
};
const err = (message: string, extra: Partial<Diagnostic> = {}) =>
diagnostics.push({ level: "error", courseId: course.id, message, ...extra });
let text: string;
try {
text = await readFile(manifestPath, "utf8");
} catch (e) {
err(`cannot read manifest ${manifestPath}: ${(e as Error).message}`, { path: manifestPath });
return course;
}
const parsed = parseManifest(text);
if (!parsed.ok) {
for (const m of parsed.errors) err(`invalid manifest: ${m}`, { path: manifestPath });
return course;
}
const manifest = parsed.manifest;
course.manifest = manifest;
course.id = manifest.id;
course.root = resolve(dir, manifest.root ?? ".");
for (const mod of manifest.modules) {
for (const lesson of mod.lessons) course.lessonIndex.set(lesson.id, { moduleId: mod.id, lesson });
}
try {
course.root = await realpath(course.root);
} catch {
err(`course root does not exist or is unreadable: ${course.root}`, { path: course.root });
return course;
}
const root = course.root;
const mdCache = new Map<string, string>();
const readMd = async (rel: string): Promise<string | null> => {
const abs = resolveInside(root, rel);
if (!abs) return null;
if (!mdCache.has(abs)) mdCache.set(abs, await Bun.file(abs).text());
return mdCache.get(abs)!;
};
/** Reports an error and returns false when `rel` does not exist inside the root. */
const requireFile = (rel: string, what: string, lessonId?: string): boolean => {
if (resolveInside(root, rel)) return true;
err(`${what} not found: ${rel}`, { lessonId, path: rel });
return false;
};
if (manifest.cover) requireFile(manifest.cover, "cover", undefined);
if (manifest.mentor) requireFile(manifest.mentor, "mentor file", undefined);
for (const d of manifest.decks ?? []) requireFile(d.file, `deck "${d.id}" file`);
for (const a of manifest.attachments ?? []) requireFile(a.file, "attachment");
for (const mod of manifest.modules) {
if (mod.code) requireFile(mod.code.dir, `module "${mod.id}" code dir`);
for (const l of mod.lessons) {
const id = l.id;
let fileOk = true;
if (l.file) fileOk = requireFile(l.file, "lesson file", id);
if (l.transcript) requireFile(l.transcript, "transcript", id);
for (const [track, t] of Object.entries(l.tracks ?? {})) {
if (t.file) requireFile(t.file, `track "${track}" file`, id);
}
if (l.browser) {
requireFile(l.browser.starter, "browser starter", id);
requireFile(l.browser.tests, "browser tests", id);
if (l.browser.solution) requireFile(l.browser.solution, "browser solution", id);
}
if (l.anchor) {
const file = l.file ?? l.hints?.file;
if (!file) {
err(`anchor "${l.anchor}" given but the lesson has no file`, { lessonId: id });
} else if (fileOk) {
const md = await readMd(file);
if (md !== null && extractSection(md, l.anchor) === null) {
err(`anchor "${l.anchor}" not found in ${file}`, { lessonId: id, path: file });
}
}
}
if (l.hints) {
if (requireFile(l.hints.file, "hints file", id)) {
const md = await readMd(l.hints.file);
if (md !== null && extractHints(md, l.hints.match).length === 0) {
err(`no hint blocks matching "${l.hints.match}" in ${l.hints.file}`, { lessonId: id, path: l.hints.file });
}
}
}
}
}
course.status = diagnostics.some((d) => d.level === "error") ? "error" : "ok";
return course;
}
function isOverlay(c: LoadedCourse): boolean {
if (!c.manifest?.root) return false;
const rel = relative(dirname(c.manifestPath), resolve(dirname(c.manifestPath), c.manifest.root));
return rel === ".." || rel.startsWith(".." + sep) || isAbsolute(rel);
}
/** Loads every manifest found under courseDirs. Duplicate ids: the overlay wins (spec section 13). */
export async function loadAll(courseDirs: string[]): Promise<LoadedCourse[]> {
const paths = await discoverManifests(courseDirs);
const loaded = await Promise.all(paths.map((p) => loadCourse(p)));
const winners = new Map<string, LoadedCourse>();
const result: LoadedCourse[] = [];
const dropped = new Set<LoadedCourse>();
for (const c of loaded) {
if (!c.manifest) {
result.push(c);
continue;
}
const prev = winners.get(c.id);
if (!prev) {
winners.set(c.id, c);
continue;
}
const [win, lose] = isOverlay(c) && !isOverlay(prev) ? [c, prev] : [prev, c];
win.diagnostics.push({
level: "warning",
courseId: win.id,
path: lose.manifestPath,
message: `shadows another manifest with the same id "${win.id}": ${lose.manifestPath}`,
});
dropped.add(lose);
winners.set(c.id, win);
}
return loaded.filter((c) => !dropped.has(c));
}
+66
View File
@@ -0,0 +1,66 @@
/** GitHub-style heading slug: lowercase, markdown stripped, punctuation dropped, spaces to hyphens. */
export function slugify(heading: string): string {
return heading
.replace(/!?\[([^\]]*)\]\([^)]*\)/g, "$1") // links and images keep their text
.replace(/[`*~]/g, "") // code spans and emphasis markers
.replace(/<[^>]+>/g, "") // inline html
.trim()
.toLowerCase()
.replace(/[^\p{L}\p{N}\p{M}\s_-]/gu, "")
.replace(/\s/g, "-");
}
interface Heading {
line: number;
level: number;
text: string;
}
function headings(lines: string[]): Heading[] {
const out: Heading[] = [];
let fence: string | null = null;
lines.forEach((line, i) => {
const f = /^ {0,3}(`{3,}|~{3,})/.exec(line);
if (f) {
const marker = f[1]![0]!;
if (fence === null) fence = marker;
else if (fence === marker) fence = null;
return;
}
if (fence !== null) return;
const h = /^ {0,3}(#{1,6})\s+(.*?)\s*#*\s*$/.exec(line);
if (h) out.push({ line: i, level: h[1]!.length, text: h[2]! });
});
return out;
}
/** Heading line plus everything up to the next heading of the same or a higher level. */
export function extractSection(markdown: string, anchor: string): string | null {
const lines = markdown.split(/\r?\n/);
const hs = headings(lines);
const seen = new Map<string, number>();
for (let i = 0; i < hs.length; i++) {
const h = hs[i]!;
const base = slugify(h.text);
const n = seen.get(base) ?? 0;
seen.set(base, n + 1);
const slug = n === 0 ? base : `${base}-${n}`; // GitHub numbers duplicates
if (slug !== anchor) continue;
const next = hs.slice(i + 1).find((x) => x.level <= h.level);
return lines.slice(h.line, next ? next.line : lines.length).join("\n").trimEnd();
}
return null;
}
/** `<details><summary>…</summary>…</details>` blocks whose summary starts with `match`, in document order. */
export function extractHints(markdown: string, match: string): { label: string; body: string }[] {
const out: { label: string; body: string }[] = [];
const re = /<details>\s*<summary>([\s\S]*?)<\/summary>([\s\S]*?)<\/details>/g;
for (const m of markdown.matchAll(re)) {
const summary = m[1]!.trim();
if (!summary.startsWith(match)) continue;
const label = summary.slice(match.length).replace(/^\s*(?:—|-|\()\s*/, "").trim();
out.push({ label, body: m[2]!.trim() });
}
return out;
}
+42
View File
@@ -0,0 +1,42 @@
import { extname } from "node:path";
import { loadAll, resolveInside, type LoadedCourse } from "./load";
import { extractSection } from "./markdown";
export interface CourseRepository {
list(): LoadedCourse[];
get(id: string): LoadedCourse | null;
/** Re-scans the course dirs. `id` is accepted for API symmetry; a rescan is cheap and keeps overlay shadowing correct. */
reload(id?: string): Promise<void>;
readLessonMarkdown(courseId: string, lessonId: string): Promise<string | null>;
/** Absolute real path if inside the course root (symlinks followed), else null. */
resolvePath(courseId: string, relPath: string): string | null;
}
const MD = new Set([".md", ".markdown"]);
export async function createCourseRepository(courseDirs: string[]): Promise<CourseRepository> {
let courses = await loadAll(courseDirs);
const repo: CourseRepository = {
list: () => courses,
get: (id) => courses.find((c) => c.id === id) ?? null,
async reload() {
courses = await loadAll(courseDirs);
},
async readLessonMarkdown(courseId, lessonId) {
const entry = repo.get(courseId)?.lessonIndex.get(lessonId);
if (!entry) return null;
const { lesson } = entry;
const rel = lesson.type === "video" ? lesson.transcript : lesson.file;
if (!rel || !MD.has(extname(rel).toLowerCase())) return null;
const abs = repo.resolvePath(courseId, rel);
if (!abs) return null;
const text = await Bun.file(abs).text();
return lesson.anchor ? extractSection(text, lesson.anchor) : text;
},
resolvePath(courseId, relPath) {
const c = repo.get(courseId);
return c ? resolveInside(c.root, relPath) : null;
},
};
return repo;
}
+147
View File
@@ -0,0 +1,147 @@
import { afterAll, describe, expect, test } from "bun:test";
import { mkdtemp, rm, symlink } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { createCourseRepository, discoverManifests, loadAll, loadCourse } from "../src/index";
const sampleDir = resolve(import.meta.dir, "../../../fixtures/courses/sample");
const tmps: string[] = [];
async function tmp(): Promise<string> {
const d = await mkdtemp(join(tmpdir(), "stc-"));
tmps.push(d);
return d;
}
afterAll(() => Promise.all(tmps.map((d) => rm(d, { recursive: true, force: true }))));
const manifest = (id: string, extra: string, lessons: string) =>
`schema: 1\nid: ${id}\ntitle: T\n${extra}modules:\n - id: m\n title: M\n lessons:\n${lessons}\n`;
describe("sample fixture", () => {
test("loads the sample fixture with no errors", async () => {
const c = await loadCourse(join(sampleDir, "stcourse.yaml"));
expect(c.diagnostics).toEqual([]);
expect(c.status).toBe("ok");
expect(c.id).toBe("sample");
expect([...c.lessonIndex.keys()]).toEqual(["m1-intro", "m1-video", "m1-code", "m2-text", "m2-ex-add"]);
});
test("repository reads lesson markdown and sections", async () => {
const repo = await createCourseRepository([sampleDir]);
expect(repo.list().map((c) => c.id)).toEqual(["sample"]);
expect(await repo.readLessonMarkdown("sample", "m1-intro")).toContain("# Introduction");
const ex = await repo.readLessonMarkdown("sample", "m2-ex-add");
expect(ex!.startsWith("### Exercise 2.1")).toBe(true);
expect(await repo.readLessonMarkdown("sample", "nope")).toBeNull();
expect(repo.get("nope")).toBeNull();
});
});
describe("discovery", () => {
test("finds manifests at depth 0 to 2 only", async () => {
const d = await tmp();
for (const p of ["stcourse.yaml", "a/stcourse.yaml", "a/b/stcourse.yaml", "a/b/c/stcourse.yaml"]) {
await Bun.write(join(d, p), "x");
}
const found = (await discoverManifests([d])).map((p) => p.slice(d.length + 1));
expect(found.sort()).toEqual(["a/b/stcourse.yaml", "a/stcourse.yaml", "stcourse.yaml"]);
});
});
describe("overlays and errors", () => {
test("overlay with missing root gives status error", async () => {
const d = await tmp();
const missing = join(d, "gone-repo");
await Bun.write(join(d, "overlay/stcourse.yaml"), manifest("broken", `root: ${missing}\n`, " - { id: l, title: L, type: text, file: a.md }"));
await Bun.write(join(d, "ok/a.md"), "# A\n");
await Bun.write(join(d, "ok/stcourse.yaml"), manifest("fine", "", " - { id: l, title: L, type: text, file: a.md }"));
const all = await loadAll([d]);
expect(all.map((c) => c.id).sort()).toEqual(["broken", "fine"]);
const broken = all.find((c) => c.id === "broken")!;
expect(broken.status).toBe("error");
expect(broken.diagnostics.some((x) => x.level === "error" && x.message.includes(missing))).toBe(true);
expect(all.find((c) => c.id === "fine")!.status).toBe("ok");
});
test("missing lesson file, anchor and hints each produce one error diagnostic naming lessonId", async () => {
const d = await tmp();
await Bun.write(join(d, "a.md"), "# A\n\n## Real\n\ntext\n");
const lessons = [
" - { id: no-file, title: L, type: text, file: missing.md }",
" - { id: no-anchor, title: L, type: text, file: a.md, anchor: ghost }",
" - { id: no-hints, title: L, type: exercise, file: a.md, anchor: real, hints: { file: a.md, match: 'Exercise 1' } }",
].join("\n");
await Bun.write(join(d, "stcourse.yaml"), manifest("bad", "", lessons));
const c = await loadCourse(join(d, "stcourse.yaml"));
expect(c.status).toBe("error");
const errs = c.diagnostics.filter((x) => x.level === "error");
expect(errs.map((e) => e.lessonId)).toEqual(["no-file", "no-anchor", "no-hints"]);
expect(errs[0]!.path).toBe("missing.md");
expect(errs[1]!.message).toContain("ghost");
expect(errs[2]!.message).toContain("Exercise 1");
});
test("invalid manifest gives an error course and does not throw", async () => {
const d = await tmp();
await Bun.write(join(d, "stcourse.yaml"), "schema: 2\nid: x\n");
const c = await loadCourse(join(d, "stcourse.yaml"));
expect(c.status).toBe("error");
expect(c.manifest).toBeNull();
expect(c.diagnostics.length).toBeGreaterThan(0);
});
test("handles spaces and unicode in paths", async () => {
const d = await tmp();
const rel = "S08 - Containerizing Applications/Æble notes.md";
await Bun.write(join(d, rel), "# Æble\n");
await Bun.write(join(d, "stcourse.yaml"), manifest("uni", "", ` - { id: l, title: L, type: text, file: "${rel}" }`));
const repo = await createCourseRepository([d]);
expect(repo.get("uni")!.status).toBe("ok");
expect(await repo.readLessonMarkdown("uni", "l")).toBe("# Æble\n");
expect(repo.resolvePath("uni", rel)).toContain("Æble notes.md");
});
test("overlay wins over in-repo manifest with same id", async () => {
const d = await tmp();
const lesson = " - { id: l, title: L, type: text, file: a.md }";
await Bun.write(join(d, "repo/a.md"), "# A\n");
await Bun.write(join(d, "repo/stcourse.yaml"), manifest("dup", "", lesson.replace("L,", "InRepo,")));
await Bun.write(join(d, "overlays/dup/stcourse.yaml"), manifest("dup", `root: ${join(d, "repo")}\n`, lesson.replace("L,", "Overlay,")));
const all = await loadAll([d]);
const dup = all.filter((c) => c.id === "dup");
expect(dup).toHaveLength(1);
expect(dup[0]!.manifestPath).toBe(join(d, "overlays/dup/stcourse.yaml"));
const warn = dup[0]!.diagnostics.find((x) => x.level === "warning");
expect(warn!.path).toBe(join(d, "repo/stcourse.yaml"));
});
test("overlay wins regardless of discovery order", async () => {
const d = await tmp();
const lesson = " - { id: l, title: L, type: text, file: a.md }";
await Bun.write(join(d, "a-repo/a.md"), "# A\n");
await Bun.write(join(d, "a-repo/stcourse.yaml"), manifest("dup", "", lesson));
await Bun.write(join(d, "z-overlay/stcourse.yaml"), manifest("dup", `root: ${join(d, "a-repo")}\n`, lesson));
const all = await loadAll([d]);
expect(all).toHaveLength(1);
expect(all[0]!.manifestPath).toBe(join(d, "z-overlay/stcourse.yaml"));
});
});
describe("resolvePath", () => {
test("rejects ../, absolute paths and symlink escape", async () => {
const d = await tmp();
await Bun.write(join(d, "root/a.md"), "# A\n");
await Bun.write(join(d, "outside.txt"), "secret");
await symlink("/etc", join(d, "root/etc-link"));
await symlink(join(d, "outside.txt"), join(d, "root/out-link.txt"));
await Bun.write(join(d, "root/stcourse.yaml"), manifest("r", "", " - { id: l, title: L, type: text, file: a.md }"));
const repo = await createCourseRepository([join(d, "root")]);
expect(repo.resolvePath("r", "a.md")).not.toBeNull();
expect(repo.resolvePath("r", "../outside.txt")).toBeNull();
expect(repo.resolvePath("r", join(d, "outside.txt"))).toBeNull();
expect(repo.resolvePath("r", "etc-link/passwd")).toBeNull();
expect(repo.resolvePath("r", "etc-link")).toBeNull();
expect(repo.resolvePath("r", "out-link.txt")).toBeNull();
expect(repo.resolvePath("r", "missing.md")).toBeNull();
expect(repo.resolvePath("nope", "a.md")).toBeNull();
});
});
@@ -0,0 +1,89 @@
import { describe, expect, test } from "bun:test";
import { extractHints, extractSection, slugify } from "../src/markdown";
describe("slugify", () => {
test("follows GitHub's algorithm", () => {
expect(slugify("Exercise 3.1 — `move`")).toBe("exercise-31--move");
expect(slugify("Why this module")).toBe("why-this-module");
expect(slugify("Rules as functions `State → State`")).toBe("rules-as-functions-state--state");
expect(slugify("**Bold** and [a link](http://x.y)")).toBe("bold-and-a-link");
});
});
const sample = `# Top
intro
## First
one
### Middle
middle body
\`\`\`sh
# not a heading
\`\`\`
#### Deeper
deep body
### Sibling
sibling body
## Last
end
`;
describe("extractSection", () => {
test("returns only the middle section, including deeper headings", () => {
const s = extractSection(sample, "middle");
expect(s).not.toBeNull();
expect(s!.startsWith("### Middle")).toBe(true);
expect(s).toContain("# not a heading");
expect(s).toContain("deep body");
expect(s).not.toContain("sibling body");
expect(s).not.toContain("one");
});
test("returns null for an unknown anchor", () => {
expect(extractSection(sample, "nope")).toBeNull();
});
test("a section runs to the end of file when nothing follows", () => {
expect(extractSection(sample, "last")).toBe("## Last\n\nend");
});
});
describe("extractHints", () => {
const md = `## Hints
<details><summary>Exercise 3.1 (move) — nudge</summary>
Think about direction.
</details>
<details><summary>Exercise 3.1 (move) — stronger hint</summary>
Add the delta.
</details>
<details><summary>Exercise 3.2 (inBounds) — nudge</summary>
Other.
</details>
`;
test("returns matching blocks in order with trimmed labels", () => {
const h = extractHints(md, "Exercise 3.1 (move)");
expect(h.map((x) => x.label)).toEqual(["nudge", "stronger hint"]);
expect(h[0]!.body).toBe("Think about direction.");
});
test("returns [] when nothing matches", () => {
expect(extractHints(md, "Exercise 9.9")).toEqual([]);
});
});