feat(course-format): markdown extraction, discovery, loading, repository
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Lucas Winther
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
7a68800f6f
commit
c78c7ec8d8
@@ -0,0 +1,4 @@
|
||||
export * from "./schema";
|
||||
export * from "./markdown";
|
||||
export * from "./load";
|
||||
export * from "./repository";
|
||||
@@ -0,0 +1,208 @@
|
||||
import { readdir, readFile, realpath } from "node:fs/promises";
|
||||
import { realpathSync } from "node:fs";
|
||||
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
import { extractHints, extractSection } from "./markdown";
|
||||
import { parseManifest, type Lesson, type Manifest } from "./schema";
|
||||
|
||||
export type Diagnostic = {
|
||||
level: "error" | "warning";
|
||||
courseId?: string;
|
||||
lessonId?: string;
|
||||
path?: string;
|
||||
message: string;
|
||||
};
|
||||
|
||||
export interface LoadedCourse {
|
||||
id: string;
|
||||
manifestPath: string;
|
||||
root: string;
|
||||
manifest: Manifest | null;
|
||||
status: "ok" | "error";
|
||||
diagnostics: Diagnostic[];
|
||||
lessonIndex: Map<string, { moduleId: string; lesson: Lesson }>;
|
||||
}
|
||||
|
||||
const MANIFEST_NAME = "stcourse.yaml";
|
||||
const SKIP_DIRS = new Set(["node_modules", ".git"]);
|
||||
|
||||
/**
|
||||
* Absolute path of `rel` under `root` if its real path (symlinks followed) is inside the real root,
|
||||
* else null. A nonexistent target is also null.
|
||||
*/
|
||||
export function resolveInside(root: string, rel: string): string | null {
|
||||
try {
|
||||
const realRoot = realpathSync(root);
|
||||
const real = realpathSync(resolve(realRoot, rel));
|
||||
return real === realRoot || real.startsWith(realRoot + sep) ? real : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Finds stcourse.yaml at depth 0-2 under each dir. */
|
||||
export async function discoverManifests(courseDirs: string[]): Promise<string[]> {
|
||||
const found: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
async function walk(dir: string, depth: number) {
|
||||
const candidate = join(dir, MANIFEST_NAME);
|
||||
if (await Bun.file(candidate).exists()) {
|
||||
if (!seen.has(candidate)) {
|
||||
seen.add(candidate);
|
||||
found.push(candidate);
|
||||
}
|
||||
}
|
||||
if (depth >= 2) return;
|
||||
let entries;
|
||||
try {
|
||||
entries = await readdir(dir, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
entries.sort((a, b) => a.name.localeCompare(b.name));
|
||||
for (const e of entries) {
|
||||
if (e.isDirectory() && !SKIP_DIRS.has(e.name)) await walk(join(dir, e.name), depth + 1);
|
||||
}
|
||||
}
|
||||
for (const d of courseDirs) await walk(resolve(d), 0);
|
||||
return found;
|
||||
}
|
||||
|
||||
export async function loadCourse(manifestPath: string): Promise<LoadedCourse> {
|
||||
manifestPath = resolve(manifestPath);
|
||||
const dir = dirname(manifestPath);
|
||||
const diagnostics: Diagnostic[] = [];
|
||||
const course: LoadedCourse = {
|
||||
id: basename(dir),
|
||||
manifestPath,
|
||||
root: dir,
|
||||
manifest: null,
|
||||
status: "error",
|
||||
diagnostics,
|
||||
lessonIndex: new Map(),
|
||||
};
|
||||
const err = (message: string, extra: Partial<Diagnostic> = {}) =>
|
||||
diagnostics.push({ level: "error", courseId: course.id, message, ...extra });
|
||||
|
||||
let text: string;
|
||||
try {
|
||||
text = await readFile(manifestPath, "utf8");
|
||||
} catch (e) {
|
||||
err(`cannot read manifest ${manifestPath}: ${(e as Error).message}`, { path: manifestPath });
|
||||
return course;
|
||||
}
|
||||
const parsed = parseManifest(text);
|
||||
if (!parsed.ok) {
|
||||
for (const m of parsed.errors) err(`invalid manifest: ${m}`, { path: manifestPath });
|
||||
return course;
|
||||
}
|
||||
const manifest = parsed.manifest;
|
||||
course.manifest = manifest;
|
||||
course.id = manifest.id;
|
||||
course.root = resolve(dir, manifest.root ?? ".");
|
||||
for (const mod of manifest.modules) {
|
||||
for (const lesson of mod.lessons) course.lessonIndex.set(lesson.id, { moduleId: mod.id, lesson });
|
||||
}
|
||||
|
||||
try {
|
||||
course.root = await realpath(course.root);
|
||||
} catch {
|
||||
err(`course root does not exist or is unreadable: ${course.root}`, { path: course.root });
|
||||
return course;
|
||||
}
|
||||
const root = course.root;
|
||||
|
||||
const mdCache = new Map<string, string>();
|
||||
const readMd = async (rel: string): Promise<string | null> => {
|
||||
const abs = resolveInside(root, rel);
|
||||
if (!abs) return null;
|
||||
if (!mdCache.has(abs)) mdCache.set(abs, await Bun.file(abs).text());
|
||||
return mdCache.get(abs)!;
|
||||
};
|
||||
/** Reports an error and returns false when `rel` does not exist inside the root. */
|
||||
const requireFile = (rel: string, what: string, lessonId?: string): boolean => {
|
||||
if (resolveInside(root, rel)) return true;
|
||||
err(`${what} not found: ${rel}`, { lessonId, path: rel });
|
||||
return false;
|
||||
};
|
||||
|
||||
if (manifest.cover) requireFile(manifest.cover, "cover", undefined);
|
||||
if (manifest.mentor) requireFile(manifest.mentor, "mentor file", undefined);
|
||||
for (const d of manifest.decks ?? []) requireFile(d.file, `deck "${d.id}" file`);
|
||||
for (const a of manifest.attachments ?? []) requireFile(a.file, "attachment");
|
||||
|
||||
for (const mod of manifest.modules) {
|
||||
if (mod.code) requireFile(mod.code.dir, `module "${mod.id}" code dir`);
|
||||
for (const l of mod.lessons) {
|
||||
const id = l.id;
|
||||
let fileOk = true;
|
||||
if (l.file) fileOk = requireFile(l.file, "lesson file", id);
|
||||
if (l.transcript) requireFile(l.transcript, "transcript", id);
|
||||
for (const [track, t] of Object.entries(l.tracks ?? {})) {
|
||||
if (t.file) requireFile(t.file, `track "${track}" file`, id);
|
||||
}
|
||||
if (l.browser) {
|
||||
requireFile(l.browser.starter, "browser starter", id);
|
||||
requireFile(l.browser.tests, "browser tests", id);
|
||||
if (l.browser.solution) requireFile(l.browser.solution, "browser solution", id);
|
||||
}
|
||||
if (l.anchor) {
|
||||
const file = l.file ?? l.hints?.file;
|
||||
if (!file) {
|
||||
err(`anchor "${l.anchor}" given but the lesson has no file`, { lessonId: id });
|
||||
} else if (fileOk) {
|
||||
const md = await readMd(file);
|
||||
if (md !== null && extractSection(md, l.anchor) === null) {
|
||||
err(`anchor "${l.anchor}" not found in ${file}`, { lessonId: id, path: file });
|
||||
}
|
||||
}
|
||||
}
|
||||
if (l.hints) {
|
||||
if (requireFile(l.hints.file, "hints file", id)) {
|
||||
const md = await readMd(l.hints.file);
|
||||
if (md !== null && extractHints(md, l.hints.match).length === 0) {
|
||||
err(`no hint blocks matching "${l.hints.match}" in ${l.hints.file}`, { lessonId: id, path: l.hints.file });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
course.status = diagnostics.some((d) => d.level === "error") ? "error" : "ok";
|
||||
return course;
|
||||
}
|
||||
|
||||
function isOverlay(c: LoadedCourse): boolean {
|
||||
if (!c.manifest?.root) return false;
|
||||
const rel = relative(dirname(c.manifestPath), resolve(dirname(c.manifestPath), c.manifest.root));
|
||||
return rel === ".." || rel.startsWith(".." + sep) || isAbsolute(rel);
|
||||
}
|
||||
|
||||
/** Loads every manifest found under courseDirs. Duplicate ids: the overlay wins (spec section 13). */
|
||||
export async function loadAll(courseDirs: string[]): Promise<LoadedCourse[]> {
|
||||
const paths = await discoverManifests(courseDirs);
|
||||
const loaded = await Promise.all(paths.map((p) => loadCourse(p)));
|
||||
const winners = new Map<string, LoadedCourse>();
|
||||
const result: LoadedCourse[] = [];
|
||||
const dropped = new Set<LoadedCourse>();
|
||||
for (const c of loaded) {
|
||||
if (!c.manifest) {
|
||||
result.push(c);
|
||||
continue;
|
||||
}
|
||||
const prev = winners.get(c.id);
|
||||
if (!prev) {
|
||||
winners.set(c.id, c);
|
||||
continue;
|
||||
}
|
||||
const [win, lose] = isOverlay(c) && !isOverlay(prev) ? [c, prev] : [prev, c];
|
||||
win.diagnostics.push({
|
||||
level: "warning",
|
||||
courseId: win.id,
|
||||
path: lose.manifestPath,
|
||||
message: `shadows another manifest with the same id "${win.id}": ${lose.manifestPath}`,
|
||||
});
|
||||
dropped.add(lose);
|
||||
winners.set(c.id, win);
|
||||
}
|
||||
return loaded.filter((c) => !dropped.has(c));
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
/** GitHub-style heading slug: lowercase, markdown stripped, punctuation dropped, spaces to hyphens. */
|
||||
export function slugify(heading: string): string {
|
||||
return heading
|
||||
.replace(/!?\[([^\]]*)\]\([^)]*\)/g, "$1") // links and images keep their text
|
||||
.replace(/[`*~]/g, "") // code spans and emphasis markers
|
||||
.replace(/<[^>]+>/g, "") // inline html
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^\p{L}\p{N}\p{M}\s_-]/gu, "")
|
||||
.replace(/\s/g, "-");
|
||||
}
|
||||
|
||||
interface Heading {
|
||||
line: number;
|
||||
level: number;
|
||||
text: string;
|
||||
}
|
||||
|
||||
function headings(lines: string[]): Heading[] {
|
||||
const out: Heading[] = [];
|
||||
let fence: string | null = null;
|
||||
lines.forEach((line, i) => {
|
||||
const f = /^ {0,3}(`{3,}|~{3,})/.exec(line);
|
||||
if (f) {
|
||||
const marker = f[1]![0]!;
|
||||
if (fence === null) fence = marker;
|
||||
else if (fence === marker) fence = null;
|
||||
return;
|
||||
}
|
||||
if (fence !== null) return;
|
||||
const h = /^ {0,3}(#{1,6})\s+(.*?)\s*#*\s*$/.exec(line);
|
||||
if (h) out.push({ line: i, level: h[1]!.length, text: h[2]! });
|
||||
});
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Heading line plus everything up to the next heading of the same or a higher level. */
|
||||
export function extractSection(markdown: string, anchor: string): string | null {
|
||||
const lines = markdown.split(/\r?\n/);
|
||||
const hs = headings(lines);
|
||||
const seen = new Map<string, number>();
|
||||
for (let i = 0; i < hs.length; i++) {
|
||||
const h = hs[i]!;
|
||||
const base = slugify(h.text);
|
||||
const n = seen.get(base) ?? 0;
|
||||
seen.set(base, n + 1);
|
||||
const slug = n === 0 ? base : `${base}-${n}`; // GitHub numbers duplicates
|
||||
if (slug !== anchor) continue;
|
||||
const next = hs.slice(i + 1).find((x) => x.level <= h.level);
|
||||
return lines.slice(h.line, next ? next.line : lines.length).join("\n").trimEnd();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** `<details><summary>…</summary>…</details>` blocks whose summary starts with `match`, in document order. */
|
||||
export function extractHints(markdown: string, match: string): { label: string; body: string }[] {
|
||||
const out: { label: string; body: string }[] = [];
|
||||
const re = /<details>\s*<summary>([\s\S]*?)<\/summary>([\s\S]*?)<\/details>/g;
|
||||
for (const m of markdown.matchAll(re)) {
|
||||
const summary = m[1]!.trim();
|
||||
if (!summary.startsWith(match)) continue;
|
||||
const label = summary.slice(match.length).replace(/^\s*(?:—|-|\()\s*/, "").trim();
|
||||
out.push({ label, body: m[2]!.trim() });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { extname } from "node:path";
|
||||
import { loadAll, resolveInside, type LoadedCourse } from "./load";
|
||||
import { extractSection } from "./markdown";
|
||||
|
||||
export interface CourseRepository {
|
||||
list(): LoadedCourse[];
|
||||
get(id: string): LoadedCourse | null;
|
||||
/** Re-scans the course dirs. `id` is accepted for API symmetry; a rescan is cheap and keeps overlay shadowing correct. */
|
||||
reload(id?: string): Promise<void>;
|
||||
readLessonMarkdown(courseId: string, lessonId: string): Promise<string | null>;
|
||||
/** Absolute real path if inside the course root (symlinks followed), else null. */
|
||||
resolvePath(courseId: string, relPath: string): string | null;
|
||||
}
|
||||
|
||||
const MD = new Set([".md", ".markdown"]);
|
||||
|
||||
export async function createCourseRepository(courseDirs: string[]): Promise<CourseRepository> {
|
||||
let courses = await loadAll(courseDirs);
|
||||
const repo: CourseRepository = {
|
||||
list: () => courses,
|
||||
get: (id) => courses.find((c) => c.id === id) ?? null,
|
||||
async reload() {
|
||||
courses = await loadAll(courseDirs);
|
||||
},
|
||||
async readLessonMarkdown(courseId, lessonId) {
|
||||
const entry = repo.get(courseId)?.lessonIndex.get(lessonId);
|
||||
if (!entry) return null;
|
||||
const { lesson } = entry;
|
||||
const rel = lesson.type === "video" ? lesson.transcript : lesson.file;
|
||||
if (!rel || !MD.has(extname(rel).toLowerCase())) return null;
|
||||
const abs = repo.resolvePath(courseId, rel);
|
||||
if (!abs) return null;
|
||||
const text = await Bun.file(abs).text();
|
||||
return lesson.anchor ? extractSection(text, lesson.anchor) : text;
|
||||
},
|
||||
resolvePath(courseId, relPath) {
|
||||
const c = repo.get(courseId);
|
||||
return c ? resolveInside(c.root, relPath) : null;
|
||||
},
|
||||
};
|
||||
return repo;
|
||||
}
|
||||
Reference in New Issue
Block a user