Lucas WintherandClaude Opus 5.5
ead7327bb1
read_file and read_lesson URL-decode, posix-normalise, reject .. and absolute paths, resolve through CourseRepository.resolvePath (symlinks followed), then guard the real course-relative path and compare real paths with every manifest browser.solution. Hidden files are refused too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
350 lines
16 KiB
TypeScript
350 lines
16 KiB
TypeScript
import { realpathSync } from "node:fs";
|
|
import { readdir, stat } from "node:fs/promises";
|
|
import { basename, extname, join, posix as posixPath, relative, sep } from "node:path";
|
|
import { slugify, type CourseRepository, type LoadedCourse } from "@stc/course-format";
|
|
import { Bm25 } from "./bm25";
|
|
|
|
export type ChunkKind = "lesson" | "transcript" | "markdown" | "code" | "deck" | "note";
|
|
export interface Chunk {
|
|
path: string; // course-relative path (for cache text: the deck or lesson it belongs to)
|
|
kind: ChunkKind;
|
|
text: string;
|
|
heading?: string;
|
|
lessonId?: string;
|
|
lessonTitle?: string;
|
|
}
|
|
export interface CourseIndex { size: number; search(query: string, k: number, extra?: Chunk[]): Chunk[] }
|
|
|
|
const SKIP_DIRS = new Set([".git", "node_modules", "dist", "build", "target", ".astro", ".next", "coverage", "bin", "obj", ".venv", "__pycache__"]);
|
|
const MD = new Set([".md", ".markdown"]);
|
|
const CODE = new Set([".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx", ".py", ".rs", ".scala", ".sc", ".fs", ".fsx", ".go", ".java", ".kt", ".rb", ".php", ".c", ".h", ".cpp", ".cs", ".sh", ".bash", ".ps1", ".sql", ".yaml", ".yml", ".toml", ".json", ".tf", ".html", ".css"]);
|
|
const CODE_NAMES = new Set(["Dockerfile", "Makefile", "docker-compose.yml", "Containerfile"]);
|
|
const CACHE_TEXT = new Set([".txt", ".md", ".vtt", ".srt"]);
|
|
// Lock files, the manifest, and instructions written for AI agents (the mentor file goes into the system
|
|
// prompt only while mentor mode is on, so it must not leak back in through retrieval).
|
|
const SKIP_FILES = new Set(["package-lock.json", "bun.lock", "yarn.lock", "pnpm-lock.yaml", "Cargo.lock", "stcourse.yaml", "AGENTS.md", "CLAUDE.md", "GEMINI.md"]);
|
|
const MAX_FILE = 200_000;
|
|
const MAX_FILES = 2_000;
|
|
const MAX_CHUNK = 1_500;
|
|
const SOLUTION_SEGMENTS = new Set(["solution", "solutions", "reference"]);
|
|
|
|
const posix = (p: string) => p.split(sep).join("/");
|
|
|
|
/**
|
|
* Files only authors may see: manifest `browser.solution` files and anything under a `solution`, `solutions`
|
|
* or `reference` folder (Pure Snake `js/solutions/`, Systemic Design `reference/` and `expansion/solution/`).
|
|
* The assistant never retrieves or reads them. `rel` is normalised first; for paths that come from a user or a
|
|
* model, use `canonicalCoursePath` + `isGuardedFile`, which also follow symlinks.
|
|
*/
|
|
export function isSolutionPath(course: LoadedCourse, rel: string): boolean {
|
|
const p = normalizeRel(rel);
|
|
if (p === null) return true; // not a course-relative path at all: refuse
|
|
if (p.split("/").slice(0, -1).some((s) => SOLUTION_SEGMENTS.has(s.toLowerCase()))) return true;
|
|
if (/(^|\/)solution\.[a-z]+$/i.test(p)) return true;
|
|
for (const m of course.manifest?.modules ?? []) for (const l of m.lessons) if (l.browser?.solution && normalizeRel(l.browser.solution) === p) return true;
|
|
return false;
|
|
}
|
|
|
|
/** Posix-normalised course-relative path, or null when it is absolute or climbs out with `..`. */
|
|
export function normalizeRel(raw: string): string | null {
|
|
if (raw.includes("\u0000")) return null;
|
|
const p = posixPath.normalize(raw.replace(/\\/g, "/")).replace(/^(\.\/)+/, "");
|
|
if (!p || p === "." || p.startsWith("/") || /^[a-zA-Z]:/.test(p)) return null;
|
|
if (p.split("/").includes("..")) return null;
|
|
return p.replace(/\/+$/, "");
|
|
}
|
|
|
|
/** Undoes URL encoding (up to three layers). A malformed escape leaves the string as it is. */
|
|
function decodeLayers(raw: string): string {
|
|
let s = raw;
|
|
for (let i = 0; i < 3 && /%[0-9a-fA-F]{2}/.test(s); i++) {
|
|
try { s = decodeURIComponent(s); } catch { break; }
|
|
}
|
|
return s;
|
|
}
|
|
|
|
/**
|
|
* Canonical form of a path a user or the model asked for: URL-decoded, backslashes as slashes, posix-normalised,
|
|
* leading `./` and `/` stripped, `..` rejected, then resolved through `CourseRepository.resolvePath` (real path,
|
|
* symlinks followed, must stay inside the course root). `rel` is the real path relative to the real root.
|
|
* Null when the path is invalid, escapes the course or does not exist.
|
|
*/
|
|
export function canonicalCoursePath(repo: CourseRepository, course: LoadedCourse, raw: string): { rel: string; abs: string } | null {
|
|
const norm = normalizeRel(decodeLayers(raw.trim()).replace(/\\/g, "/").replace(/^\/+/, ""));
|
|
if (!norm) return null;
|
|
const abs = repo.resolvePath(course.id, norm);
|
|
if (!abs) return null;
|
|
let realRoot: string;
|
|
try { realRoot = realpathSync(course.root); } catch { return null; }
|
|
const rel = relative(realRoot, abs).split(sep).join("/");
|
|
if (!rel || rel.startsWith("..")) return null;
|
|
return { rel, abs };
|
|
}
|
|
|
|
/**
|
|
* The guard for tool reads, on canonical paths only: solution files (by the folder and name rules on the real
|
|
* relative path, and by comparing real paths with every manifest `browser.solution`) and hidden files or folders.
|
|
*/
|
|
export function isGuardedFile(repo: CourseRepository, course: LoadedCourse, canonical: { rel: string; abs: string }): boolean {
|
|
if (canonical.rel.split("/").some((s) => s.startsWith("."))) return true;
|
|
if (isSolutionPath(course, canonical.rel)) return true;
|
|
for (const m of course.manifest?.modules ?? []) {
|
|
for (const l of m.lessons) {
|
|
if (!l.browser?.solution) continue;
|
|
const sol = repo.resolvePath(course.id, l.browser.solution);
|
|
if (sol && sol === canonical.abs) return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
async function walk(root: string, onFile: (abs: string, rel: string) => Promise<boolean>, depth = 0, base = root): Promise<boolean> {
|
|
if (depth > 8) return true;
|
|
let entries;
|
|
try { entries = await readdir(root, { withFileTypes: true }); } catch { return true; }
|
|
entries.sort((a, b) => a.name.localeCompare(b.name));
|
|
for (const e of entries) {
|
|
if (e.name.startsWith(".") && e.name !== ".") continue;
|
|
const abs = join(root, e.name);
|
|
if (e.isDirectory()) {
|
|
if (SKIP_DIRS.has(e.name)) continue;
|
|
if (!(await walk(abs, onFile, depth + 1, base))) return false;
|
|
} else if (e.isFile()) {
|
|
if (!(await onFile(abs, posix(relative(base, abs))))) return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/** Splits Markdown at headings (outside code fences), then long sections at paragraph breaks. */
|
|
export function splitMarkdown(text: string): { heading?: string; text: string }[] {
|
|
const sections: { heading?: string; lines: string[] }[] = [{ lines: [] }];
|
|
let fence = false;
|
|
for (const line of text.split("\n")) {
|
|
if (/^\s*(```|~~~)/.test(line)) fence = !fence;
|
|
const h = !fence && /^(#{1,6})\s+(.*)$/.exec(line);
|
|
if (h) sections.push({ heading: h[2]!.replace(/#+\s*$/, "").trim(), lines: [line] });
|
|
else sections[sections.length - 1]!.lines.push(line);
|
|
}
|
|
const out: { heading?: string; text: string }[] = [];
|
|
for (const s of sections) {
|
|
const body = s.lines.join("\n").trim();
|
|
if (!body) continue;
|
|
out.push(...splitLong(body).map((t) => ({ heading: s.heading, text: t })));
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function splitLong(text: string): string[] {
|
|
if (text.length <= MAX_CHUNK) return [text];
|
|
const out: string[] = [];
|
|
let cur = "";
|
|
for (const para of text.split(/\n{2,}/)) {
|
|
if (cur && cur.length + para.length > MAX_CHUNK) { out.push(cur); cur = ""; }
|
|
if (para.length > MAX_CHUNK) {
|
|
for (let i = 0; i < para.length; i += MAX_CHUNK) out.push(para.slice(i, i + MAX_CHUNK));
|
|
} else cur = cur ? `${cur}\n\n${para}` : para;
|
|
}
|
|
if (cur) out.push(cur);
|
|
return out;
|
|
}
|
|
|
|
type LessonRef = { lessonId: string; lessonTitle: string; anchor?: string; transcript: boolean };
|
|
|
|
function lessonRefs(course: LoadedCourse): Map<string, LessonRef[]> {
|
|
const refs = new Map<string, LessonRef[]>();
|
|
const add = (p: string | undefined, r: LessonRef) => {
|
|
if (!p) return;
|
|
const key = posix(p).replace(/^\.\//, "");
|
|
refs.set(key, [...(refs.get(key) ?? []), r]);
|
|
};
|
|
for (const m of course.manifest?.modules ?? []) {
|
|
for (const l of m.lessons) {
|
|
const base = { lessonId: l.id, lessonTitle: l.title };
|
|
if (l.type !== "video") add(l.file, { ...base, anchor: l.anchor, transcript: false });
|
|
add(l.transcript, { ...base, transcript: true });
|
|
if (l.hints?.file && l.hints.file !== l.file) add(l.hints.file, { ...base, transcript: false });
|
|
for (const t of Object.values(l.tracks ?? {})) add(t.file, { ...base, transcript: false });
|
|
}
|
|
}
|
|
return refs;
|
|
}
|
|
|
|
function pickLesson(refs: LessonRef[] | undefined, heading?: string): LessonRef | undefined {
|
|
if (!refs?.length) return undefined;
|
|
if (heading) {
|
|
const slug = slugify(heading);
|
|
const hit = refs.find((r) => r.anchor && r.anchor === slug);
|
|
if (hit) return hit;
|
|
}
|
|
return refs.find((r) => !r.anchor) ?? refs[0];
|
|
}
|
|
|
|
function lessonTitleOf(course: LoadedCourse, lessonId: string | undefined): string | undefined {
|
|
return lessonId ? course.lessonIndex.get(lessonId)?.lesson.title : undefined;
|
|
}
|
|
|
|
async function readJson<T>(path: string): Promise<T | null> {
|
|
try {
|
|
const f = Bun.file(path);
|
|
if (!(await f.exists()) || f.size > MAX_FILE * 25) return null;
|
|
return (await f.json()) as T;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
const clock = (s: number) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
|
|
|
|
export async function buildCourseIndex(course: LoadedCourse, dataDir: string): Promise<CourseIndex> {
|
|
const bm = new Bm25<Chunk>();
|
|
const seen = new Set<string>();
|
|
const refs = lessonRefs(course);
|
|
const mentor = course.manifest?.mentor ? posix(course.manifest.mentor).replace(/^\.\//, "") : null;
|
|
const codeDirs = (course.manifest?.modules ?? []).flatMap((m) => (m.code ? [posix(m.code.dir).replace(/\/$/, "") + "/"] : []));
|
|
const add = (c: Chunk) => {
|
|
const key = `${c.kind}\u0000${c.text}`;
|
|
if (seen.has(key)) return;
|
|
seen.add(key);
|
|
bm.add([c.heading ?? "", c.lessonTitle ?? "", c.path, c.text].join("\n"), c);
|
|
};
|
|
|
|
let files = 0;
|
|
if (course.status === "ok") {
|
|
await walk(course.root, async (abs, rel) => {
|
|
if (files >= MAX_FILES) return false;
|
|
const ext = extname(rel).toLowerCase();
|
|
const name = basename(rel);
|
|
const isMd = MD.has(ext);
|
|
const isCode = !isMd && (CODE.has(ext) || CODE_NAMES.has(name) || codeDirs.some((d) => rel.startsWith(d)));
|
|
if ((!isMd && !isCode) || SKIP_FILES.has(name) || rel === mentor || isSolutionPath(course, rel)) return true;
|
|
let size = 0;
|
|
try { size = (await stat(abs)).size; } catch { return true; }
|
|
if (size > MAX_FILE || size === 0) return true;
|
|
const text = await Bun.file(abs).text();
|
|
if (isCode && text.includes("\u0000")) return true;
|
|
files++;
|
|
const r = refs.get(rel);
|
|
if (isMd) {
|
|
for (const part of splitMarkdown(text)) {
|
|
const ref = pickLesson(r, part.heading);
|
|
add({
|
|
path: rel,
|
|
kind: ref ? (ref.transcript ? "transcript" : "lesson") : "markdown",
|
|
text: part.text,
|
|
heading: part.heading,
|
|
lessonId: ref?.lessonId,
|
|
lessonTitle: ref?.lessonTitle,
|
|
});
|
|
}
|
|
} else {
|
|
const lines = text.split("\n");
|
|
for (let i = 0; i < lines.length; i += 60) {
|
|
add({ path: rel, kind: "code", text: lines.slice(i, i + 60).join("\n"), heading: lines.length > 60 ? `lines ${i + 1}-${Math.min(lines.length, i + 60)}` : undefined });
|
|
}
|
|
}
|
|
return true;
|
|
});
|
|
}
|
|
|
|
const decks = new Map((course.manifest?.decks ?? []).map((d) => [d.id, d]));
|
|
|
|
// The importer's deck cache: <dataDir>/cache/decks/<courseId>/<deckId>/text.json = [{ page, text }].
|
|
for (const deck of decks.values()) {
|
|
const pages = await readJson<{ page?: number; text?: string }[]>(join(dataDir, "cache", "decks", course.id, deck.id, "text.json"));
|
|
if (!Array.isArray(pages)) continue;
|
|
const lessonId = course.lessonIndex.has(deck.attachTo) ? deck.attachTo : undefined;
|
|
for (const p of pages) {
|
|
const text = typeof p?.text === "string" ? p.text.trim() : "";
|
|
if (!text) continue;
|
|
for (const part of splitLong(text)) {
|
|
add({ path: posix(deck.file), kind: "deck", text: part, heading: p.page ? `Slide ${p.page}` : deck.title, lessonId, lessonTitle: lessonTitleOf(course, lessonId) });
|
|
}
|
|
}
|
|
}
|
|
|
|
// The importer's transcription cache: <dataDir>/cache/transcripts/<courseId>/<lessonId>.json =
|
|
// { lessonId, text, segments: [{ start, end, text }] }. Skipped when the lesson has a transcript file in the
|
|
// course, which the transcription job writes alongside and which is indexed above.
|
|
const trDir = join(dataDir, "cache", "transcripts", course.id);
|
|
for (const name of await readdir(trDir).catch(() => [] as string[])) {
|
|
if (extname(name) !== ".json") continue;
|
|
const lessonId = name.slice(0, -5);
|
|
const entry = course.lessonIndex.get(lessonId);
|
|
if (!entry || entry.lesson.transcript) continue;
|
|
const tr = await readJson<{ text?: string; segments?: { start?: number; text?: string }[] }>(join(trDir, name));
|
|
if (!tr) continue;
|
|
const segs = Array.isArray(tr.segments) ? tr.segments.filter((s) => typeof s?.text === "string" && s.text.trim()) : [];
|
|
const blocks: { at: number; text: string }[] = [];
|
|
if (segs.length) {
|
|
for (const s of segs) {
|
|
const last = blocks[blocks.length - 1];
|
|
if (last && last.text.length + s.text!.length < MAX_CHUNK) last.text += ` ${s.text!.trim()}`;
|
|
else blocks.push({ at: Number(s.start) || 0, text: s.text!.trim() });
|
|
}
|
|
} else if (typeof tr.text === "string" && tr.text.trim()) {
|
|
blocks.push(...splitLong(tr.text.trim()).map((t) => ({ at: -1, text: t })));
|
|
}
|
|
for (const b of blocks) {
|
|
add({ path: entry.lesson.file ? posix(entry.lesson.file) : `transcripts/${lessonId}`, kind: "transcript", text: b.text, heading: b.at >= 0 ? `at ${clock(b.at)}` : undefined, lessonId, lessonTitle: entry.lesson.title });
|
|
}
|
|
}
|
|
|
|
// Older layout: plain text under <dataDir>/cache/<courseId>/**.
|
|
const cacheRoot = join(dataDir, "cache", course.id);
|
|
await walk(cacheRoot, async (abs, rel) => {
|
|
if (!CACHE_TEXT.has(extname(rel).toLowerCase())) return true;
|
|
const size = (await stat(abs).catch(() => null))?.size ?? 0;
|
|
if (!size || size > MAX_FILE * 5) return true;
|
|
const text = await Bun.file(abs).text();
|
|
const segs = rel.split("/");
|
|
const deck = segs.map((s) => decks.get(s)).find(Boolean);
|
|
const lessonSeg = segs.map((s) => s.replace(/\.[^.]+$/, "")).find((s) => course.lessonIndex.has(s));
|
|
const page = /(?:page|slide)[-_ ]?(\d+)/i.exec(basename(rel))?.[1];
|
|
if (deck) {
|
|
const lessonId = course.lessonIndex.has(deck.attachTo) ? deck.attachTo : undefined;
|
|
for (const part of splitLong(text.trim())) {
|
|
add({ path: posix(deck.file), kind: "deck", text: part, heading: page ? `Slide ${Number(page)}` : deck.title, lessonId, lessonTitle: lessonTitleOf(course, lessonId) });
|
|
}
|
|
} else {
|
|
const kind: ChunkKind = /transcript|\.vtt$|\.srt$/i.test(rel) || lessonSeg ? "transcript" : "deck";
|
|
for (const part of splitLong(text.trim())) {
|
|
add({ path: `cache/${rel}`, kind, text: part, lessonId: lessonSeg, lessonTitle: lessonTitleOf(course, lessonSeg) });
|
|
}
|
|
}
|
|
return true;
|
|
});
|
|
|
|
return {
|
|
get size() { return bm.size; },
|
|
search(query, k, extra = []) {
|
|
return bm.search(query, k, extra.map((c) => ({ text: [c.heading ?? "", c.lessonTitle ?? "", c.text].join("\n"), item: c }))).map((r) => r.item);
|
|
},
|
|
};
|
|
}
|
|
|
|
/** The user's notes as chunks, scored at query time and never cached. */
|
|
export function noteChunks(course: LoadedCourse, notes: { id: string; lessonId: string; body: string; anchorKind: string; anchorValue: string | null }[]): Chunk[] {
|
|
return notes.map((n) => ({
|
|
path: "notes",
|
|
kind: "note" as const,
|
|
text: n.body,
|
|
heading: n.anchorKind === "video" && n.anchorValue ? `at ${n.anchorValue}s` : n.anchorKind === "slide" && n.anchorValue ? `slide ${n.anchorValue}` : undefined,
|
|
lessonId: n.lessonId,
|
|
lessonTitle: lessonTitleOf(course, n.lessonId),
|
|
}));
|
|
}
|
|
|
|
const TTL_MS = 5 * 60_000;
|
|
const cache = new WeakMap<LoadedCourse, { at: number; index: Promise<CourseIndex> }>();
|
|
|
|
/** Cached per LoadedCourse object: a repository reload yields new objects, and entries expire after 5 minutes. */
|
|
export function getCourseIndex(course: LoadedCourse, dataDir: string): Promise<CourseIndex> {
|
|
const hit = cache.get(course);
|
|
if (hit && Date.now() - hit.at < TTL_MS) return hit.index;
|
|
const index = buildCourseIndex(course, dataDir);
|
|
cache.set(course, { at: Date.now(), index });
|
|
index.catch(() => cache.delete(course));
|
|
return index;
|
|
}
|