From fc37da9789e1ef459b45a933f1905eca5b5ad6ac Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 03:11:41 +0100 Subject: [PATCH 01/62] docs(studio): WP-C implementation plan Co-Authored-By: Claude Opus 5.5 --- docs/superpowers/plans/2026-10-30-studio.md | 77 +++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-30-studio.md diff --git a/docs/superpowers/plans/2026-10-30-studio.md b/docs/superpowers/plans/2026-10-30-studio.md new file mode 100644 index 0000000..97362bd --- /dev/null +++ b/docs/superpowers/plans/2026-10-30-studio.md @@ -0,0 +1,77 @@ +# STCourses WP-C Studio Implementation Plan + +> **For agentic workers:** executed inline by the WP-C agent (superpowers:executing-plans). Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Browser studio for studio-managed courses: create a course, edit the manifest, lessons (Markdown with live preview), deck links, attachments and exercises in a per-author draft worktree; every save is a commit by the author; publish merges the draft into the published branch that learners read; Sync with a remote only on request; conflicts are shown and resolved by the author, never automatically. + +**Architecture:** +- `packages/feature-studio/src/server/git.ts`: a thin `git` CLI wrapper over `Bun.$` (explicit identity, no hooks, no signing) plus a per-course async mutex. +- `packages/feature-studio/src/server/store.ts`: `StudioStore`, the git model of spec section 12: + - `/courses/`: a **bare** repo (it holds no `stcourse.yaml`, so course discovery never loads it); + - `/worktrees//published`: worktree on the published branch (`main`); the only tree learners read; + - `/worktrees//drafts/`: worktree on `draft/` (depth 3 under `worktrees/`, so discovery, which stops at depth 2, never loads drafts). +- Publish = `git merge --no-ff draft/` inside the published worktree. A conflicting merge is aborted at once, so the published tree is never left half-merged, and the conflicting paths are returned. The author then runs "Update draft" (merge of `main`, or `origin/main` for Sync, into the draft worktree), resolves each file in the studio, and publishes again. +- Save uses optimistic concurrency: each file carries the hash of the content the editor loaded; a mismatch with the file on disk (an external edit) gives 409 with both versions. Nothing is overwritten. +- `watch.ts`: polls the published HEAD of each managed course (default every 3 s, unref'd timer) and calls `ctx.courses.reload(id)` when it changed (external commit, pull, publish). +- `feature.ts`: `studioFeature(ctx)` with `/api/studio/*` routes. Access: owner, `author` role, or a `course_authors` row for that course. Courses not managed by the studio (overlays, fixture/imported folders) are listed read-only: "This course is managed outside STCourses". +- `draft-writer.ts`: `createDraftWriter(ctx)` for WP-D. +- Web (`src/web/*`): `/studio` (course list, New course) and `/studio/:courseId` (outline sidebar, editors, status bar with Publish / Update draft / Sync and the conflict view). Work look from the mockup. The lesson preview sits behind a `LessonPreview` adapter; v1 is a small safe Markdown renderer; Phase 2 swaps in the learner `LessonView` (contract request). + +**Tech Stack:** Bun 1.3, TypeScript strict, git CLI 2.4x via `Bun.$`, React 19 + React Router 6, `@stc/course-format` for validation, `@stc/ui` primitives. + +**Spec:** sections 5, 9.1, 12 (git model), 13. Brief: `.superpowers/sdd/2026-10-30-stcourses/phase-1-brief.md` (WP-C). + +## Global Constraints + +Inherits every item of the Global Constraints in `docs/superpowers/plans/2026-10-30-stcourses.md`. Additionally: +- Never write into a course whose root is outside `` (overlays) — the studio refuses writes for non-managed courses. +- Paths written through the studio must be relative, stay inside the worktree, and never touch `.git`. +- No new DB tables (authors use the existing `course_authors`); `migrationsDirs` is exported empty. + +## Review Focus + +1. A conflicting external commit to `main` is detected at publish and shown; the published tree keeps the external content (test: acceptance). +2. Drafts never reach learners: before publish the `CourseRepository` still serves the published content (test: acceptance). +3. An external edit to a draft file between load and save gives 409, not an overwrite (test: store). +4. Paths with spaces and non-ASCII characters save, commit and publish (test: store). +5. Path traversal (`..`, absolute, `.git/…`) is rejected (test: store). + +## Tasks + +### Task 1: git wrapper and StudioStore (create, drafts, save) + +**Files:** Create `packages/feature-studio/src/server/{git.ts,store.ts,paths.ts}`; Test `packages/feature-studio/test/store.test.ts`. + +**Interfaces (Produces):** `createStudioStore(dataDir): StudioStore` with `createCourse`, `isManaged`, `listManaged`, `ensureDraft`, `readFile`, `listFiles`, `saveFiles`, `status`. + +- [ ] Step 1: failing tests: create gives bare repo + published worktree with a valid manifest; draft worktree on `draft/`; save commits with the author's name/email; stale base hash gives a conflict; traversal rejected; non-ASCII path works. +- [ ] Step 2: run, FAIL. Step 3: implement. Step 4: PASS. Step 5: commit. + +### Task 2: publish, update draft, conflict resolution, sync + +**Files:** extend `store.ts`; Test `packages/feature-studio/test/publish.test.ts`. + +- [ ] Step 1: failing tests: publish merges; external conflicting commit gives `{ok:false, conflicts}` and leaves `main` untouched; dirty published tree refuses publish; update draft shows ours/theirs; resolve + publish succeeds; abort; Sync with a local bare remote (pull fast-forward, push, conflict reported); no remote gives an error. +- [ ] Steps 2–5 as above. + +### Task 3: watcher and draft writer + +**Files:** `src/server/{watch.ts,draft-writer.ts}`; Test `test/watch.test.ts`, `test/draft-writer.test.ts`. + +- [ ] Failing tests: external commit triggers `courses.reload`; `writeDraftFiles` commits on `draft/` and the published tree is unchanged. Implement, pass, commit. + +### Task 4: HTTP feature and acceptance + +**Files:** `src/server/feature.ts`, `src/index.ts`; Test `test/feature.test.ts`, `test/acceptance.test.ts`. + +- [ ] Failing tests: 401/403 for anonymous and learners; overlay course read-only; create → edit lesson → publish → learner sees it via `CourseRepository` and `/api/media`; conflicting external commit shown, not overwritten; manifest validation errors returned. Implement, pass, commit. + +### Task 5: Studio web UI + +**Files:** `src/web/{api.ts,markdown.ts,preview.tsx,StudioHome.tsx,CourseStudio.tsx,editors/*.tsx,studio.css}`, `src/web.ts`; Test `test/markdown.test.ts`. + +- [ ] Failing test for the Markdown renderer (escaping, headings, fences, lists, links without `javascript:`). Implement UI. Commit. + +### Task 6: wiring, dev server, contract requests, verification + +- [ ] Append to `apps/web/src/features.ts`; dev entry `packages/feature-studio/dev/serve.ts`; `docs/contract-requests/studio.md`; run `bun test`, typecheck, Postgres run; headless Chrome screenshots of `/studio` and `/studio/:id`; report. From 10cf93c27eec12759cda0dfa0668d04d4093da46 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 03:11:53 +0100 Subject: [PATCH 02/62] docs(import): WP-B plan Co-Authored-By: Claude Opus 5.5 --- docs/superpowers/plans/2026-10-30-import.md | 56 +++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-30-import.md diff --git a/docs/superpowers/plans/2026-10-30-import.md b/docs/superpowers/plans/2026-10-30-import.md new file mode 100644 index 0000000..ba21859 --- /dev/null +++ b/docs/superpowers/plans/2026-10-30-import.md @@ -0,0 +1,56 @@ +# WP-B Importer Implementation Plan + +> Executed inline by the WP-B agent, task by task, TDD. Spec: `docs/superpowers/specs/2026-10-30-stcourses-design.md` sections 5, 8, 13. + +**Goal:** `packages/feature-import` turns a Packt video zip, a companion code repo, a slide PDF or a folder of videos into an ordinary course folder under `/courses//`, as background jobs with visible progress, plus an owner-only admin UI at `/admin/import`. + +**Architecture:** +- `importFeature(ctx)` returns a `Feature` with job handlers `import.packt-video`, `import.code-repo`, `import.pdf-deck`, `import.video-folder`, `import.transcribe` and owner-only `/api/import/*` routes. `ctx` is read lazily (inside handlers), because the job queue is built from the handlers. +- Courses are built in `/tmp/` and renamed into `/courses/` in one step. Attach jobs (code, deck) only touch courses under `/courses` and write the manifest last, through temp file + rename. +- Media: **copied** (`cp --reflink=auto`, so instant on CoW filesystems), or hard-linked with `options.media = "link"` (falls back to copy across filesystems). Symlinks out of the course are not an option: the media route 404s on paths that escape the course root. The originals are only ever read. +- PDFs: PyMuPDF through `py/pdf_tool.py` run as `python -I` in a subprocess (`STC_PYTHON`, default `python3`). Chosen over pdfjs/mupdf-wasm because it rasterises and extracts text natively, is already installed, is AGPL-compatible and costs no idle memory in the server. If it is missing, the deck is still attached and the job reports a warning. +- Derived data goes to `/cache/`: `decks///p-001.jpg…` + `text.json`, `covers//*.jpg`, `transcripts//.json`. + +## Global Constraints + +Inherited unchanged from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun conventions, AGPL, design tokens, no licensed content in the repo, commit trailer). + +## Review Focus + +1. Original files are never modified; attach jobs refuse courses outside `/courses` (overlays, authored repos). +2. Paths with spaces/non-ASCII (`S08 - Containerizing Applications`) survive unzip, copy and manifest round trip. +3. Packt parsing is structural (chapter blocks plus the section summary lists), not regexes on "Video N.M:" prefixes. +4. Failure leaves no half-built course visible; a failed PDF/ffmpeg step degrades to a warning where the course is still valid. +5. Routes: 401 without a user, 403 without the owner role. + +## Tasks + +### Task 1: Utilities (slug, HTML entities, natural sort, exec, manifest IO, sources) +- Files: `src/util/*.ts`, `test/util.test.ts` +- Interfaces: `slugify`, `uniqueCourseId`, `decodeEntities`, `naturalSort`, `run`, `readRawManifest`, `writeManifest` (validates with `ManifestSchema`), `openSource(path)` (zip via `unzip`, or directory). +- [ ] Failing tests for each helper; [ ] implement; [ ] commit. + +### Task 2: Fixtures and media tooling +- Files: `test/fixtures.ts` (generates fake Packt zip, repo zip, PDF, video folder in a temp dir), `src/media.ts` (ffprobe duration, ffmpeg frame, copy/link), `src/pdf.ts`, `py/pdf_tool.py`. +- [ ] Failing tests (frame extraction, duration, PDF pages+text); [ ] implement; [ ] commit. + +### Task 3: Packt parser and `import.packt-video` +- Files: `src/packt.ts`, `src/jobs/packt-video.ts`, `test/packt.test.ts` +- [ ] Parser test on the synthetic index (summary-only titles, missing chapter); [ ] job test (2 sections, 3 clips, restricted manifest, source filled, course loads `ok`, cover candidates); [ ] implement; [ ] commit. + +### Task 4: `import.video-folder`, `import.pdf-deck` +- [ ] Tests: natural order, deck cache pages/text, attach to lesson and module, refuses unmanaged course; [ ] implement; [ ] commit. + +### Task 5: `import.code-repo` +- [ ] Tests: folder `S02 - …` attaches to module 2 by number, by name when no number, PDFs become deck/attachment, git URL validation; [ ] implement; [ ] commit. + +### Task 6: `import.transcribe` +- [ ] Tests with a fake Whisper server; skipped when `STC_TRANSCRIBE_URL` unset; [ ] implement; [ ] commit. + +### Task 7: Routes and `importFeature` +- [ ] Tests: 401/403/owner, enqueue validation, queue integration (SQLite and Postgres), covers; [ ] implement; wire `main.ts`; [ ] commit. + +### Task 8: Web UI `importWeb` +- [ ] Page with form, job list with progress, cover picker; register in `apps/web/src/features.ts`; screenshot check; [ ] commit. + +### Task 9: Manual check against `/home/user/example_course_content`, report. From 34f27d05c9482428bb4ebb9ea0b1b9eae739333e Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 03:13:22 +0100 Subject: [PATCH 03/62] feat(import): utilities, media tooling, PyMuPDF helper, synthetic fixtures Co-Authored-By: Claude Opus 5.5 --- packages/feature-import/package.json | 3 +- packages/feature-import/py/pdf_tool.py | 50 ++++++++ packages/feature-import/src/media.ts | 40 ++++++ packages/feature-import/src/pdf.ts | 23 ++++ packages/feature-import/src/util/exec.ts | 30 +++++ packages/feature-import/src/util/manifest.ts | 21 ++++ packages/feature-import/src/util/text.ts | 48 +++++++ packages/feature-import/test/fixtures.ts | 125 +++++++++++++++++++ packages/feature-import/test/media.test.ts | 38 ++++++ packages/feature-import/test/util.test.ts | 41 ++++++ 10 files changed, 418 insertions(+), 1 deletion(-) create mode 100644 packages/feature-import/py/pdf_tool.py create mode 100644 packages/feature-import/src/media.ts create mode 100644 packages/feature-import/src/pdf.ts create mode 100644 packages/feature-import/src/util/exec.ts create mode 100644 packages/feature-import/src/util/manifest.ts create mode 100644 packages/feature-import/src/util/text.ts create mode 100644 packages/feature-import/test/fixtures.ts create mode 100644 packages/feature-import/test/media.test.ts create mode 100644 packages/feature-import/test/util.test.ts diff --git a/packages/feature-import/package.json b/packages/feature-import/package.json index cab5a10..0b65795 100644 --- a/packages/feature-import/package.json +++ b/packages/feature-import/package.json @@ -3,5 +3,6 @@ "version": "0.0.0", "private": true, "type": "module", - "license": "AGPL-3.0-only" + "license": "AGPL-3.0-only", + "main": "src/index.ts" } diff --git a/packages/feature-import/py/pdf_tool.py b/packages/feature-import/py/pdf_tool.py new file mode 100644 index 0000000..0bff2b7 --- /dev/null +++ b/packages/feature-import/py/pdf_tool.py @@ -0,0 +1,50 @@ +"""PDF helper for the STCourses importer. Run as: python -I pdf_tool.py render [--width 1280] [--quality 80] + +Writes outdir/p-001.jpg ... and outdir/text.json ([{"page": 1, "text": "..."}]). +Prints one "page N/M" line per page on stdout and a final JSON line {"pages": M, "title": "..."}. +""" +import json +import os +import sys + +try: + import pymupdf as fitz +except ImportError: # older PyMuPDF + import fitz + + +def render(pdf, outdir, width, quality): + os.makedirs(outdir, exist_ok=True) + doc = fitz.open(pdf) + n = len(doc) + texts = [] + for i, page in enumerate(doc, start=1): + zoom = width / max(page.rect.width, 1) + pix = page.get_pixmap(matrix=fitz.Matrix(zoom, zoom), alpha=False) + pix.save(os.path.join(outdir, "p-%03d.jpg" % i), jpg_quality=quality) + texts.append({"page": i, "text": page.get_text().strip()}) + print("page %d/%d" % (i, n), flush=True) + with open(os.path.join(outdir, "text.json"), "w", encoding="utf-8") as f: + json.dump(texts, f, ensure_ascii=False) + print(json.dumps({"pages": n, "title": (doc.metadata or {}).get("title") or ""}), flush=True) + + +def main(argv): + if len(argv) < 4 or argv[1] != "render": + print("usage: pdf_tool.py render [--width N] [--quality N]", file=sys.stderr) + return 2 + width, quality = 1280, 80 + rest = argv[4:] + while rest: + flag, val = rest[0], rest[1] if len(rest) > 1 else "" + if flag == "--width": + width = int(val) + elif flag == "--quality": + quality = int(val) + rest = rest[2:] + render(argv[2], argv[3], width, quality) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/packages/feature-import/src/media.ts b/packages/feature-import/src/media.ts new file mode 100644 index 0000000..0fab184 --- /dev/null +++ b/packages/feature-import/src/media.ts @@ -0,0 +1,40 @@ +import { link, mkdir, copyFile } from "node:fs/promises"; +import { dirname } from "node:path"; +import { run } from "./util/exec"; + +export interface Tools { ffmpeg: string; ffprobe: string } + +export async function probeDuration(t: Tools, file: string): Promise { + try { + const out = await run([t.ffprobe, "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", file]); + const s = Number(out.trim()); + return Number.isFinite(s) && s > 0 ? Math.round(s) : undefined; + } catch { + return undefined; + } +} + +/** One JPEG frame at `fraction` of the clip, 640px wide. */ +export async function extractFrame(t: Tools, file: string, out: string, fraction = 0.2): Promise { + await mkdir(dirname(out), { recursive: true }); + const dur = (await probeDuration(t, file)) ?? 0; + const at = Math.max(0, dur * fraction); + await run([t.ffmpeg, "-v", "error", "-y", "-ss", String(at), "-i", file, "-frames:v", "1", "-vf", "scale=640:-2", "-q:v", "4", out]); +} + +/** + * Puts a large file into the course. "copy" uses `cp --reflink=auto` (a CoW clone where the filesystem supports it, + * a plain copy elsewhere); "link" makes a hard link and falls back to copying across filesystems. + * Either way the original is only read, and the course folder stays self-contained (the media route refuses symlinks that leave it). + */ +export async function putFile(src: string, dest: string, mode: "copy" | "link" = "copy"): Promise { + await mkdir(dirname(dest), { recursive: true }); + if (mode === "link") { + try { await link(src, dest); return; } catch { /* cross-device or unsupported: copy */ } + } + try { + await run(["cp", "--reflink=auto", "--", src, dest]); + } catch { + await copyFile(src, dest); + } +} diff --git a/packages/feature-import/src/pdf.ts b/packages/feature-import/src/pdf.ts new file mode 100644 index 0000000..c1b4366 --- /dev/null +++ b/packages/feature-import/src/pdf.ts @@ -0,0 +1,23 @@ +import { join } from "node:path"; +import { run } from "./util/exec"; + +export interface DeckRender { pages: number; title: string } + +const TOOL = join(import.meta.dir, "..", "py", "pdf_tool.py"); + +/** + * Renders page images (p-001.jpg ...) and text.json into `outDir` with PyMuPDF in a subprocess, + * so the server process never holds a PDF engine. `python` is STC_PYTHON (default python3). + */ +export async function renderPdf(python: string, pdf: string, outDir: string, opts: { onPage?: (page: number, total: number) => void; signal?: AbortSignal } = {}): Promise { + let last = ""; + await run([python, "-I", TOOL, "render", pdf, outDir], { + signal: opts.signal, + onLine(l) { + const m = /^page (\d+)\/(\d+)$/.exec(l); + if (m) opts.onPage?.(Number(m[1]), Number(m[2])); + else if (l.trim()) last = l; + }, + }); + return JSON.parse(last) as DeckRender; +} diff --git a/packages/feature-import/src/util/exec.ts b/packages/feature-import/src/util/exec.ts new file mode 100644 index 0000000..78b3c2f --- /dev/null +++ b/packages/feature-import/src/util/exec.ts @@ -0,0 +1,30 @@ +/** Runs a command (no shell). Resolves with stdout; rejects with the stderr tail on a non-zero exit. */ +export async function run(cmd: string[], opts: { signal?: AbortSignal; env?: Record; onLine?: (line: string) => void } = {}): Promise { + const proc = Bun.spawn(cmd, { + stdout: "pipe", stderr: "pipe", stdin: "ignore", + env: { ...process.env, LC_ALL: "C.UTF-8", ...opts.env } as Record, + signal: opts.signal, + }); + const out = (async () => { + let text = ""; + let pending = ""; + const dec = new TextDecoder(); + for await (const chunk of proc.stdout) { + const s = dec.decode(chunk, { stream: true }); + text += s; + if (opts.onLine) { + pending += s; + const lines = pending.split("\n"); + pending = lines.pop()!; + for (const l of lines) opts.onLine(l); + } + } + return text; + })(); + const [stdout, stderr, code] = await Promise.all([out, new Response(proc.stderr).text(), proc.exited]); + if (code !== 0) { + const tail = stderr.trim().split("\n").slice(-6).join("\n"); + throw new Error(`${cmd[0]} exited with ${code}${tail ? `: ${tail}` : ""}`); + } + return stdout; +} diff --git a/packages/feature-import/src/util/manifest.ts b/packages/feature-import/src/util/manifest.ts new file mode 100644 index 0000000..80b4ab3 --- /dev/null +++ b/packages/feature-import/src/util/manifest.ts @@ -0,0 +1,21 @@ +import { readFile, rename, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { ManifestSchema } from "@stc/course-format"; + +export type RawManifest = Record; + +export async function readRawManifest(courseDir: string): Promise { + return Bun.YAML.parse(await readFile(join(courseDir, "stcourse.yaml"), "utf8")) as RawManifest; +} + +/** Validates against the course-format schema, then writes stcourse.yaml via temp file + rename. */ +export async function writeManifest(courseDir: string, manifest: RawManifest): Promise { + const r = ManifestSchema.safeParse(manifest); + if (!r.success) { + throw new Error(`generated manifest is invalid: ${r.error.issues.map((i) => `${i.path.join(".") || "(root)"}: ${i.message}`).join("; ")}`); + } + const target = join(courseDir, "stcourse.yaml"); + const tmp = `${target}.${process.pid}.tmp`; + await writeFile(tmp, Bun.YAML.stringify(manifest, null, 2) + "\n"); + await rename(tmp, target); +} diff --git a/packages/feature-import/src/util/text.ts b/packages/feature-import/src/util/text.ts new file mode 100644 index 0000000..6fffbab --- /dev/null +++ b/packages/feature-import/src/util/text.ts @@ -0,0 +1,48 @@ +export function slugify(s: string): string { + const slug = s + .normalize("NFKD") + .replace(/[̀-ͯ]/g, "") + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 48) + .replace(/-+$/g, ""); + return slug || "course"; +} + +export function uniqueId(base: string, taken: (id: string) => boolean): string { + if (!taken(base)) return base; + for (let i = 2; ; i++) if (!taken(`${base}-${i}`)) return `${base}-${i}`; +} + +const NAMED: Record = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " " }; +export function decodeEntities(s: string): string { + return s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e: string) => { + if (e[0] === "#") { + const code = e[1]!.toLowerCase() === "x" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10); + return Number.isFinite(code) ? String.fromCodePoint(code) : m; + } + return NAMED[e.toLowerCase()] ?? m; + }); +} + +export function stripTags(s: string): string { + return decodeEntities(s.replace(/<[^>]*>/g, "")).replace(/\s+/g, " ").trim(); +} + +const collator = new Intl.Collator("en", { numeric: true, sensitivity: "base" }); +export function naturalSort(list: string[]): string[] { + return [...list].sort((a, b) => collator.compare(a, b) || (a < b ? -1 : a > b ? 1 : 0)); +} + +/** Lowercase alphanumeric words, for loose name matching. */ +export function words(s: string): string[] { + return s.normalize("NFKD").replace(/[̀-ͯ]/g, "").toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length > 1); +} +/** Share of the words of `a` that also occur in `b`. */ +export function containment(a: string, b: string): number { + const wa = words(a); + if (wa.length === 0) return 0; + const wb = new Set(words(b)); + return wa.filter((w) => wb.has(w)).length / wa.length; +} diff --git a/packages/feature-import/test/fixtures.ts b/packages/feature-import/test/fixtures.ts new file mode 100644 index 0000000..6e78abf --- /dev/null +++ b/packages/feature-import/test/fixtures.ts @@ -0,0 +1,125 @@ +// Tiny synthetic inputs for the importer tests. Nothing here is real course material. +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { run } from "../src/util/exec"; + +// PyMuPDF is needed for PDF tests: set STC_PYTHON, or this machine's venv is used when present. +const VENV_PY = "/home/user/.venvs/pdf/bin/python"; +export const PYTHON = process.env.STC_PYTHON ?? (existsSync(VENV_PY) ? VENV_PY : "python3"); + +export async function tmp(prefix = "stc-imp-"): Promise { + return mkdtemp(join(tmpdir(), prefix)); +} + +export async function makeVideo(path: string, seconds = 2): Promise { + await mkdir(dirname(path), { recursive: true }); + await run(["ffmpeg", "-v", "error", "-y", "-f", "lavfi", "-i", `testsrc=duration=${seconds}:size=160x90:rate=10`, "-f", "lavfi", "-i", `sine=frequency=440:duration=${seconds}`, + "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-shortest", path]); +} + +/** Minimal valid PDF with one line of text per page. */ +export function pdfBytes(pages: string[]): Uint8Array { + const objs: string[] = []; + const kids = pages.map((_, i) => `${4 + i * 2} 0 R`).join(" "); + objs.push("<< /Type /Catalog /Pages 2 0 R >>"); + objs.push(`<< /Type /Pages /Kids [${kids}] /Count ${pages.length} >>`); + objs.push("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"); + pages.forEach((text, i) => { + const content = `BT /F1 24 Tf 40 150 Td (${text.replace(/[()\\]/g, "")}) Tj ET`; + objs.push(`<< /Type /Page /Parent 2 0 R /MediaBox [0 0 320 200] /Resources << /Font << /F1 3 0 R >> >> /Contents ${5 + i * 2} 0 R >>`); + objs.push(`<< /Length ${content.length} >>\nstream\n${content}\nendstream`); + }); + let out = "%PDF-1.4\n"; + const offsets: number[] = []; + objs.forEach((o, i) => { offsets.push(out.length); out += `${i + 1} 0 obj\n${o}\nendobj\n`; }); + const xref = out.length; + out += `xref\n0 ${objs.length + 1}\n0000000000 65535 f \n` + offsets.map((o) => `${String(o).padStart(10, "0")} 00000 n \n`).join(""); + out += `trailer\n<< /Size ${objs.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`; + return new TextEncoder().encode(out); +} + +export async function makePdf(path: string, pages = ["First slide alpha", "Second slide beta"]): Promise { + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, pdfBytes(pages)); +} + +export async function zipDir(srcDir: string, zipPath: string): Promise { + await run([PYTHON, "-I", "-c", + "import sys,os,zipfile\nsrc,out=sys.argv[1:3]\nz=zipfile.ZipFile(out,'w',zipfile.ZIP_STORED)\nfor r,d,f in os.walk(src):\n for n in f:\n p=os.path.join(r,n); z.write(p, os.path.relpath(p,src))\nz.close()", + srcDir, zipPath]); +} + +const li = (t: string) => `
  • ${t}
    • `; +const clipHtml = (s: number, m: number, title: string) => ` +
  • + +

    ${s}.${m} ${title}

    + In this video, ${title}, you'll learn it. +
    Watch Video
    + Download +
    • `; + +export const PACKT_SECTIONS = [ + { title: "Intro to Docker", clips: ["Course Introduction", "Docker & Friends"] }, + { title: "Images (Sample)", clips: ["Topic Introduction - Images"] }, +]; + +/** Mimics the real package layout: summary lists without "Video N.M:" prefixes, then chapter blocks. */ +export function packtIndexHtml(opts: { dropChapter?: number } = {}): string { + const summary = PACKT_SECTIONS.map((s, i) => ` +
  • + +

    Section ${i + 1}: ${s.title}

    +

    Video ${i + 1}.1: ${s.clips[0]}

    +
      ${s.clips.map(li).join("\n")}
    +
    Browse Videos
    +
    • `).join(""); + const chapters = PACKT_SECTIONS.map((s, i) => i + 1 === opts.dropChapter ? "" : ` + `).join(""); + return `Fake +

    Fake Docker Course

    +
      ${summary}
    ${chapters}
    `; +} + +export async function makePacktTree(root: string): Promise { + await mkdir(join(root, "package", "videos"), { recursive: true }); + await writeFile(join(root, "package", "index.html"), packtIndexHtml()); + await writeFile(join(root, "Video Package_User Guide.pdf"), "not used"); + await makeVideo(join(root, "package/videos/video1_1.mp4"), 2); + for (const n of ["video1_2", "video2_1"]) await Bun.write(join(root, "package/videos", `${n}.mp4`), Bun.file(join(root, "package/videos/video1_1.mp4"))); +} + +export async function makePacktZip(dir: string): Promise { + const tree = join(dir, "packt-tree"); + await makePacktTree(tree); + const zip = join(dir, "video_zip.zip"); + await zipDir(tree, zip); + return zip; +} + +/** Companion repo with folders that have spaces; S01 has a deck and a cheat sheet, S02 has code. */ +export async function makeRepoTree(root: string): Promise { + const top = join(root, "Course-Repo-main"); + await makePdf(join(top, "S01 - Intro to Docker", "Slides_Intro.pdf"), ["Repo deck one", "Repo deck two", "Repo deck three"]); + await makePdf(join(top, "S01 - Intro to Docker", "Docker_CheatSheet.pdf"), ["cheats"]); + await mkdir(join(top, "S02 - Images (Sample)", "app"), { recursive: true }); + await writeFile(join(top, "S02 - Images (Sample)", "app", "main.py"), "print('hi')\n"); + await writeFile(join(top, "S02 - Images (Sample)", "Ünïcode notes.txt"), "notes\n"); + await mkdir(join(top, "Bonus Extras"), { recursive: true }); + await writeFile(join(top, "Bonus Extras", "x.txt"), "x"); + await writeFile(join(top, "README.md"), "# repo\n"); + return top; +} + +export async function makeRepoZip(dir: string): Promise { + const tree = join(dir, "repo-tree"); + await makeRepoTree(tree); + const zip = join(dir, "repo.zip"); + await zipDir(tree, zip); + return zip; +} diff --git a/packages/feature-import/test/media.test.ts b/packages/feature-import/test/media.test.ts new file mode 100644 index 0000000..7704dc3 --- /dev/null +++ b/packages/feature-import/test/media.test.ts @@ -0,0 +1,38 @@ +import { expect, test } from "bun:test"; +import { join } from "node:path"; +import { readFile, stat } from "node:fs/promises"; +import { extractFrame, probeDuration, putFile } from "../src/media"; +import { renderPdf } from "../src/pdf"; +import { makePdf, makeVideo, PYTHON, tmp } from "./fixtures"; + +const tools = { ffmpeg: "ffmpeg", ffprobe: "ffprobe" }; + +test("duration and frame extraction on a tiny clip", async () => { + const d = await tmp(); + await makeVideo(join(d, "a.mp4"), 2); + expect(await probeDuration(tools, join(d, "a.mp4"))).toBe(2); + await extractFrame(tools, join(d, "a.mp4"), join(d, "out", "f.jpg")); + expect((await stat(join(d, "out", "f.jpg"))).size).toBeGreaterThan(100); +}); + +test("putFile copies and links without touching the original", async () => { + const d = await tmp(); + await makeVideo(join(d, "sp ace", "ö.mp4"), 1); + const before = await readFile(join(d, "sp ace", "ö.mp4")); + await putFile(join(d, "sp ace", "ö.mp4"), join(d, "c", "x.mp4"), "copy"); + await putFile(join(d, "sp ace", "ö.mp4"), join(d, "l", "x.mp4"), "link"); + expect(Buffer.compare(await readFile(join(d, "c", "x.mp4")), before)).toBe(0); + expect((await stat(join(d, "l", "x.mp4"))).ino).toBe((await stat(join(d, "sp ace", "ö.mp4"))).ino); +}); + +test("renderPdf writes page images and text", async () => { + const d = await tmp(); + await makePdf(join(d, "deck.pdf"), ["Hello alpha", "Second beta"]); + const seen: number[] = []; + const r = await renderPdf(PYTHON, join(d, "deck.pdf"), join(d, "out"), { onPage: (p) => seen.push(p) }); + expect(r.pages).toBe(2); + expect(seen).toEqual([1, 2]); + expect((await stat(join(d, "out", "p-002.jpg"))).size).toBeGreaterThan(100); + const text = JSON.parse(await readFile(join(d, "out", "text.json"), "utf8")); + expect(text[0].text).toContain("Hello alpha"); +}); diff --git a/packages/feature-import/test/util.test.ts b/packages/feature-import/test/util.test.ts new file mode 100644 index 0000000..7f04d61 --- /dev/null +++ b/packages/feature-import/test/util.test.ts @@ -0,0 +1,41 @@ +import { expect, test } from "bun:test"; +import { mkdtemp, mkdir, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { decodeEntities, naturalSort, slugify, uniqueId } from "../src/util/text"; +import { run } from "../src/util/exec"; +import { readRawManifest, writeManifest } from "../src/util/manifest"; + +test("slugify folds accents and falls back", () => { + expect(slugify("Mastering Docker Essentials - Hands-on DevOps!")).toBe("mastering-docker-essentials-hands-on-devops"); + expect(slugify("Café Ünïcode")).toBe("cafe-unicode"); + expect(slugify("日本語")).toBe("course"); +}); + +test("uniqueId appends a counter", () => { + expect(uniqueId("a", (x) => x === "a" || x === "a-2")).toBe("a-3"); + expect(uniqueId("a", () => false)).toBe("a"); +}); + +test("decodeEntities handles named and numeric entities", () => { + expect(decodeEntities("you'll & <b> "x" A")).toBe(`you'll & "x" A`); +}); + +test("naturalSort orders numbers numerically", () => { + expect(naturalSort(["10 b.mp4", "2 a.mp4", "1 z.mp4"])).toEqual(["1 z.mp4", "2 a.mp4", "10 b.mp4"]); +}); + +test("run reports stderr on failure", async () => { + expect(await run(["echo", "hi"])).toContain("hi"); + await expect(run(["sh", "-c", "echo oops >&2; exit 3"])).rejects.toThrow(/oops/); +}); + +test("writeManifest validates and round-trips", async () => { + const dir = await mkdtemp(join(tmpdir(), "stc-m-")); + await mkdir(join(dir, "c")); + const m = { schema: 1, id: "x", title: "T: with colon", visibility: "restricted", modules: [{ id: "m1", title: "S08 - Foo", lessons: [] }] }; + await writeManifest(join(dir, "c"), m); + expect((await readRawManifest(join(dir, "c")) as any).modules[0].title).toBe("S08 - Foo"); + await expect(writeManifest(join(dir, "c"), { ...m, id: "BAD ID" })).rejects.toThrow(/id/); + await writeFile(join(dir, "c", "marker"), ""); +}); From c2a3464937810be6394ea89680e7f31042060f25 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 03:13:33 +0100 Subject: [PATCH 04/62] docs(ai): WP-D implementation plan Co-Authored-By: Claude Opus 5.5 --- docs/superpowers/plans/2026-10-30-ai.md | 178 ++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-30-ai.md diff --git a/docs/superpowers/plans/2026-10-30-ai.md b/docs/superpowers/plans/2026-10-30-ai.md new file mode 100644 index 0000000..6578ce5 --- /dev/null +++ b/docs/superpowers/plans/2026-10-30-ai.md @@ -0,0 +1,178 @@ +# WP-D AI Implementation Plan + +> **For agentic workers:** executed inline by the WP-D agent with TDD (failing test first), one commit per task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Opt-in AI for STCourses: a course-aware learner assistant (retrieval, course tools, MCP tools, mentor mode) and a studio generator that writes house-style drafts. AI is off until an admin configures a server provider or a user saves their own key (BYOK). + +**Architecture:** +- `packages/feature-ai` exports `aiFeature(ctx, opts?): Feature`, `aiWeb: WebFeature` and `migrationsDirs`. +- Providers implement the frozen `AiProvider` contract with plain `fetch` (no SDK): Anthropic Messages API (streaming SSE, tool use, default model `claude-opus-5-5`) and OpenAI-compatible chat completions (covers Ollama). +- Settings live in the existing `ai_settings` and `user_api_keys` tables (no new tables). Secrets are AES-GCM encrypted with a key derived by HKDF-SHA-256 from `ctx.config.secret`; they never leave the server. +- Retrieval is an in-process BM25 index per loaded course (course Markdown, transcripts, code, deck/transcript text under `/cache//`), plus the requesting user's notes at query time. Solution files are never indexed. +- The assistant runs a small tool loop over `search_course`, `read_lesson`, `list_notes` and admin-configured MCP servers (stdio and streamable HTTP), and streams NDJSON events to the browser. +- The generator builds house-style prompts (Pure Snake module shape, Systemic Design beats), parses `=== FILE: … ===` blocks from the model output, and hands them to a `DraftSink`. The default sink writes to `/drafts///`; Phase 2 swaps in the studio's `createDraftWriter(ctx)`. + +**Tech Stack:** Bun 1.3, TypeScript strict, WebCrypto, `Bun.SQL`, `Bun.spawn`, React 19 + React Router 6, `@stc/ui` primitives. + +**Spec:** sections 9.2, 7, 2 (anti-goals). Brief: `.superpowers/sdd/2026-10-30-stcourses/phase-1-brief.md` (WP-D). + +## Global Constraints + +Inherited from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun only, plain CSS with tokens, both databases, AGPL, never write into the reference course repos, never commit licensed content, keyboard rules, commit trailer). Package-specific: + +- **AI is off by default.** No provider configured and no BYOK key means every AI route answers `enabled: false` / 409, and the UI says how to turn it on. +- **Keys never go to the client.** API responses carry `hasApiKey: boolean`, never ciphertext or plaintext. +- **No real network in tests.** Providers take an injectable `fetch`; MCP tests use a local fake server (stdio child process and `Bun.serve` on port 0). +- **Mentor mode is on by default** and its rules go into the system prompt; when off, the UI shows "Mentor mode off: I'll give full answers". +- **Generated material is never published.** The generator only writes through a `DraftSink`. + +## Review Focus + +1. **Secrets:** BYOK and server keys are encrypted at rest (AES-GCM, random IV, HKDF key from `STC_SECRET`), a wrong secret fails to decrypt, and no route returns key material. Tests: Tasks 1, 2, 9. +2. **Mentor anti-goal:** mentor rules are in the system prompt unless the learner switched them off for that course; solution files (`browser.solution`, `solutions/`, `reference/`, `expansion/solution/`) are never retrieved or readable by tools. Tests: Tasks 5, 7. +3. **Generation never publishes:** drafts land only under the sink location, paths are sanitised (no `..`, no absolute paths). Test: Task 8. +4. **Access control:** assistant routes require that the user can view the course (`canView`); generator routes require owner, the `author` role or a `course_authors` row; admin routes require owner. Test: Task 9. +5. **Thinking blocks:** the Anthropic provider replays an assistant turn's raw content blocks (thinking, text, tool_use) unchanged inside one tool loop. Test: Task 3. + +--- + +### Task 1: Package scaffold and secret box + +**Files:** `packages/feature-ai/{package.json,src/index.ts,src/crypto.ts}`, test `packages/feature-ai/test/crypto.test.ts` + +**Interfaces (Produces):** +- `createSecretBox(secret: string): { encrypt(plain: string): Promise; decrypt(token: string): Promise }`. HKDF-SHA-256 (salt `"stcourses/byok"`, info `"aes-gcm-256/v1"`) → AES-GCM-256; token `v1..`. +- `migrationsDirs: string[]` (empty: the package adds no tables). + +- [ ] Tests: round trip; two encryptions differ (random IV); a box with another secret rejects; a tampered token rejects. +- [ ] Implement; commit `feat(ai): package scaffold and AES-GCM secret box`. + +### Task 2: Settings repository + +**Files:** `src/settings.ts`, test `test/settings.test.ts` (uses `forEachDb` from `packages/db/test/helpers.ts`) + +**Interfaces:** +- `ProviderKind = "anthropic" | "openai"`; `ProviderConfig = { kind, model?, baseUrl?, effort? }`. +- `McpServerConfig = { id, name, transport: "stdio" | "http", command?, args?, env?, url?, headers?, courseIds?: string[] }` (no `courseIds` = global). +- `aiSettingsRepo(db, box)`: + - `getServer(): Promise<{ provider: (ProviderConfig & { hasApiKey }) | null; mcpServers }>`, `saveServer({ provider: (ProviderConfig & { apiKey?: string; clearApiKey?: boolean }) | null, mcpServers })`, `serverApiKey(): Promise`. + - `getUser(userId): Promise<{ byok: (ProviderConfig & { hasApiKey }) | null; mentorOff: Record }>`, `saveUserKey(userId, cfg & { apiKey })`, `removeUserKey(userId)`, `userApiKey(userId, kind)`, `setMentorOff(userId, courseId, off)`, `isMentorOff(userId, courseId)`. +- Storage: `ai_settings` row `('server','')` holds `{ provider, apiKeyCipher, mcpServers }`; row `('user', userId)` holds `{ byok, mentorOff }`. The per-course flag lives in `config_json` because the frozen primary key is `(scope, owner_id)`; the `mentor_off` column mirrors "off for every course" (always 0 for now). Contract request filed. Keys go in `user_api_keys` encrypted. Upserts are select-then-insert-or-update in a transaction. + +- [ ] Tests (both DBs): empty server config is `null` provider; saving a key stores ciphertext only (raw row does not contain the key) and `serverApiKey()` returns it; omitting `apiKey` on save keeps the old key; BYOK save/remove; mentor toggle per course is independent. +- [ ] Implement; commit `feat(ai): settings repository for server config, BYOK keys and mentor toggles`. + +### Task 3: SSE parsing and the Anthropic provider + +**Files:** `src/providers/sse.ts`, `src/providers/anthropic.ts`, test `test/anthropic.test.ts` + +**Interfaces:** +- `readSse(body: ReadableStream): AsyncIterable<{ event: string; data: string }>`. +- `createAnthropicProvider({ apiKey, model = "claude-opus-5-5", baseUrl = "https://api.anthropic.com", effort?, maxTokens = 64000, fetch? }): AiProvider`. + - POST `/v1/messages` with `stream: true`, headers `x-api-key`, `anthropic-version: 2023-06-01`. No `thinking`, `temperature` or forced `tool_choice` (Opus 5.5 rejects them). On the first-party endpoint, `fallbacks: "default"` with beta header `server-side-fallback-2026-07-01`. + - Message mapping: `system` messages join into `system`; an assistant `ChatMessage` with `toolCallId` and `name` is a tool call (content = JSON input); `tool` messages become `tool_result` blocks; consecutive same-role blocks merge. + - The provider remembers each response's full content blocks (thinking with signature, text, tool_use) by tool_use id and replays them unchanged for the matching assistant tool-call messages. + - Streams `{ text }` per `text_delta`, `{ toolCall }` per finished `tool_use` block; HTTP errors and `error` events throw with the API message; `stop_reason: "refusal"` yields a plain sentence. + +- [ ] Tests with a fake `fetch` that returns canned SSE: text streams in order; a tool_use with split `input_json_delta` yields one parsed toolCall; request body has model, system, tools as `input_schema`, no `thinking`/`temperature`; a second call after a tool round replays the thinking block with its signature unchanged before `tool_result`; non-2xx throws with the error message. +- [ ] Implement; commit `feat(ai): Anthropic provider over fetch with streaming tool use`. + +### Task 4: OpenAI-compatible provider + +**Files:** `src/providers/openai.ts`, `src/providers/index.ts`, test `test/openai.test.ts` + +**Interfaces:** +- `createOpenAiProvider({ baseUrl, apiKey?, model, fetch? }): AiProvider`. POST `/chat/completions`, `stream: true`, tools as `{ type: "function", function: { name, description, parameters } }`. Assistant tool-call messages map to `tool_calls`; `tool` messages to `{ role: "tool", tool_call_id }`. Accumulates `delta.tool_calls` by index and emits on finish or `[DONE]`. +- `createProvider(cfg: ProviderConfig, apiKey: string | null, fetchImpl?): AiProvider`. + +- [ ] Tests: text deltas; fragmented tool-call arguments; no `Authorization` header without a key (Ollama); request mapping of a tool round. +- [ ] Implement; commit `feat(ai): OpenAI-compatible provider (covers Ollama)`. + +### Task 5: Retrieval + +**Files:** `src/retrieval/{bm25.ts,course-index.ts}`, test `test/retrieval.test.ts` + +**Interfaces:** +- `Bm25` with `add(text, item)`, `search(query, k): { item, score }[]`. +- `Chunk = { path, kind: "lesson" | "transcript" | "markdown" | "code" | "deck" | "note", text, heading?, lessonId?, lessonTitle? }`. +- `buildCourseIndex(course: LoadedCourse, dataDir: string): Promise`; `CourseIndex.search(query, k, extra?: Chunk[]): Chunk[]`; `isSolutionPath(course, rel): boolean`. +- `getCourseIndex(course, dataDir)` caches per `LoadedCourse` object (a repository reload yields new objects) with a 5-minute TTL. +- Sources: every `.md` under the course root (depth-limited, skipping `.git`, `node_modules`), manifest code dirs and common code extensions (size cap 200 KB per file, 2 000 files per course), `.md/.txt/.vtt/.srt/.json` text under `/cache//`. Markdown splits at headings; long chunks split at ~1 500 chars. Paths are mapped to lessons through `file`, `transcript` and `hints.file` (anchor heading when it matches). +- Never indexed: `browser.solution` files and any path segment `solutions`, `solution`, `reference` under `expansion/`, or `reference/` in Systemic-Design style modules (see `isSolutionPath`). + +- [ ] Tests on the `sample` fixture: "sum of its arguments" hits `lessons/04-exercise.md` with lessonId `m2-text` or `m2-ex-add`; `exercises/add/solution.js` never appears; a deck text file in a temp `cache/sample/decks/d1/page-1.txt` is found with kind `deck`; note chunks passed as `extra` rank. +- [ ] Implement; commit `feat(ai): BM25 course retrieval over lessons, transcripts, code, deck text and notes`. + +### Task 6: MCP client + +**Files:** `src/mcp/client.ts`, test `test/mcp.test.ts`, fixture `test/fixtures/fake-mcp.ts` + +**Interfaces:** +- `connectMcp(cfg: McpServerConfig, opts?: { fetch?, timeoutMs? }): Promise`; `McpClient = { tools(): Promise<{ name, description, inputSchema }[]>; call(name, args): Promise; close(): Promise }`. +- JSON-RPC 2.0: `initialize` (protocolVersion `2025-06-18`), `notifications/initialized`, `tools/list`, `tools/call`. stdio = newline-delimited JSON over `Bun.spawn`; streamable HTTP = POST with `Accept: application/json, text/event-stream`, JSON or SSE replies, `Mcp-Session-Id` echoed. +- `mcpServersFor(courseId, servers)` selects global servers plus ones listing the course. + +- [ ] Tests: the fake server over stdio lists `echo` and `tools/call` returns its text; the same over HTTP (fake via `Bun.serve` on port 0, replying with SSE for `tools/call`) with the session id echoed; a server that exits gives a rejected promise, not a hang. +- [ ] Implement; commit `feat(ai): MCP client over stdio and streamable HTTP`. + +### Task 7: Assistant core + +**Files:** `src/assistant/{prompt.ts,tools.ts,loop.ts}`, test `test/assistant.test.ts` + +**Interfaces:** +- `buildSystemPrompt({ courseTitle, mentorText: string | null, mentorOn, sources: Chunk[], lessonTitle? }): string`. With mentor on and text present: a `## Course mentor rules` section with the file's text. With mentor off: a line saying full answers are allowed. Sources are numbered `[n] path (lesson title)` with their text; the prompt asks for citations by path. +- `courseTools({ ctx, course, userId, index })`: `ToolSpec[]` plus `run(name, input)` for `search_course`, `read_lesson` (refuses solution files) and `list_notes`. +- `runAssistant({ provider, messages, tools, signal, maxSteps = 6 }): AsyncIterable` where `AssistantEvent = { type: "text", text } | { type: "tool", name } | { type: "error", message } | { type: "done", text }`. Tool calls become assistant messages with `toolCallId`/`name` and `tool` result messages; unknown tools and tool errors are fed back as error text. +- Citations: `citationsFor(chunks): { path, lessonId?, lessonTitle?, kind }[]` deduplicated. + +- [ ] Tests: mentor on puts the sample `MENTOR.md` text in the system prompt; mentor off omits it and states full answers; a fake provider that calls `search_course` then answers gets the tool result and the loop ends with text; `read_lesson` on the exercise returns markdown; solution paths are refused. +- [ ] Implement; commit `feat(ai): assistant prompt, course tools and tool loop`. + +### Task 8: Generator and draft sink + +**Files:** `src/generator/{templates.ts,parse.ts,generate.ts,draft-sink.ts}`, test `test/generator.test.ts` + +**Interfaces:** +- `DraftSink = { writeDraftFiles(courseId, authorId, files: { path, content }[], message): Promise<{ location: string; paths: string[] }> }`; `createFsDraftSink(dataDir)` writes to `/drafts///` plus `DRAFT.json` (author, message, time). Same signature as the studio's `createDraftWriter(ctx).writeDraftFiles`. +- `GenerateRequest = { courseId, kind: "document" | "exercise" | "questions", style: "pure-snake" | "systemic-design", topic, sources: ({ kind: "lesson", lessonId } | { kind: "file", path } | { kind: "text", label, text })[], instructions? }`. +- `buildGeneratorPrompt(req, material, course)`: house-style system prompt (module shape: why, objectives, concepts, puzzle with "Think about it first" fold, everyday-programming bridge, exercises, hints as nudge / stronger hint / near-solution folds, check your understanding, in production, further reading; or Observe → Predict (+Discussion fold) → Spec → Build (Hint 1 — nudge / Hint 2 — concept / Hint 3 — pseudo-code) → Name it → Expansion Drop (+what this drop measures) → Further reading). Exercises: `README.md`, `starter.js`, `tests.js` (`export const tests` per the `ExerciseTest` contract), `solution.js`, shared fixture JSON across tracks when the course has tracks. Questions: check-your-understanding list with at least one predict and one "what would break if". +- `parseFiles(text): { files, warnings }` reads `=== FILE: ===` … `=== END FILE ===`; rejects absolute and `..` paths. +- `checkHouseStyle(req, files): string[]` warnings for missing beats/tiers. +- `generateDraft({ provider, req, course, ctx, sink, authorId }): Promise<{ location, paths, warnings }>`; every Markdown file starts with a draft marker comment. + +- [ ] Tests: prompt for each style contains its beats; parse handles two files and rejects `../x`; fake provider output written by the fs sink lands under `/drafts/sample/…` and nothing changes in the course folder; missing tiers give warnings. +- [ ] Implement; commit `feat(ai): house-style generator writing drafts through a DraftSink`. + +### Task 9: Feature routes and jobs + +**Files:** `src/feature.ts`, test `test/feature.test.ts` + +**Interfaces:** `aiFeature(ctx, opts?: { fetch?; draftSink?; providerFactory?(cfg, key): AiProvider }): Feature` with: +- `GET /api/ai/status?courseId=` → `{ enabled, source: "byok" | "server" | null, mentor?: { available, on } }`. +- `GET|PUT /api/ai/admin/config` (owner), `POST /api/ai/admin/mcp/test` (owner; lists a server's tools). +- `GET /api/ai/account`, `PUT /api/ai/account/key`, `DELETE /api/ai/account/key` (signed in). +- `PUT /api/ai/courses/:courseId/mentor` `{ off }`. +- `POST /api/ai/courses/:courseId/chat` `{ message, conversationId? }` → `application/x-ndjson` stream: `{type:"meta", conversationId, mentorOn, citations}`, `text`, `tool`, `done`, `error`. Stores user and assistant turns in `ai_conversations`/`ai_messages`. +- `GET /api/ai/courses/:courseId/conversations`, `GET|DELETE /api/ai/conversations/:id`. +- `GET /api/ai/studio/sources?courseId=`, `POST /api/ai/studio/generate` (enqueues job `ai.generate`), `GET /api/ai/studio/jobs/:id`. +- Job `ai.generate` runs `generateDraft` with the requesting user's provider. + +- [ ] Tests through `createApp` with a fake `AuthService`: status disabled by default; admin config hides the key; non-owner gets 403; BYOK enables the assistant for that user only; chat with a fake provider streams text and `meta.citations` include `lessons/04-exercise.md`; system prompt holds mentor rules, and not after `PUT mentor {off:true}`; anonymous on a restricted course gets 404; generate as a learner gets 403; the job handler writes drafts under `/drafts/sample`. +- [ ] Implement; commit `feat(ai): HTTP routes, chat streaming and generation job`. + +### Task 10: Web UI + +**Files:** `src/web/{index.ts,api.ts,AssistantPage.tsx,AdminAiPage.tsx,AccountAiPage.tsx,GeneratePage.tsx,ai.css}`, `apps/web/src/features.ts` (one line), test `test/web.test.tsx` + +- `aiWeb: WebFeature` routes `/c/:courseId/assistant`, `/admin/ai` (nav admin "AI"), `/account/ai`, `/studio/generate`. +- Assistant: course name, mentor segmented toggle, the plain banner "Mentor mode off: I'll give full answers" when off, conversation rows, sources as rows with paths, composer with a primary Send pill. Disabled state explains how to enable AI. +- Admin: provider kind, model, base URL, write-only key, MCP server rows. Account: BYOK form and remove. Generate: course, kind, style, sources checklist, topic, instructions, job progress, list of draft files with location. +- `readNdjson(stream)` client helper. + +- [ ] Tests: `readNdjson` splits across chunk boundaries; `renderToString` of the banner component shows the exact sentence when off and nothing when on. +- [ ] Implement, check in headless Chrome via `packages/feature-ai/scripts/dev.ts` on port 3464; commit `feat(ai): assistant panel, AI settings and studio generator UI`. + +### Task 11: Contract requests and report + +- [ ] `docs/contract-requests/ai.md`: feature factories in `startServer`; tool-call fields on `ChatMessage`; per-course `ai_settings` key; studio `createDraftWriter` wiring; cache layout for deck text and transcripts; learner lesson URL. +- [ ] Run `bun test`, `bun run typecheck`, the Postgres run; write `wp-ai-report.md`. From e1938b5be8a3f15bf1a78e6915dae42302c716c9 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 03:13:50 +0100 Subject: [PATCH 05/62] feat(ai): package scaffold and AES-GCM secret box Co-Authored-By: Claude Opus 5.5 --- packages/feature-ai/package.json | 3 +- packages/feature-ai/src/crypto.ts | 41 +++++++++++++++++++++++++ packages/feature-ai/src/index.ts | 4 +++ packages/feature-ai/test/crypto.test.ts | 27 ++++++++++++++++ 4 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 packages/feature-ai/src/crypto.ts create mode 100644 packages/feature-ai/src/index.ts create mode 100644 packages/feature-ai/test/crypto.test.ts diff --git a/packages/feature-ai/package.json b/packages/feature-ai/package.json index 689f090..0a7eb1e 100644 --- a/packages/feature-ai/package.json +++ b/packages/feature-ai/package.json @@ -3,5 +3,6 @@ "version": "0.0.0", "private": true, "type": "module", - "license": "AGPL-3.0-only" + "license": "AGPL-3.0-only", + "main": "src/index.ts" } diff --git a/packages/feature-ai/src/crypto.ts b/packages/feature-ai/src/crypto.ts new file mode 100644 index 0000000..10f6456 --- /dev/null +++ b/packages/feature-ai/src/crypto.ts @@ -0,0 +1,41 @@ +/** + * Secret box for API keys at rest: AES-GCM-256 with a key derived by HKDF-SHA-256 from STC_SECRET. + * Token format: `v1..`. + */ +export interface SecretBox { + encrypt(plain: string): Promise; + decrypt(token: string): Promise; +} + +const enc = new TextEncoder(); +const dec = new TextDecoder(); +const b64 = (u: Uint8Array) => Buffer.from(u).toString("base64"); +const unb64 = (s: string) => new Uint8Array(Buffer.from(s, "base64")); + +export function createSecretBox(secret: string): SecretBox { + let keyP: Promise | null = null; + const key = () => + (keyP ??= (async () => { + const ikm = await crypto.subtle.importKey("raw", enc.encode(secret), "HKDF", false, ["deriveKey"]); + return crypto.subtle.deriveKey( + { name: "HKDF", hash: "SHA-256", salt: enc.encode("stcourses/byok"), info: enc.encode("aes-gcm-256/v1") }, + ikm, + { name: "AES-GCM", length: 256 }, + false, + ["encrypt", "decrypt"], + ); + })()); + return { + async encrypt(plain) { + const iv = crypto.getRandomValues(new Uint8Array(12)); + const ct = new Uint8Array(await crypto.subtle.encrypt({ name: "AES-GCM", iv }, await key(), enc.encode(plain))); + return `v1.${b64(iv)}.${b64(ct)}`; + }, + async decrypt(token) { + const [v, iv, ct] = token.split("."); + if (v !== "v1" || !iv || !ct) throw new Error("Unrecognised secret format"); + const plain = await crypto.subtle.decrypt({ name: "AES-GCM", iv: unb64(iv) }, await key(), unb64(ct)); + return dec.decode(plain); + }, + }; +} diff --git a/packages/feature-ai/src/index.ts b/packages/feature-ai/src/index.ts new file mode 100644 index 0000000..404b2e2 --- /dev/null +++ b/packages/feature-ai/src/index.ts @@ -0,0 +1,4 @@ +export { createSecretBox, type SecretBox } from "./crypto"; + +/** Migration folders for this package. Empty: the AI package uses the ai_* tables from 0001_init.sql. */ +export const migrationsDirs: string[] = []; diff --git a/packages/feature-ai/test/crypto.test.ts b/packages/feature-ai/test/crypto.test.ts new file mode 100644 index 0000000..7bea5b8 --- /dev/null +++ b/packages/feature-ai/test/crypto.test.ts @@ -0,0 +1,27 @@ +import { expect, test } from "bun:test"; +import { createSecretBox } from "../src/crypto"; + +test("round trips and uses a fresh IV each time", async () => { + const box = createSecretBox("s3cret"); + const a = await box.encrypt("sk-ant-123"); + const b = await box.encrypt("sk-ant-123"); + expect(a).not.toBe(b); + expect(a).not.toContain("sk-ant-123"); + expect(a.startsWith("v1.")).toBe(true); + expect(await box.decrypt(a)).toBe("sk-ant-123"); + expect(await box.decrypt(b)).toBe("sk-ant-123"); +}); + +test("another secret cannot decrypt", async () => { + const token = await createSecretBox("one").encrypt("key"); + await expect(createSecretBox("two").decrypt(token)).rejects.toThrow(); +}); + +test("a tampered token is rejected", async () => { + const token = await createSecretBox("one").encrypt("key"); + const parts = token.split("."); + const ct = Buffer.from(parts[2]!, "base64"); + ct[0] = ct[0]! ^ 1; + await expect(createSecretBox("one").decrypt(`${parts[0]}.${parts[1]}.${ct.toString("base64")}`)).rejects.toThrow(); + await expect(createSecretBox("one").decrypt("garbage")).rejects.toThrow(); +}); From bccbc692e902a7577e845284ab82d108b0859242 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 03:14:29 +0100 Subject: [PATCH 06/62] feat(ai): settings repository for server config, BYOK keys and mentor toggles Co-Authored-By: Claude Opus 5.5 --- packages/feature-ai/src/index.ts | 1 + packages/feature-ai/src/settings.ts | 131 ++++++++++++++++++++++ packages/feature-ai/test/settings.test.ts | 63 +++++++++++ 3 files changed, 195 insertions(+) create mode 100644 packages/feature-ai/src/settings.ts create mode 100644 packages/feature-ai/test/settings.test.ts diff --git a/packages/feature-ai/src/index.ts b/packages/feature-ai/src/index.ts index 404b2e2..19b39f1 100644 --- a/packages/feature-ai/src/index.ts +++ b/packages/feature-ai/src/index.ts @@ -2,3 +2,4 @@ export { createSecretBox, type SecretBox } from "./crypto"; /** Migration folders for this package. Empty: the AI package uses the ai_* tables from 0001_init.sql. */ export const migrationsDirs: string[] = []; +export * from "./settings"; diff --git a/packages/feature-ai/src/settings.ts b/packages/feature-ai/src/settings.ts new file mode 100644 index 0000000..0d1360c --- /dev/null +++ b/packages/feature-ai/src/settings.ts @@ -0,0 +1,131 @@ +import type { Db } from "@stc/db"; +import type { SecretBox } from "./crypto"; + +export type ProviderKind = "anthropic" | "openai"; +export type Effort = "low" | "medium" | "high" | "xhigh" | "max"; +export interface ProviderConfig { kind: ProviderKind; model?: string; baseUrl?: string; effort?: Effort } +export type PublicProviderConfig = ProviderConfig & { hasApiKey: boolean }; + +export interface McpServerConfig { + id: string; + name: string; + transport: "stdio" | "http"; + command?: string; + args?: string[]; + env?: Record; + url?: string; + headers?: Record; + /** Courses this server is offered in. Absent or empty: every course (global). */ + courseIds?: string[]; +} + +export interface ServerSettings { provider: PublicProviderConfig | null; mcpServers: McpServerConfig[] } +export interface ServerSettingsInput { + provider: (ProviderConfig & { apiKey?: string; clearApiKey?: boolean }) | null; + mcpServers: McpServerConfig[]; +} +export interface UserSettings { byok: PublicProviderConfig | null; mentorOff: Record } + +type ServerRow = { provider: ProviderConfig | null; apiKeyCipher: string | null; mcpServers: McpServerConfig[] }; +type UserRow = { byok: ProviderConfig | null; mentorOff: Record }; + +const pickProvider = (p: ProviderConfig): ProviderConfig => { + const out: ProviderConfig = { kind: p.kind }; + if (p.model) out.model = p.model; + if (p.baseUrl) out.baseUrl = p.baseUrl; + if (p.effort) out.effort = p.effort; + return out; +}; + +const parse = (s: string | undefined, fallback: T): T => { + if (!s) return fallback; + try { return { ...fallback, ...JSON.parse(s) }; } catch { return fallback; } +}; + +/** + * AI settings in the ai_settings and user_api_keys tables of 0001_init.sql. + * - ('server', '') holds the admin provider (key encrypted) and MCP servers. + * - ('user', ) holds the BYOK provider choice and per-course mentor toggles. The frozen primary key + * is (scope, owner_id), so the per-course flag lives in config_json; the mentor_off column means + * "off for every course" and stays 0 (see docs/contract-requests/ai.md). + * Keys never leave this module except through serverApiKey/userApiKey, which only server code calls. + */ +export function aiSettingsRepo(db: Db, box: SecretBox) { + const read = async (scope: "server" | "user", owner: string): Promise => { + const rows = await db.sql`SELECT config_json FROM ai_settings WHERE scope = ${scope} AND owner_id = ${owner}`; + return rows[0]?.config_json; + }; + const write = async (scope: "server" | "user", owner: string, value: unknown) => { + const json = JSON.stringify(value); + const now = new Date().toISOString(); + await db.sql.begin(async (tx) => { + const exists = await tx`SELECT 1 AS x FROM ai_settings WHERE scope = ${scope} AND owner_id = ${owner}`; + if (exists.length) await tx`UPDATE ai_settings SET config_json = ${json}, updated_at = ${now} WHERE scope = ${scope} AND owner_id = ${owner}`; + else await tx`INSERT INTO ai_settings (scope, owner_id, mentor_off, config_json, updated_at) VALUES (${scope}, ${owner}, 0, ${json}, ${now})`; + }); + }; + const serverRow = async (): Promise => parse(await read("server", ""), { provider: null, apiKeyCipher: null, mcpServers: [] }); + const userRow = async (userId: string): Promise => parse(await read("user", userId), { byok: null, mentorOff: {} }); + const keyRow = async (userId: string) => + (await db.sql`SELECT provider, ciphertext FROM user_api_keys WHERE user_id = ${userId}`)[0] as { provider: string; ciphertext: string } | undefined; + + return { + async getServer(): Promise { + const r = await serverRow(); + return { provider: r.provider ? { ...r.provider, hasApiKey: !!r.apiKeyCipher } : null, mcpServers: r.mcpServers ?? [] }; + }, + async saveServer(input: ServerSettingsInput): Promise { + const prev = await serverRow(); + let apiKeyCipher: string | null = null; + if (input.provider) { + if (input.provider.apiKey) apiKeyCipher = await box.encrypt(input.provider.apiKey); + else if (!input.provider.clearApiKey) apiKeyCipher = prev.apiKeyCipher; + } + await write("server", "", { + provider: input.provider ? pickProvider(input.provider) : null, + apiKeyCipher, + mcpServers: input.mcpServers ?? [], + }); + }, + async serverApiKey(): Promise { + const r = await serverRow(); + return r.provider && r.apiKeyCipher ? box.decrypt(r.apiKeyCipher) : null; + }, + async getUser(userId: string): Promise { + const r = await userRow(userId); + const k = await keyRow(userId); + return { byok: r.byok && k ? { ...r.byok, hasApiKey: true } : null, mentorOff: r.mentorOff ?? {} }; + }, + async saveUserKey(userId: string, cfg: ProviderConfig & { apiKey: string }): Promise { + const ciphertext = await box.encrypt(cfg.apiKey); + const now = new Date().toISOString(); + await db.sql.begin(async (tx) => { + await tx`DELETE FROM user_api_keys WHERE user_id = ${userId}`; + await tx`INSERT INTO user_api_keys (user_id, provider, ciphertext, created_at) VALUES (${userId}, ${cfg.kind}, ${ciphertext}, ${now})`; + }); + const r = await userRow(userId); + await write("user", userId, { ...r, byok: pickProvider(cfg) }); + }, + async removeUserKey(userId: string): Promise { + await db.sql`DELETE FROM user_api_keys WHERE user_id = ${userId}`; + const r = await userRow(userId); + await write("user", userId, { ...r, byok: null }); + }, + async userApiKey(userId: string): Promise { + const k = await keyRow(userId); + return k ? box.decrypt(k.ciphertext) : null; + }, + async setMentorOff(userId: string, courseId: string, off: boolean): Promise { + const r = await userRow(userId); + const mentorOff = { ...r.mentorOff }; + if (off) mentorOff[courseId] = true; + else delete mentorOff[courseId]; + await write("user", userId, { ...r, mentorOff }); + }, + async isMentorOff(userId: string, courseId: string): Promise { + return (await userRow(userId)).mentorOff?.[courseId] === true; + }, + }; +} + +export type AiSettingsRepo = ReturnType; diff --git a/packages/feature-ai/test/settings.test.ts b/packages/feature-ai/test/settings.test.ts new file mode 100644 index 0000000..589c47f --- /dev/null +++ b/packages/feature-ai/test/settings.test.ts @@ -0,0 +1,63 @@ +import { expect } from "bun:test"; +import { forEachDb } from "../../db/test/helpers"; +import { createSecretBox } from "../src/crypto"; +import { aiSettingsRepo } from "../src/settings"; + +const box = createSecretBox("test-secret"); + +forEachDb("server config is empty by default and hides the key", async (db) => { + const r = aiSettingsRepo(db, box); + expect(await r.getServer()).toEqual({ provider: null, mcpServers: [] }); + expect(await r.serverApiKey()).toBeNull(); + + await r.saveServer({ provider: { kind: "anthropic", model: "claude-opus-5-5", apiKey: "sk-ant-SECRET" }, mcpServers: [] }); + const got = await r.getServer(); + expect(got.provider).toEqual({ kind: "anthropic", model: "claude-opus-5-5", hasApiKey: true }); + expect(JSON.stringify(got)).not.toContain("SECRET"); + const raw = await db.sql`SELECT config_json FROM ai_settings WHERE scope = 'server'`; + expect(raw[0].config_json).not.toContain("SECRET"); + expect(await r.serverApiKey()).toBe("sk-ant-SECRET"); + + // Saving without apiKey keeps the old key; clearApiKey removes it. + await r.saveServer({ provider: { kind: "anthropic", model: "other" }, mcpServers: [{ id: "fs", name: "Files", transport: "stdio", command: "x" }] }); + expect(await r.serverApiKey()).toBe("sk-ant-SECRET"); + expect((await r.getServer()).mcpServers).toHaveLength(1); + await r.saveServer({ provider: { kind: "openai", baseUrl: "http://ollama:11434/v1", model: "llama3", clearApiKey: true }, mcpServers: [] }); + expect(await r.serverApiKey()).toBeNull(); + expect((await r.getServer()).provider).toEqual({ kind: "openai", baseUrl: "http://ollama:11434/v1", model: "llama3", hasApiKey: false }); + await r.saveServer({ provider: null, mcpServers: [] }); + expect((await r.getServer()).provider).toBeNull(); +}); + +forEachDb("BYOK keys are encrypted, per user and removable", async (db) => { + const r = aiSettingsRepo(db, box); + expect((await r.getUser("u1")).byok).toBeNull(); + await r.saveUserKey("u1", { kind: "anthropic", apiKey: "sk-user-SECRET" }); + expect((await r.getUser("u1")).byok).toEqual({ kind: "anthropic", hasApiKey: true }); + expect((await r.getUser("u2")).byok).toBeNull(); + const raw = await db.sql`SELECT ciphertext FROM user_api_keys WHERE user_id = 'u1'`; + expect(raw[0].ciphertext).not.toContain("SECRET"); + expect(await r.userApiKey("u1")).toBe("sk-user-SECRET"); + expect(await r.userApiKey("u2")).toBeNull(); + // replacing with another provider drops the old key + await r.saveUserKey("u1", { kind: "openai", baseUrl: "https://api.example.com/v1", model: "m", apiKey: "k2" }); + expect(await r.userApiKey("u1")).toBe("k2"); + expect((await db.sql`SELECT COUNT(*) AS n FROM user_api_keys WHERE user_id = 'u1'`)[0].n.toString()).toBe("1"); + await r.removeUserKey("u1"); + expect((await r.getUser("u1")).byok).toBeNull(); + expect(await r.userApiKey("u1")).toBeNull(); +}); + +forEachDb("mentor mode is on by default and toggles per course", async (db) => { + const r = aiSettingsRepo(db, box); + expect(await r.isMentorOff("u1", "sample")).toBe(false); + await r.setMentorOff("u1", "sample", true); + expect(await r.isMentorOff("u1", "sample")).toBe(true); + expect(await r.isMentorOff("u1", "pure-snake")).toBe(false); + expect(await r.isMentorOff("u2", "sample")).toBe(false); + await r.saveUserKey("u1", { kind: "anthropic", apiKey: "k" }); + expect(await r.isMentorOff("u1", "sample")).toBe(true); + await r.setMentorOff("u1", "sample", false); + expect(await r.isMentorOff("u1", "sample")).toBe(false); + expect((await r.getUser("u1")).byok?.kind).toBe("anthropic"); +}); From a6f887f31c9da02ff3d5755f9034a8f4df9adcf2 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 03:14:47 +0100 Subject: [PATCH 07/62] feat(studio): git wrapper and StudioStore (create course, drafts, save as commit) Co-Authored-By: Claude Opus 5.5 --- packages/feature-studio/src/server/git.ts | 65 +++ packages/feature-studio/src/server/store.ts | 456 ++++++++++++++++++++ packages/feature-studio/test/helpers.ts | 25 ++ packages/feature-studio/test/store.test.ts | 118 +++++ 4 files changed, 664 insertions(+) create mode 100644 packages/feature-studio/src/server/git.ts create mode 100644 packages/feature-studio/src/server/store.ts create mode 100644 packages/feature-studio/test/helpers.ts create mode 100644 packages/feature-studio/test/store.test.ts diff --git a/packages/feature-studio/src/server/git.ts b/packages/feature-studio/src/server/git.ts new file mode 100644 index 0000000..9ce4dc4 --- /dev/null +++ b/packages/feature-studio/src/server/git.ts @@ -0,0 +1,65 @@ +import { $ } from "bun"; + +export interface Identity { name: string; email: string } +export interface GitResult { code: number; stdout: string; stderr: string } + +/** Settings that make studio git runs deterministic, whatever the machine's global config says. */ +const BASE = [ + "-c", "core.hooksPath=/dev/null", + "-c", "commit.gpgsign=false", + "-c", "core.quotepath=false", + "-c", "merge.conflictstyle=merge", + "-c", "pull.rebase=false", + "-c", "init.defaultBranch=main", +]; + +function env(who?: Identity): Record { + const e: Record = {}; + for (const [k, v] of Object.entries(process.env)) { + if (v !== undefined && !k.startsWith("GIT_")) e[k] = v; // never inherit GIT_DIR, GIT_INDEX_FILE, ... + } + e.GIT_TERMINAL_PROMPT = "0"; + e.GIT_SSH_COMMAND = "ssh -o BatchMode=yes"; + // A fallback identity so git never refuses a merge/commit; real saves always pass `who`. + const id = who ?? { name: "STCourses", email: "studio@stcourses.invalid" }; + e.GIT_AUTHOR_NAME = id.name; + e.GIT_AUTHOR_EMAIL = id.email; + e.GIT_COMMITTER_NAME = id.name; + e.GIT_COMMITTER_EMAIL = id.email; + return e; +} + +/** Runs `git ` in `cwd`. Never throws on a non-zero exit; use `gitOk` for that. */ +export async function git(cwd: string, args: string[], who?: Identity): Promise { + const r = await $`git ${BASE} ${args}`.cwd(cwd).env(env(who)).quiet().nothrow(); + return { code: r.exitCode, stdout: r.stdout.toString(), stderr: r.stderr.toString() }; +} + +export class GitError extends Error { + constructor(public args: string[], public result: GitResult) { + super(`git ${args.join(" ")} failed (${result.code}): ${result.stderr.trim() || result.stdout.trim()}`); + } +} + +export async function gitOk(cwd: string, args: string[], who?: Identity): Promise { + const r = await git(cwd, args, who); + if (r.code !== 0) throw new GitError(args, r); + return r.stdout; +} + +/** NUL-separated output (`-z`) to a list, so paths with spaces or newlines survive. */ +export function splitZ(out: string): string[] { + return out.split("\0").filter(Boolean); +} + +const locks = new Map>(); + +/** Serialises git work per key (the course's bare repo path) across every store instance in the process. */ +export function withLock(key: string, fn: () => Promise): Promise { + const prev = locks.get(key) ?? Promise.resolve(); + const next = prev.then(fn, fn); + const tail = next.catch(() => {}); + locks.set(key, tail); + tail.then(() => { if (locks.get(key) === tail) locks.delete(key); }); + return next; +} diff --git a/packages/feature-studio/src/server/store.ts b/packages/feature-studio/src/server/store.ts new file mode 100644 index 0000000..cb63def --- /dev/null +++ b/packages/feature-studio/src/server/store.ts @@ -0,0 +1,456 @@ +import { existsSync, mkdirSync, readdirSync, realpathSync, rmSync, statSync } from "node:fs"; +import { mkdir, rm, unlink, writeFile } from "node:fs/promises"; +import { dirname, isAbsolute, join, normalize, sep } from "node:path"; +import { loadCourse, type Diagnostic } from "@stc/course-format"; +import { git, gitOk, splitZ, withLock, type Identity } from "./git"; + +export type { Identity } from "./git"; + +export const MANIFEST = "stcourse.yaml"; +const ID_RE = /^[a-z0-9][a-z0-9-]*$/; + +export function contentHash(content: string | Uint8Array): string { + return new Bun.CryptoHasher("sha256").update(content).digest("hex"); +} + +/** Branch- and path-safe form of a user id. Distinct ids may collide only if they differ in unsafe characters. */ +export function safeAuthor(authorId: string): string { + const s = authorId.replace(/[^A-Za-z0-9_-]/g, "_"); + return s === "" ? "_" : s; +} + +/** Normalised relative path, or throws. Rejects absolute paths, `..`, `.git` and NUL. */ +export function checkRelPath(p: string): string { + if (typeof p !== "string" || p === "" || p.includes("\0") || isAbsolute(p) || p.includes("\\")) throw new Error(`invalid path: ${JSON.stringify(p)}`); + const n = normalize(p); + const parts = n.split(sep); + if (n === "." || parts[0] === ".." || parts.some((x) => x === ".." || x.toLowerCase() === ".git") || n.endsWith(sep)) { + throw new Error(`invalid path: ${JSON.stringify(p)}`); + } + return n; +} + +export interface FileWrite { + path: string; + content?: string | Uint8Array; + /** Hash of the content the editor started from: `null` = the file must not exist; omitted = no check. */ + baseHash?: string | null; + delete?: boolean; +} + +export interface FileConflict { path: string; current: string | null; currentHash: string | null } +export type SaveResult = { ok: true; commit: string | null } | { ok: false; conflicts: FileConflict[] }; + +export interface DraftStatus { + courseId: string; + branch: string; + publishedBranch: string; + draftHead: string; + publishedHead: string; + /** Draft commits not yet published. */ + ahead: number; + /** Published commits (external edits, pulls, other authors' publishes) not yet in the draft. */ + behind: number; + merging: boolean; + conflicts: string[]; + remote: string | null; + /** Uncommitted changes in the published worktree (someone editing it outside the studio). */ + publishedDirty: string[]; +} + +export type PublishResult = + | { ok: true; commit: string; published: boolean } + | { ok: false; reason: "conflict"; conflicts: string[] } + | { ok: false; reason: "dirty"; files: string[] } + | { ok: false; reason: "invalid"; diagnostics: Diagnostic[] } + | { ok: false; reason: "merging" }; + +export type MergeResult = { ok: true; commit: string } | { ok: false; reason: "conflict"; conflicts: string[] } | { ok: false; reason: "merging" | "no-remote" | "dirty"; message: string }; + +export interface ConflictFile { path: string; base: string | null; ours: string | null; theirs: string | null; working: string | null } + +export type SyncResult = + | { ok: true; pulled: boolean; pushed: boolean; head: string } + | { ok: false; reason: "no-remote" | "dirty" | "network"; message: string } + | { ok: false; reason: "conflict"; conflicts: string[] }; + +export class StudioError extends Error { + constructor(message: string, public status = 400) { super(message); } +} + +export interface StudioStoreOptions { dataDir: string; publishedBranch?: string } + +export function createStudioStore(opts: StudioStoreOptions) { + const dataDir = opts.dataDir; + const main = opts.publishedBranch ?? "main"; + const coursesDir = join(dataDir, "courses"); + const worktreesDir = join(dataDir, "worktrees"); + + const paths = (courseId: string) => { + if (!ID_RE.test(courseId)) throw new StudioError(`invalid course id: ${courseId}`, 404); + const root = join(worktreesDir, courseId); + return { + bare: join(coursesDir, courseId), + root, + published: join(root, "published"), + draftsDir: join(root, "drafts"), + draft: (authorId: string) => join(root, "drafts", safeAuthor(authorId)), + }; + }; + const branchOf = (authorId: string) => `draft/${safeAuthor(authorId)}`; + + const isManaged = (courseId: string): boolean => { + if (!ID_RE.test(courseId)) return false; + const p = paths(courseId); + return existsSync(join(p.bare, "HEAD")) && existsSync(join(p.published, ".git")); + }; + const requireManaged = (courseId: string) => { + if (!isManaged(courseId)) throw new StudioError(`course ${courseId} is not managed by the studio`, 404); + return paths(courseId); + }; + const lock = (courseId: string, fn: () => Promise) => withLock(paths(courseId).bare, fn); + + const rev = async (cwd: string, ref: string) => (await gitOk(cwd, ["rev-parse", ref])).trim(); + const count = async (cwd: string, range: string) => Number((await gitOk(cwd, ["rev-list", "--count", range])).trim()); + const unmerged = async (cwd: string) => [...new Set(splitZ(await gitOk(cwd, ["diff", "--name-only", "-z", "--diff-filter=U"])))]; + const isMerging = async (cwd: string) => (await git(cwd, ["rev-parse", "-q", "--verify", "MERGE_HEAD"])).code === 0; + const dirtyFiles = async (cwd: string) => + splitZ(await gitOk(cwd, ["status", "--porcelain", "-z", "--untracked-files=all"])).map((l) => l.slice(3)).filter(Boolean); + const remoteUrl = async (bare: string) => { + const r = await git(bare, ["remote", "get-url", "origin"]); + return r.code === 0 ? r.stdout.trim() : null; + }; + + /** Absolute path for `rel` inside `root`; also refuses when an existing parent is a symlink out of the root. */ + const inside = (root: string, rel: string): string => { + const n = checkRelPath(rel); + const abs = join(root, n); + const realRoot = realpathSync(root); + let probe = dirname(abs); + while (!existsSync(probe)) probe = dirname(probe); + const realProbe = realpathSync(probe); + if (realProbe !== realRoot && !realProbe.startsWith(realRoot + sep)) throw new Error(`invalid path (escapes the course): ${rel}`); + if (existsSync(abs)) { + const real = realpathSync(abs); + if (real !== realRoot && !real.startsWith(realRoot + sep)) throw new Error(`invalid path (escapes the course): ${rel}`); + } + return abs; + }; + + async function ensureDraftUnlocked(courseId: string, authorId: string): Promise { + const p = requireManaged(courseId); + const dir = p.draft(authorId); + if (existsSync(join(dir, ".git"))) return dir; + mkdirSync(p.draftsDir, { recursive: true }); + await git(p.bare, ["worktree", "prune"]); + const branch = branchOf(authorId); + const exists = (await git(p.bare, ["rev-parse", "-q", "--verify", `refs/heads/${branch}`])).code === 0; + await gitOk(p.bare, exists ? ["worktree", "add", dir, branch] : ["worktree", "add", "-b", branch, dir, main]); + return dir; + } + + async function readFileIn(dir: string, rel: string): Promise<{ content: string; hash: string } | null> { + let abs: string; + try { abs = inside(dir, rel); } catch { return null; } + if (!existsSync(abs) || !statSync(abs).isFile()) return null; + const bytes = new Uint8Array(await Bun.file(abs).arrayBuffer()); + return { content: new TextDecoder().decode(bytes), hash: contentHash(bytes) }; + } + + async function commitAll(dir: string, who: Identity, message: string, pathspecs?: string[]): Promise { + await gitOk(dir, ["add", "-A", "--", ...(pathspecs ?? ["."])]); + if ((await git(dir, ["diff", "--cached", "--quiet"])).code === 0) return null; + await gitOk(dir, ["commit", "-q", "-m", message], who); + return rev(dir, "HEAD"); + } + + async function writeFiles(dir: string, files: FileWrite[]): Promise { + const touched: string[] = []; + for (const f of files) { + const abs = inside(dir, f.path); + if (f.delete) { + if (existsSync(abs)) await unlink(abs); + } else { + await mkdir(dirname(abs), { recursive: true }); + await writeFile(abs, f.content ?? ""); + } + touched.push(checkRelPath(f.path)); + } + return touched; + } + + const api = { + dataDir, + publishedBranch: main, + paths, + branchOf, + isManaged, + + async listManaged(): Promise { + if (!existsSync(coursesDir)) return []; + return readdirSync(coursesDir).filter((id) => ID_RE.test(id) && isManaged(id)).sort(); + }, + + async createCourse(input: { id: string; title: string; pitch?: string }, who: Identity): Promise { + const { id, title } = input; + if (!ID_RE.test(id)) throw new StudioError(`invalid course id "${id}": use lowercase letters, digits and dashes`); + if (!title.trim()) throw new StudioError("title is required"); + const p = paths(id); + if (existsSync(p.bare) || existsSync(p.root)) throw new StudioError(`course "${id}" already exists`, 409); + await lock(id, async () => { + mkdirSync(coursesDir, { recursive: true }); + const tmp = join(dataDir, "tmp", `new-${id}-${Date.now()}`); + mkdirSync(join(tmp, "lessons"), { recursive: true }); + try { + await gitOk(dataDir, ["init", "-q", "--bare", "-b", main, p.bare]); + const manifest = { + schema: 1, id, title: title.trim(), ...(input.pitch ? { pitch: input.pitch } : {}), + authors: [who.name], visibility: "restricted", source: { kind: "authored" }, + modules: [{ id: "m1", title: "Getting started", lessons: [{ id: "welcome", title: "Welcome", type: "text", file: "lessons/welcome.md" }] }], + }; + await Bun.write(join(tmp, MANIFEST), Bun.YAML.stringify(manifest, null, 2) + "\n"); + await Bun.write(join(tmp, "lessons/welcome.md"), `# Welcome\n\nThis is the first lesson of ${title.trim()}.\n`); + const gd = ["--git-dir", p.bare, "--work-tree", tmp]; + await gitOk(tmp, [...gd, "add", "-A"]); + await gitOk(tmp, [...gd, "commit", "-q", "-m", `Create course ${id}`], who); + mkdirSync(p.root, { recursive: true }); + await gitOk(p.bare, ["worktree", "add", p.published, main]); + } catch (e) { + rmSync(p.bare, { recursive: true, force: true }); + rmSync(p.root, { recursive: true, force: true }); + throw e; + } finally { + await rm(tmp, { recursive: true, force: true }); + } + }); + }, + + ensureDraft: (courseId: string, authorId: string) => lock(courseId, () => ensureDraftUnlocked(courseId, authorId)), + + async listFiles(courseId: string, authorId: string): Promise { + const dir = await api.ensureDraft(courseId, authorId); + const tracked = splitZ(await gitOk(dir, ["ls-files", "-z", "--cached", "--others", "--exclude-standard"])); + return [...new Set(tracked)].filter((f) => existsSync(join(dir, f))).sort(); + }, + + async readFile(courseId: string, authorId: string, rel: string) { + const dir = await api.ensureDraft(courseId, authorId); + return readFileIn(dir, rel); + }, + + async readPublished(courseId: string, rel: string) { + return readFileIn(requireManaged(courseId).published, rel); + }, + + draftDir: (courseId: string, authorId: string) => api.ensureDraft(courseId, authorId), + + async saveFiles(courseId: string, who: Identity & { id: string }, files: FileWrite[], message: string): Promise { + for (const f of files) checkRelPath(f.path); + if (files.length === 0) throw new StudioError("nothing to save"); + return lock(courseId, async () => { + const dir = await ensureDraftUnlocked(courseId, who.id); + if (await isMerging(dir)) throw new StudioError("the draft has a merge in progress: resolve or abort it first", 409); + const conflicts: FileConflict[] = []; + for (const f of files) { + if (f.baseHash === undefined) continue; + const cur = await readFileIn(dir, f.path); + if ((cur?.hash ?? null) !== f.baseHash) conflicts.push({ path: checkRelPath(f.path), current: cur?.content ?? null, currentHash: cur?.hash ?? null }); + } + if (conflicts.length) return { ok: false, conflicts }; + const touched = await writeFiles(dir, files); + const commit = await commitAll(dir, who, message.trim() || "Edit in studio", touched); + return { ok: true, commit }; + }); + }, + + async status(courseId: string, authorId: string): Promise { + const p = requireManaged(courseId); + const dir = await api.ensureDraft(courseId, authorId); + const branch = branchOf(authorId); + const merging = await isMerging(dir); + return { + courseId, branch, publishedBranch: main, + draftHead: await rev(dir, "HEAD"), + publishedHead: await rev(p.bare, main), + ahead: await count(p.bare, `${main}..${branch}`), + behind: await count(p.bare, `${branch}..${main}`), + merging, + conflicts: merging ? await unmerged(dir) : [], + remote: await remoteUrl(p.bare), + publishedDirty: await dirtyFiles(p.published), + }; + }, + + async publishedHead(courseId: string): Promise { + return rev(requireManaged(courseId).published, "HEAD"); + }, + + /** Validates the draft as the learner loader would; error diagnostics block publishing. */ + async validateDraft(courseId: string, authorId: string): Promise { + return api.validateDraftUnlocked(courseId, await api.ensureDraft(courseId, authorId)); + }, + + publish(courseId: string, who: Identity & { id: string }): Promise { + return lock(courseId, async () => { + const p = requireManaged(courseId); + const dir = await ensureDraftUnlocked(courseId, who.id); + const branch = branchOf(who.id); + if (await isMerging(dir)) return { ok: false, reason: "merging" }; + const dirty = await dirtyFiles(p.published); + if (dirty.length) return { ok: false, reason: "dirty", files: dirty }; + const diags = (await api.validateDraftUnlocked(courseId, dir)).filter((d) => d.level === "error"); + if (diags.length) return { ok: false, reason: "invalid", diagnostics: diags }; + if ((await count(p.bare, `${main}..${branch}`)) === 0) return { ok: true, commit: await rev(p.published, "HEAD"), published: false }; + const r = await git(p.published, ["merge", "--no-ff", "-m", `Publish ${branch}`, branch], who); + if (r.code !== 0) { + const conflicts = await unmerged(p.published); + await git(p.published, ["merge", "--abort"]); + if (!conflicts.length) throw new StudioError(`publish failed: ${r.stderr.trim() || r.stdout.trim()}`, 500); + return { ok: false, reason: "conflict", conflicts }; + } + // Bring the draft level with what was just published, so "behind" stays meaningful. + await git(dir, ["merge", "-q", "--ff-only", main]); + return { ok: true, commit: await rev(p.published, "HEAD"), published: true }; + }); + }, + + async validateDraftUnlocked(courseId: string, dir: string): Promise { + const c = await loadCourse(join(dir, MANIFEST)); + const diags = [...c.diagnostics]; + if (c.manifest && c.manifest.id !== courseId) diags.push({ level: "error", courseId, path: MANIFEST, message: `manifest id "${c.manifest.id}" must stay "${courseId}"` }); + if (c.manifest?.root) diags.push({ level: "error", courseId, path: MANIFEST, message: "studio courses cannot set root" }); + return diags; + }, + + /** Merges the published branch (or the remote's) into the author's draft. Conflicts stay in the draft for the author to resolve. */ + updateDraft(courseId: string, who: Identity & { id: string }, from: "published" | "remote" = "published"): Promise { + return lock(courseId, async () => { + const p = requireManaged(courseId); + const dir = await ensureDraftUnlocked(courseId, who.id); + if (await isMerging(dir)) return { ok: false, reason: "merging", message: "a merge is already in progress" }; + let target = main; + if (from === "remote") { + if (!(await remoteUrl(p.bare))) return { ok: false, reason: "no-remote", message: "no remote is configured" }; + const f = await git(p.bare, ["fetch", "-q", "origin"]); + if (f.code !== 0) return { ok: false, reason: "no-remote", message: f.stderr.trim() }; + target = `origin/${main}`; + if ((await git(p.bare, ["rev-parse", "-q", "--verify", `refs/remotes/${target}`])).code !== 0) return { ok: true, commit: await rev(dir, "HEAD") }; + } + const r = await git(dir, ["merge", "--no-ff", "-m", `Update draft from ${target}`, target], who); + if (r.code !== 0) { + const conflicts = await unmerged(dir); + if (!conflicts.length) { + await git(dir, ["merge", "--abort"]); + throw new StudioError(`update failed: ${r.stderr.trim() || r.stdout.trim()}`, 500); + } + return { ok: false, reason: "conflict", conflicts }; + } + return { ok: true, commit: await rev(dir, "HEAD") }; + }); + }, + + async conflicts(courseId: string, authorId: string): Promise { + const dir = await api.ensureDraft(courseId, authorId); + if (!(await isMerging(dir))) return []; + const stage = async (n: number, path: string) => { + const r = await git(dir, ["show", `:${n}:${path}`]); + return r.code === 0 ? r.stdout : null; + }; + const out: ConflictFile[] = []; + for (const path of await unmerged(dir)) { + out.push({ path, base: await stage(1, path), ours: await stage(2, path), theirs: await stage(3, path), working: (await readFileIn(dir, path))?.content ?? null }); + } + return out; + }, + + /** Writes the author's resolution (`content: null` deletes) and commits the merge once nothing is unmerged. */ + resolve(courseId: string, who: Identity & { id: string }, files: { path: string; content: string | null }[]): Promise<{ ok: true; remaining: string[]; commit: string | null }> { + return lock(courseId, async () => { + const dir = await ensureDraftUnlocked(courseId, who.id); + if (!(await isMerging(dir))) throw new StudioError("no merge in progress", 409); + const touched = await writeFiles(dir, files.map((f) => (f.content === null ? { path: f.path, delete: true } : { path: f.path, content: f.content }))); + await gitOk(dir, ["add", "-A", "--", ...touched]); + const remaining = await unmerged(dir); + if (remaining.length) return { ok: true, remaining, commit: null }; + await gitOk(dir, ["commit", "-q", "--no-edit"], who); + return { ok: true, remaining: [], commit: await rev(dir, "HEAD") }; + }); + }, + + abortMerge(courseId: string, authorId: string): Promise { + return lock(courseId, async () => { + const dir = await ensureDraftUnlocked(courseId, authorId); + if (await isMerging(dir)) await gitOk(dir, ["merge", "--abort"]); + }); + }, + + async remote(courseId: string): Promise { + return remoteUrl(requireManaged(courseId).bare); + }, + + setRemote(courseId: string, url: string | null): Promise { + return lock(courseId, async () => { + const p = requireManaged(courseId); + const cur = await remoteUrl(p.bare); + if (!url) { + if (cur) await gitOk(p.bare, ["remote", "remove", "origin"]); + return; + } + if (/^\s*-/.test(url) || /\s/.test(url.trim())) throw new StudioError("invalid remote url"); + await gitOk(p.bare, cur ? ["remote", "set-url", "origin", url.trim()] : ["remote", "add", "origin", url.trim()]); + // Bare repos created with `init --bare` have no fetch refspec for remote-tracking branches by default. + await gitOk(p.bare, ["config", "remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*"]); + }); + }, + + /** + * Explicit Sync with the remote: pull (fast-forward, or a merge without conflicts) and push the published branch. + * A conflicting divergence is aborted and reported; the author merges the remote into the draft to resolve it. + */ + sync(courseId: string, who: Identity): Promise { + return lock(courseId, async () => { + const p = requireManaged(courseId); + if (!(await remoteUrl(p.bare))) return { ok: false, reason: "no-remote", message: "no remote is configured for this course" }; + const dirty = await dirtyFiles(p.published); + if (dirty.length) return { ok: false, reason: "dirty", message: `uncommitted changes in the published worktree: ${dirty.join(", ")}` }; + const f = await git(p.bare, ["fetch", "-q", "origin"]); + if (f.code !== 0) return { ok: false, reason: "network", message: f.stderr.trim() || "fetch failed" }; + const remoteRef = `refs/remotes/origin/${main}`; + const hasRemote = (await git(p.bare, ["rev-parse", "-q", "--verify", remoteRef])).code === 0; + let pulled = false; + let pushed = false; + if (hasRemote) { + const local = await rev(p.bare, main); + const remote = await rev(p.bare, remoteRef); + if (local !== remote) { + const localBehind = (await git(p.bare, ["merge-base", "--is-ancestor", local, remote])).code === 0; + const remoteBehind = (await git(p.bare, ["merge-base", "--is-ancestor", remote, local])).code === 0; + if (localBehind) { + await gitOk(p.published, ["merge", "-q", "--ff-only", remoteRef]); + pulled = true; + } else if (!remoteBehind) { + const r = await git(p.published, ["merge", "--no-ff", "-m", `Sync with origin/${main}`, remoteRef], who); + if (r.code !== 0) { + const conflicts = await unmerged(p.published); + await git(p.published, ["merge", "--abort"]); + return conflicts.length ? { ok: false, reason: "conflict", conflicts } : { ok: false, reason: "network", message: r.stderr.trim() }; + } + pulled = true; + } + } + } + const ahead = hasRemote ? await count(p.bare, `${remoteRef}..${main}`) : 1; + if (ahead > 0) { + const r = await git(p.bare, ["push", "-q", "origin", `${main}:${main}`]); + if (r.code !== 0) return { ok: false, reason: "network", message: r.stderr.trim() || "push failed" }; + await git(p.bare, ["fetch", "-q", "origin"]); + pushed = true; + } + return { ok: true, pulled, pushed, head: await rev(p.published, "HEAD") }; + }); + }, + }; + return api; +} + +export type StudioStore = ReturnType; diff --git a/packages/feature-studio/test/helpers.ts b/packages/feature-studio/test/helpers.ts new file mode 100644 index 0000000..c4516e8 --- /dev/null +++ b/packages/feature-studio/test/helpers.ts @@ -0,0 +1,25 @@ +import { mkdtempSync, realpathSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { $ } from "bun"; +import { createStudioStore, type Identity } from "../src/server/store"; + +export const ada: Identity & { id: string } = { id: "u-ada", name: "Ada Lovelace", email: "ada@example.org" }; +export const bob: Identity & { id: string } = { id: "u-bob", name: "Bob Ext", email: "bob@example.org" }; + +export function tempDir(prefix = "stc-studio-"): string { + return realpathSync(mkdtempSync(join(tmpdir(), prefix))); +} + +export async function newStore() { + const dataDir = tempDir(); + const store = createStudioStore({ dataDir }); + return { dataDir, store }; +} + +/** Runs git as an outside tool (VS Code, a terminal) would: plain identity, in the given dir. */ +export async function extGit(cwd: string, ...args: string[]): Promise { + const r = await $`git -c user.name=${bob.name} -c user.email=${bob.email} -c commit.gpgsign=false -c core.hooksPath=/dev/null ${args}`.cwd(cwd).quiet().nothrow(); + if (r.exitCode !== 0) throw new Error(`git ${args.join(" ")} failed: ${r.stderr.toString()}`); + return r.stdout.toString().trim(); +} diff --git a/packages/feature-studio/test/store.test.ts b/packages/feature-studio/test/store.test.ts new file mode 100644 index 0000000..6377f42 --- /dev/null +++ b/packages/feature-studio/test/store.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, test } from "bun:test"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { loadCourse } from "@stc/course-format"; +import { contentHash } from "../src/server/store"; +import { ada, bob, extGit, newStore } from "./helpers"; + +describe("createCourse", () => { + test("makes a bare repo, a published worktree on main and a valid manifest", async () => { + const { dataDir, store } = await newStore(); + await store.createCourse({ id: "intro-git", title: "Intro: Git & you" }, ada); + expect(existsSync(join(dataDir, "courses/intro-git/HEAD"))).toBe(true); + expect(existsSync(join(dataDir, "courses/intro-git/stcourse.yaml"))).toBe(false); + const pub = store.paths("intro-git").published; + expect(pub).toBe(join(dataDir, "worktrees/intro-git/published")); + expect(await extGit(pub, "rev-parse", "--abbrev-ref", "HEAD")).toBe("main"); + const loaded = await loadCourse(join(pub, "stcourse.yaml")); + expect(loaded.status).toBe("ok"); + expect(loaded.manifest?.title).toBe("Intro: Git & you"); + expect(await extGit(pub, "log", "-1", "--format=%an <%ae>")).toBe("Ada Lovelace "); + expect(store.isManaged("intro-git")).toBe(true); + expect(await store.listManaged()).toEqual(["intro-git"]); + }); + + test("rejects bad and duplicate ids", async () => { + const { store } = await newStore(); + await expect(store.createCourse({ id: "Bad Id", title: "x" }, ada)).rejects.toThrow(/id/); + await store.createCourse({ id: "c1", title: "x" }, ada); + await expect(store.createCourse({ id: "c1", title: "y" }, ada)).rejects.toThrow(/exists/); + }); +}); + +describe("drafts and saving", () => { + test("a draft worktree per author on draft/; save commits as the author; published untouched", async () => { + const { store } = await newStore(); + await store.createCourse({ id: "c1", title: "C" }, ada); + const dir = await store.ensureDraft("c1", ada.id); + expect(dir).toBe(store.paths("c1").draft(ada.id)); + expect(await extGit(dir, "rev-parse", "--abbrev-ref", "HEAD")).toBe("draft/u-ada"); + const cur = await store.readFile("c1", ada.id, "lessons/welcome.md"); + expect(cur).not.toBeNull(); + const r = await store.saveFiles("c1", ada, [{ path: "lessons/welcome.md", content: "# Hello draft\n", baseHash: cur!.hash }], "Edit welcome"); + expect(r.ok).toBe(true); + expect(await extGit(dir, "log", "-1", "--format=%an <%ae>|%s")).toBe("Ada Lovelace |Edit welcome"); + expect(readFileSync(join(store.paths("c1").published, "lessons/welcome.md"), "utf8")).not.toContain("Hello draft"); + const st = await store.status("c1", ada.id); + expect(st.ahead).toBe(1); + expect(st.behind).toBe(0); + expect(st.merging).toBe(false); + }); + + test("saving unchanged content makes no commit", async () => { + const { store } = await newStore(); + await store.createCourse({ id: "c1", title: "C" }, ada); + const cur = (await store.readFile("c1", ada.id, "lessons/welcome.md"))!; + const r = await store.saveFiles("c1", ada, [{ path: "lessons/welcome.md", content: cur.content, baseHash: cur.hash }], "noop"); + expect(r).toEqual({ ok: true, commit: null }); + }); + + test("an external edit between load and save is a conflict, not an overwrite", async () => { + const { store } = await newStore(); + await store.createCourse({ id: "c1", title: "C" }, ada); + const cur = (await store.readFile("c1", ada.id, "lessons/welcome.md"))!; + const file = join(store.paths("c1").draft(ada.id), "lessons/welcome.md"); + writeFileSync(file, "edited in VS Code\n"); + const r = await store.saveFiles("c1", ada, [{ path: "lessons/welcome.md", content: "studio text\n", baseHash: cur.hash }], "x"); + expect(r.ok).toBe(false); + if (r.ok) return; + expect(r.conflicts).toEqual([{ path: "lessons/welcome.md", current: "edited in VS Code\n", currentHash: contentHash("edited in VS Code\n") }]); + expect(readFileSync(file, "utf8")).toBe("edited in VS Code\n"); + }); + + test("baseHash null means the file must not exist yet", async () => { + const { store } = await newStore(); + await store.createCourse({ id: "c1", title: "C" }, ada); + const r = await store.saveFiles("c1", ada, [{ path: "lessons/welcome.md", content: "x", baseHash: null }], "x"); + expect(r.ok).toBe(false); + }); + + test("paths with spaces and non-ASCII characters", async () => { + const { store } = await newStore(); + await store.createCourse({ id: "c1", title: "C" }, ada); + const p = "S08 - Containerizing/Æble notes ü.md"; + const r = await store.saveFiles("c1", ada, [{ path: p, content: "æøå\n", baseHash: null }], "Add notes"); + expect(r.ok).toBe(true); + expect((await store.readFile("c1", ada.id, p))?.content).toBe("æøå\n"); + expect(await store.listFiles("c1", ada.id)).toContain(p); + }); + + test("binary content and deletion", async () => { + const { store } = await newStore(); + await store.createCourse({ id: "c1", title: "C" }, ada); + const bytes = new Uint8Array([0, 1, 2, 255]); + expect((await store.saveFiles("c1", ada, [{ path: "media/a.bin", content: bytes }], "bin")).ok).toBe(true); + expect(new Uint8Array(readFileSync(join(store.paths("c1").draft(ada.id), "media/a.bin")))).toEqual(bytes); + expect((await store.saveFiles("c1", ada, [{ path: "media/a.bin", delete: true }], "rm")).ok).toBe(true); + expect(await store.listFiles("c1", ada.id)).not.toContain("media/a.bin"); + }); + + test("path traversal and .git are rejected", async () => { + const { store } = await newStore(); + await store.createCourse({ id: "c1", title: "C" }, ada); + for (const bad of ["../x.md", "/etc/passwd", ".git/config", "a/../../x", "", "a\0b"]) { + await expect(store.saveFiles("c1", ada, [{ path: bad, content: "x" }], "x")).rejects.toThrow(/path/i); + } + expect(await store.readFile("c1", ada.id, "../../../etc/passwd")).toBeNull(); + }); + + test("authors get separate drafts; odd author ids are made safe", async () => { + const { store } = await newStore(); + await store.createCourse({ id: "c1", title: "C" }, ada); + const a = await store.ensureDraft("c1", ada.id); + const b = await store.ensureDraft("c1", "bob/../x y"); + expect(a).not.toBe(b); + expect(await extGit(b, "rev-parse", "--abbrev-ref", "HEAD")).toMatch(/^draft\/[A-Za-z0-9_-]+$/); + void bob; + }); +}); From 07e2223c8a80c63dd9cee1e15579c838a16ae677 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 03:15:09 +0100 Subject: [PATCH 08/62] feat(import): Packt parser and packt-video job Co-Authored-By: Claude Opus 5.5 --- packages/feature-import/src/context.ts | 102 ++++++++++++++++++ packages/feature-import/src/jobs/common.ts | 48 +++++++++ .../feature-import/src/jobs/packt-video.ts | 86 +++++++++++++++ packages/feature-import/src/packt.ts | 74 +++++++++++++ packages/feature-import/src/util/source.ts | 55 ++++++++++ packages/feature-import/test/harness.ts | 21 ++++ .../feature-import/test/packt-video.test.ts | 59 ++++++++++ packages/feature-import/test/packt.test.ts | 28 +++++ 8 files changed, 473 insertions(+) create mode 100644 packages/feature-import/src/context.ts create mode 100644 packages/feature-import/src/jobs/common.ts create mode 100644 packages/feature-import/src/jobs/packt-video.ts create mode 100644 packages/feature-import/src/packt.ts create mode 100644 packages/feature-import/src/util/source.ts create mode 100644 packages/feature-import/test/harness.ts create mode 100644 packages/feature-import/test/packt-video.test.ts create mode 100644 packages/feature-import/test/packt.test.ts diff --git a/packages/feature-import/src/context.ts b/packages/feature-import/src/context.ts new file mode 100644 index 0000000..caeb3d2 --- /dev/null +++ b/packages/feature-import/src/context.ts @@ -0,0 +1,102 @@ +import { mkdir, readdir, rename, stat } from "node:fs/promises"; +import { join, relative, resolve } from "node:path"; +import type { ServerContext } from "@stc/contracts"; +import { readRawManifest, writeManifest, type RawManifest } from "./util/manifest"; +import { slugify, uniqueId } from "./util/text"; +import { extractFrame, type Tools } from "./media"; + +export interface ImportEnv { [k: string]: string | undefined } +export interface ImportOptions { env?: ImportEnv } + +export interface Deps { + ctx: ServerContext; + env: ImportEnv; + tools: Tools; + python: string; +} + +export const coursesDir = (d: Deps) => resolve(d.ctx.config.dataDir, "courses"); +export const cacheDir = (d: Deps) => resolve(d.ctx.config.dataDir, "cache"); +export const deckCacheDir = (d: Deps, courseId: string, deckId: string) => join(cacheDir(d), "decks", courseId, deckId); +export const coverCacheDir = (d: Deps, courseId: string) => join(cacheDir(d), "covers", courseId); +export const transcriptCacheDir = (d: Deps, courseId: string) => join(cacheDir(d), "transcripts", courseId); + +export const isPlainId = (s: string) => /^[a-z0-9][a-z0-9-]*$/.test(s); + +/** A fresh course id from a title, unique among managed folders and every course the server already knows. */ +export async function freshCourseId(d: Deps, wanted: string | undefined, title: string): Promise { + if (wanted !== undefined) { + if (!isPlainId(wanted)) throw new Error(`invalid courseId "${wanted}" (use lowercase letters, digits and dashes)`); + if (await exists(join(coursesDir(d), wanted)) || d.ctx.courses.get(wanted)) throw new Error(`a course with id "${wanted}" already exists`); + return wanted; + } + const taken = new Set(d.ctx.courses.list().map((c) => c.id)); + for (const e of await readdir(coursesDir(d)).catch(() => [] as string[])) taken.add(e); + return uniqueId(slugify(title), (x) => taken.has(x)); +} + +export async function exists(p: string): Promise { + return stat(p).then(() => true, () => false); +} + +/** Scratch folder on the same filesystem as the courses, so finished work is moved in with one rename. */ +export async function stageDir(d: Deps, label: string): Promise { + const dir = resolve(d.ctx.config.dataDir, "tmp", `${label}-${Bun.randomUUIDv7()}`); + await mkdir(dir, { recursive: true }); + return dir; +} + +export async function publishCourse(d: Deps, stage: string, id: string): Promise { + await mkdir(coursesDir(d), { recursive: true }); + const dest = join(coursesDir(d), id); + await rename(stage, dest); + return dest; +} + +export interface ManagedCourse { id: string; dir: string; manifest: RawManifest } + +/** An existing course the importer may add to. Only folders directly under /courses qualify: overlays and authored repos are never edited. */ +export async function managedCourse(d: Deps, courseId: string | undefined): Promise { + if (!courseId) throw new Error("options.courseId is required: choose the course to add to"); + const dir = join(coursesDir(d), courseId); + const rel = relative(coursesDir(d), dir); + if (!isPlainId(courseId) || rel.startsWith("..") || !(await exists(join(dir, "stcourse.yaml")))) { + throw new Error(`course "${courseId}" is not an imported course under ${coursesDir(d)}; the importer only edits courses it created`); + } + const manifest = await readRawManifest(dir); + if (manifest.id !== courseId) throw new Error(`course folder "${courseId}" has manifest id "${manifest.id}"`); + return { id: courseId, dir, manifest }; +} + +/** Reloads the repository and turns load problems into warnings. */ +export async function finish(d: Deps, id: string, warnings: string[]): Promise { + await d.ctx.courses.reload(id); + const c = d.ctx.courses.get(id); + if (!c) { + warnings.push(`course "${id}" is not visible to the server: add ${coursesDir(d)} to STC_COURSE_DIRS`); + return; + } + for (const diag of c.diagnostics) warnings.push(`${diag.level}: ${diag.message}${diag.path ? ` (${diag.path})` : ""}`); +} + +export async function saveManifest(dir: string, manifest: RawManifest): Promise { + await writeManifest(dir, manifest); +} + +export function sourceBlock(kind: string, origin: string) { + return { kind, origin, importedAt: new Date().toISOString() }; +} + +/** Writes up to `count` frame candidates from evenly spaced clips; returns the candidate file names. Failures are skipped. */ +export async function frameCandidates(d: Deps, courseId: string, clips: string[], count = 3): Promise { + const out: string[] = []; + const n = Math.min(count, clips.length); + for (let i = 0; i < n; i++) { + const name = `frame-${i + 1}.jpg`; + try { + await extractFrame(d.tools, clips[Math.floor((i * clips.length) / n)]!, join(coverCacheDir(d, courseId), name), 0.2); + out.push(name); + } catch { /* a bad clip must not fail the import */ } + } + return out; +} diff --git a/packages/feature-import/src/jobs/common.ts b/packages/feature-import/src/jobs/common.ts new file mode 100644 index 0000000..f8158fc --- /dev/null +++ b/packages/feature-import/src/jobs/common.ts @@ -0,0 +1,48 @@ +import type { ImportResult, ImportSource, JobContext } from "@stc/contracts"; +import { copyFile, mkdir, rm } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { coverCacheDir, exists, type Deps } from "../context"; + +export type Opts = Record; + +export function parsePayload(payload: unknown, kind: ImportSource["kind"]): { path: string; options: Opts } { + const p = payload as Partial | null; + if (!p || typeof p.path !== "string" || p.path === "") throw new Error("payload.path is required"); + if (p.kind !== undefined && p.kind !== kind) throw new Error(`payload.kind "${p.kind}" does not match job kind "${kind}"`); + return { path: p.path, options: (p.options ?? {}) as Opts }; +} + +export const optStr = (o: Opts, k: string): string | undefined => (typeof o[k] === "string" && (o[k] as string).trim() !== "" ? (o[k] as string).trim() : undefined); +export const optBool = (o: Opts, k: string): boolean => o[k] === true || o[k] === "true"; + +/** Runs a job body with a scratch folder that is always removed, and a result shape shared by all import jobs. */ +export async function withStage(stage: string, fn: () => Promise): Promise { + try { + return await fn(); + } finally { + await rm(stage, { recursive: true, force: true }); + } +} + +/** Copies the first cover candidate into the course (relative path returned) so the library has a cover straight away. */ +export async function adoptCover(d: Deps, courseId: string, courseDir: string, candidate: string | undefined): Promise { + if (!candidate) return undefined; + const src = join(coverCacheDir(d, courseId), candidate); + if (!(await exists(src))) return undefined; + const rel = "media/cover.jpg"; + await mkdir(dirname(join(courseDir, rel)), { recursive: true }); + await copyFile(src, join(courseDir, rel)); + return rel; +} + +export async function maybeQueueTranscription(d: Deps, courseId: string, wanted: boolean, warnings: string[]): Promise { + if (!wanted) return; + if (!d.env.STC_TRANSCRIBE_URL) { + warnings.push("transcription requested but STC_TRANSCRIBE_URL is not set; skipped"); + return; + } + await d.ctx.jobs.enqueue("import.transcribe", { courseId }); +} + +export type Reporter = JobContext["reportProgress"]; +export type { ImportResult }; diff --git a/packages/feature-import/src/jobs/packt-video.ts b/packages/feature-import/src/jobs/packt-video.ts new file mode 100644 index 0000000..cca74f1 --- /dev/null +++ b/packages/feature-import/src/jobs/packt-video.ts @@ -0,0 +1,86 @@ +import { mkdir, rename } from "node:fs/promises"; +import { basename, dirname, join, posix } from "node:path"; +import type { ImportResult, JobContext } from "@stc/contracts"; +import { finish, frameCandidates, freshCourseId, publishCourse, saveManifest, sourceBlock, stageDir, type Deps } from "../context"; +import { probeDuration } from "../media"; +import { parsePacktIndex } from "../packt"; +import { openSource } from "../util/source"; +import { adoptCover, maybeQueueTranscription, optBool, optStr, parsePayload, withStage } from "./common"; + +export async function importPacktVideo(d: Deps, payload: unknown, jc: JobContext): Promise { + const { path, options } = parsePayload(payload, "packt-video"); + const warnings: string[] = []; + await jc.reportProgress(0.02, "Reading the package"); + const src = await openSource(path); + + // The player page is the shallowest index.html that lists sections. + const candidates = src.entries.filter((e) => /(^|\/)index\.html$/i.test(e)).sort((a, b) => a.split("/").length - b.split("/").length); + let indexEntry: string | undefined; + let index: ReturnType | undefined; + for (const c of candidates) { + try { index = parsePacktIndex(await src.readText(c)); indexEntry = c; break; } catch { /* try the next */ } + } + if (!index || indexEntry === undefined) throw new Error("no Packt player index.html (with \"Section N: …\" headings) found in the package"); + warnings.push(...index.warnings); + const base = dirname(indexEntry) === "." ? "" : dirname(indexEntry); + const at = (rel: string) => posix.normalize(base ? `${base}/${rel}` : rel); + + const title = optStr(options, "title") ?? index.title; + const id = await freshCourseId(d, optStr(options, "courseId"), title); + const stage = await stageDir(d, "packt"); + + return withStage(stage, async () => { + const known = new Set(src.entries); + const modules: any[] = []; + const used = new Set(); + const clipFiles: string[] = []; + const total = index.sections.reduce((n, s) => n + s.clips.length, 0) || 1; + let done = 0; + for (const sec of index.sections) { + const mid = `m${sec.number}`; + const entries: { clip: (typeof sec.clips)[number]; entry: string }[] = []; + for (const clip of sec.clips) { + const entry = at(clip.video); + if (!known.has(entry)) { warnings.push(`video for ${sec.number}.${clip.number} is missing from the package: ${clip.video}`); continue; } + entries.push({ clip, entry }); + used.add(entry); + } + await jc.reportProgress(0.05 + 0.65 * (done / total), `Copying section ${sec.number}: ${sec.title}`); + const tmp = join(stage, "x"); + await src.extract(entries.map((e) => e.entry), tmp, jc.signal); + const lessons: any[] = []; + for (const { clip, entry } of entries) { + const rel = `media/${basename(entry)}`; + await mkdir(join(stage, "media"), { recursive: true }); + await rename(join(tmp, entry), join(stage, rel)); + clipFiles.push(join(stage, rel)); + const duration = await probeDuration(d.tools, join(stage, rel)); + lessons.push({ id: `${mid}-l${clip.number}`, title: clip.title, type: "video", file: rel, ...(duration ? { duration } : {}) }); + done++; + } + modules.push({ id: mid, title: sec.title.replace(/^Section\s+\d+\s*[:.-]\s*/i, ""), lessons }); + } + const unused = src.entries.filter((e) => /\.(mp4|webm|m4v|mov)$/i.test(e) && !used.has(e)); + if (unused.length) warnings.push(`${unused.length} video file(s) in the package are not listed in index.html and were left out`); + + await jc.reportProgress(0.75, "Making cover candidates"); + const covers = await frameCandidates(d, id, clipFiles); + if (covers.length === 0 && clipFiles.length) warnings.push("could not extract cover frames (is ffmpeg available?)"); + const cover = await adoptCover(d, id, stage, covers[0]); + + const manifest = { + schema: 1, id, title, + visibility: "restricted", + ...(cover ? { cover } : {}), + source: sourceBlock("packt-video", basename(path)), + modules, + }; + await saveManifest(stage, manifest); + await jc.reportProgress(0.9, "Publishing the course"); + const dir = await publishCourse(d, stage, id); + await finish(d, id, warnings); + await maybeQueueTranscription(d, id, optBool(options, "transcribe"), warnings); + await jc.reportProgress(1, `Imported ${total} clips`); + return { courseId: id, manifestPath: join(dir, "stcourse.yaml"), warnings }; + }); +} diff --git a/packages/feature-import/src/packt.ts b/packages/feature-import/src/packt.ts new file mode 100644 index 0000000..07073d2 --- /dev/null +++ b/packages/feature-import/src/packt.ts @@ -0,0 +1,74 @@ +import { decodeEntities, stripTags } from "./util/text"; + +export interface PacktClip { number: number; title: string; video: string; } +export interface PacktSection { number: number; title: string; clips: PacktClip[] } +export interface PacktIndex { title: string; sections: PacktSection[]; warnings: string[] } + +/** + * Parses the Packt player's index.html structurally: + * - the section overview: `

    Section N: …

    ` followed by a `
      ` of clip titles, in order, usually WITHOUT "Video N.M:" prefixes; + * - the per-section chapter blocks (`class="chapter" id="content-N"`) with one `
    • ` per clip: `

      N.M title

      ` and `data-video="…"`. + * Chapter blocks are authoritative for the video file; the overview list fills in any title or whole section that is missing. + */ +export function parsePacktIndex(html: string): PacktIndex { + const warnings: string[] = []; + const intro = /]*class="intro"[\s\S]*?]*>([\s\S]*?)<\/h1>/i.exec(html); + const titleTag = /]*>([\s\S]*?)<\/title>/i.exec(html); + const title = stripTags(intro?.[1] ?? titleTag?.[1] ?? "") || "Imported course"; + + const chapterStart = html.search(/]*class="chapter"/i); + const overview = chapterStart >= 0 ? html.slice(0, chapterStart) : html; + const chaptersHtml = chapterStart >= 0 ? html.slice(chapterStart) : ""; + + // overview: sections in document order + const secRe = /]*>\s*Section\s+(\d+)\s*[:.-]\s*([\s\S]*?)<\/h1>/gi; + const heads = [...overview.matchAll(secRe)]; + if (heads.length === 0) throw new Error("not a Packt video package: no \"Section N: …\" headings found in index.html"); + const overviewTitles = new Map(); + const sectionTitles = new Map(); + heads.forEach((h, i) => { + const n = Number(h[1]); + sectionTitles.set(n, stripTags(h[2]!)); + const block = overview.slice(h.index! + h[0].length, heads[i + 1]?.index ?? overview.length); + const ul = /]*>([\s\S]*?)<\/ul>/i.exec(block); + const items = ul ? [...ul[1]!.matchAll(/]*>([\s\S]*?)<\/li>/gi)].map((m) => stripTags(m[1]!)).filter(Boolean) : []; + const firstP = /]*>\s*Video\s+\d+\.\d+\s*:\s*([\s\S]*?)<\/p>/i.exec(block); + if (items.length === 0 && firstP) items.push(stripTags(firstP[1]!)); + overviewTitles.set(n, items); + }); + + // chapters: clips with their files + const chapterClips = new Map(); + const chapRe = /]*class="chapter"[^>]*id="content-(\d+)"[^>]*>/gi; + const chaps = [...chaptersHtml.matchAll(chapRe)]; + chaps.forEach((c, i) => { + const sec = Number(c[1]); + const block = chaptersHtml.slice(c.index! + c[0].length, chaps[i + 1]?.index ?? chaptersHtml.length); + const clipRe = /]*>\s*(?:Video\s+)?(\d+)\.(\d+)\s*[:.]?\s*([\s\S]*?)<\/h1>/gi; + const hs = [...block.matchAll(clipRe)]; + const clips: PacktClip[] = []; + hs.forEach((h, j) => { + const part = block.slice(h.index!, hs[j + 1]?.index ?? block.length); + const vid = /data-video="([^"]+)"/i.exec(part)?.[1] ?? /href="([^"]+\.(?:mp4|webm|m4v|mov))"/i.exec(part)?.[1]; + const num = Number(h[2]); + if (!vid) { warnings.push(`clip ${sec}.${num} has no video file in index.html`); return; } + clips.push({ number: num, title: stripTags(h[3]!), video: decodeEntities(vid) }); + }); + chapterClips.set(sec, clips); + }); + + const sections: PacktSection[] = []; + for (const n of [...sectionTitles.keys()].sort((a, b) => a - b)) { + let clips = chapterClips.get(n); + const titles = overviewTitles.get(n) ?? []; + if (!clips || clips.length === 0) { + warnings.push(`section ${n} has no chapter block; clips were taken from the overview list and mapped to videos/video${n}_M.mp4 in order`); + clips = titles.map((t, i) => ({ number: i + 1, title: t, video: `videos/video${n}_${i + 1}.mp4` })); + } else { + clips = clips.map((c, i) => ({ ...c, title: c.title || titles[i] || `Video ${n}.${c.number}` })); + if (titles.length && titles.length !== clips.length) warnings.push(`section ${n}: overview lists ${titles.length} clips but the chapter has ${clips.length}`); + } + sections.push({ number: n, title: sectionTitles.get(n)!, clips }); + } + return { title, sections, warnings }; +} diff --git a/packages/feature-import/src/util/source.ts b/packages/feature-import/src/util/source.ts new file mode 100644 index 0000000..43bf664 --- /dev/null +++ b/packages/feature-import/src/util/source.ts @@ -0,0 +1,55 @@ +import { cp, mkdir, readdir, readFile, stat } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { run } from "./exec"; + +/** A read-only view of an input: a zip file or an already extracted folder. Nothing in here writes to the input. */ +export interface Source { + kind: "zip" | "dir"; + entries: string[]; // files only, "/"-separated, relative to the source root + readText(entry: string): Promise; + /** Copies the given entries to `dest`, keeping their relative paths. */ + extract(entries: string[], dest: string, signal?: AbortSignal): Promise; +} + +const globEscape = (s: string) => s.replace(/[\\[\]*?]/g, (c) => `\\${c}`); + +async function walk(root: string, rel = "", out: string[] = []): Promise { + for (const e of await readdir(join(root, rel), { withFileTypes: true })) { + const r = rel ? `${rel}/${e.name}` : e.name; + if (e.isDirectory()) await walk(root, r, out); + else if (e.isFile()) out.push(r); + } + return out; +} + +export async function openSource(path: string): Promise { + const st = await stat(path).catch(() => null); + if (!st) throw new Error(`source not found: ${path}`); + if (st.isDirectory()) { + return { + kind: "dir", + entries: await walk(path), + readText: (e) => readFile(join(path, e), "utf8"), + async extract(entries, dest) { + for (const e of entries) { + await mkdir(dirname(join(dest, e)), { recursive: true }); + await cp(join(path, e), join(dest, e)); + } + }, + }; + } + const listing = await run(["unzip", "-Z1", path]); + const entries = listing.split("\n").filter((l) => l && !l.endsWith("/")); + return { + kind: "zip", + entries, + readText: (e) => run(["unzip", "-p", path, globEscape(e)]), + async extract(list, dest, signal) { + await mkdir(dest, { recursive: true }); + // unzip treats arguments as patterns, so glob characters in names are escaped; batches keep argv small. + for (let i = 0; i < list.length; i += 200) { + await run(["unzip", "-q", "-o", path, ...list.slice(i, i + 200).map(globEscape), "-d", dest], { signal }); + } + }, + }; +} diff --git a/packages/feature-import/test/harness.ts b/packages/feature-import/test/harness.ts new file mode 100644 index 0000000..d39d528 --- /dev/null +++ b/packages/feature-import/test/harness.ts @@ -0,0 +1,21 @@ +import { join } from "node:path"; +import type { JobContext, ServerContext } from "@stc/contracts"; +import { createCourseRepository } from "@stc/course-format"; +import type { Deps } from "../src/context"; +import { PYTHON, tmp } from "./fixtures"; + +export interface TestEnv { deps: Deps; dataDir: string; queued: { kind: string; payload: unknown }[]; jc: JobContext; progress: [number, string | undefined][] } + +export async function makeEnv(env: Record = {}): Promise { + const dataDir = await tmp("stc-data-"); + const courses = await createCourseRepository([join(dataDir, "courses")]); + const queued: TestEnv["queued"] = []; + const progress: TestEnv["progress"] = []; + const ctx = { + config: { port: 0, databaseUrl: "sqlite://:memory:", dataDir, courseDirs: [join(dataDir, "courses")], secret: "t", publicUrl: "http://x" }, + courses, + jobs: { enqueue: async (kind: string, payload: unknown) => { queued.push({ kind, payload }); return "j1"; }, get: async () => null, list: async () => [] }, + } as unknown as ServerContext; + const jc: JobContext = { signal: new AbortController().signal, reportProgress: async (f, m) => { progress.push([f, m]); } }; + return { deps: { ctx, env, tools: { ffmpeg: "ffmpeg", ffprobe: "ffprobe" }, python: PYTHON }, dataDir, queued, jc, progress }; +} diff --git a/packages/feature-import/test/packt-video.test.ts b/packages/feature-import/test/packt-video.test.ts new file mode 100644 index 0000000..63b4c8f --- /dev/null +++ b/packages/feature-import/test/packt-video.test.ts @@ -0,0 +1,59 @@ +import { expect, test } from "bun:test"; +import { readdir, readFile, stat } from "node:fs/promises"; +import { join } from "node:path"; +import { importPacktVideo } from "../src/jobs/packt-video"; +import { makeEnv } from "./harness"; +import { makePacktZip, tmp } from "./fixtures"; + +test("imports a Packt zip into a restricted course with sections, titles and media", async () => { + const env = await makeEnv(); + const zip = await makePacktZip(await tmp()); + const before = await readFile(zip); + const r = await importPacktVideo(env.deps, { kind: "packt-video", path: zip, options: { title: "Docker Test" } }, env.jc); + expect(r.courseId).toBe("docker-test"); + expect(r.warnings).toEqual([]); + const c = env.deps.ctx.courses.get("docker-test")!; + expect(c.status).toBe("ok"); + expect(c.diagnostics).toEqual([]); + const m = c.manifest!; + expect(m.visibility).toBe("restricted"); + expect(m.source?.kind).toBe("packt-video"); + expect(m.source?.origin).toBe("video_zip.zip"); + expect(m.source?.importedAt).toBeTruthy(); + expect(m.modules.map((x) => x.title)).toEqual(["Intro to Docker", "Images (Sample)"]); + expect(m.modules.map((x) => x.lessons.map((l) => l.title))).toEqual([["Course Introduction", "Docker & Friends"], ["Topic Introduction - Images"]]); + expect(m.modules[0]!.lessons[0]).toMatchObject({ id: "m1-l1", type: "video", file: "media/video1_1.mp4", duration: 2 }); + expect(env.deps.ctx.courses.resolvePath("docker-test", "media/video2_1.mp4")).toBeTruthy(); + expect(m.cover).toBe("media/cover.jpg"); + expect((await readdir(join(env.dataDir, "cache/covers/docker-test"))).sort()).toEqual(["frame-1.jpg", "frame-2.jpg", "frame-3.jpg"]); + expect(await readdir(join(env.dataDir, "tmp"))).toEqual([]); // scratch removed + expect(Buffer.compare(await readFile(zip), before)).toBe(0); // original untouched + expect(env.progress.at(-1)![0]).toBe(1); +}); + +test("second import of the same title gets a fresh id; transcribe requested without a URL warns", async () => { + const env = await makeEnv(); + const zip = await makePacktZip(await tmp()); + await importPacktVideo(env.deps, { path: zip, options: { title: "Same" } }, env.jc); + const r = await importPacktVideo(env.deps, { path: zip, options: { title: "Same", transcribe: true } }, env.jc); + expect(r.courseId).toBe("same-2"); + expect(r.warnings.join()).toContain("STC_TRANSCRIBE_URL is not set"); + expect(env.queued).toEqual([]); +}); + +test("a zip that is not a Packt package fails and leaves nothing behind", async () => { + const env = await makeEnv(); + const d = await tmp(); + await Bun.write(join(d, "t", "readme.txt"), "x"); + const { zipDir } = await import("./fixtures"); + await zipDir(join(d, "t"), join(d, "bad.zip")); + await expect(importPacktVideo(env.deps, { path: join(d, "bad.zip") }, env.jc)).rejects.toThrow(/index\.html/); + expect(await stat(join(env.dataDir, "courses")).catch(() => null)).toBeNull(); +}); + +test("transcribe option queues the transcription job when a URL is configured", async () => { + const env = await makeEnv({ STC_TRANSCRIBE_URL: "http://127.0.0.1:1/v1/audio/transcriptions" }); + const zip = await makePacktZip(await tmp()); + const r = await importPacktVideo(env.deps, { path: zip, options: { transcribe: true } }, env.jc); + expect(env.queued).toEqual([{ kind: "import.transcribe", payload: { courseId: r.courseId } }]); +}); diff --git a/packages/feature-import/test/packt.test.ts b/packages/feature-import/test/packt.test.ts new file mode 100644 index 0000000..dbdf306 --- /dev/null +++ b/packages/feature-import/test/packt.test.ts @@ -0,0 +1,28 @@ +import { expect, test } from "bun:test"; +import { parsePacktIndex } from "../src/packt"; +import { packtIndexHtml } from "./fixtures"; + +test("sections, clip titles and video files come from the structure", () => { + const p = parsePacktIndex(packtIndexHtml()); + expect(p.title).toBe("Fake Docker Course"); + expect(p.sections.map((s) => s.title)).toEqual(["Intro to Docker", "Images (Sample)"]); + expect(p.sections[0]!.clips).toEqual([ + { number: 1, title: "Course Introduction", video: "videos/video1_1.mp4" }, + { number: 2, title: "Docker & Friends", video: "videos/video1_2.mp4" }, + ]); + expect(p.sections[1]!.clips[0]!.video).toBe("videos/video2_1.mp4"); + expect(p.warnings).toEqual([]); +}); + +test("a section without a chapter block falls back to the overview list", () => { + const p = parsePacktIndex(packtIndexHtml({ dropChapter: 1 })); + expect(p.sections[0]!.clips.map((c) => [c.title, c.video])).toEqual([ + ["Course Introduction", "videos/video1_1.mp4"], + ["Docker & Friends", "videos/video1_2.mp4"], + ]); + expect(p.warnings.join()).toContain("section 1 has no chapter block"); +}); + +test("not a Packt page is a clear error", () => { + expect(() => parsePacktIndex("

      Hello

      ")).toThrow(/no "Section N/); +}); From c2385b472ce476f5d2db76a97cda5b35c7cad7b4 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 03:15:20 +0100 Subject: [PATCH 09/62] feat(studio): publish as merge, update draft, conflict resolution, explicit sync Co-Authored-By: Claude Opus 5.5 --- packages/feature-studio/test/publish.test.ts | 146 +++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 packages/feature-studio/test/publish.test.ts diff --git a/packages/feature-studio/test/publish.test.ts b/packages/feature-studio/test/publish.test.ts new file mode 100644 index 0000000..cc5c33e --- /dev/null +++ b/packages/feature-studio/test/publish.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { ada, bob, extGit, newStore, tempDir } from "./helpers"; + +async function courseWithEdit(text = "# Draft text\n") { + const s = await newStore(); + await s.store.createCourse({ id: "c1", title: "C" }, ada); + const cur = (await s.store.readFile("c1", ada.id, "lessons/welcome.md"))!; + await s.store.saveFiles("c1", ada, [{ path: "lessons/welcome.md", content: text, baseHash: cur.hash }], "Draft edit"); + return { ...s, pub: s.store.paths("c1").published }; +} + +describe("publish", () => { + test("merges the draft into main; the draft is level afterwards", async () => { + const { store, pub } = await courseWithEdit(); + const r = await store.publish("c1", ada); + expect(r).toMatchObject({ ok: true, published: true }); + expect(readFileSync(join(pub, "lessons/welcome.md"), "utf8")).toBe("# Draft text\n"); + expect(await extGit(pub, "log", "-1", "--format=%an|%s")).toBe("Ada Lovelace|Publish draft/u-ada"); + const st = await store.status("c1", ada.id); + expect([st.ahead, st.behind]).toEqual([0, 0]); + expect(await store.publish("c1", ada)).toMatchObject({ ok: true, published: false }); + }); + + test("a conflicting external commit on main is reported and main keeps the external content", async () => { + const { store, pub } = await courseWithEdit("# Studio version\n"); + writeFileSync(join(pub, "lessons/welcome.md"), "# External version\n"); + await extGit(pub, "commit", "-qam", "External edit"); + const head = await extGit(pub, "rev-parse", "HEAD"); + const r = await store.publish("c1", ada); + expect(r).toEqual({ ok: false, reason: "conflict", conflicts: ["lessons/welcome.md"] }); + expect(readFileSync(join(pub, "lessons/welcome.md"), "utf8")).toBe("# External version\n"); + expect(await extGit(pub, "rev-parse", "HEAD")).toBe(head); + expect(await extGit(pub, "status", "--porcelain")).toBe(""); + expect((await store.status("c1", ada.id)).behind).toBe(1); + }); + + test("uncommitted edits in the published worktree block publishing", async () => { + const { store, pub } = await courseWithEdit(); + writeFileSync(join(pub, "lessons/welcome.md"), "half-typed in an editor"); + const r = await store.publish("c1", ada); + expect(r).toEqual({ ok: false, reason: "dirty", files: ["lessons/welcome.md"] }); + expect(readFileSync(join(pub, "lessons/welcome.md"), "utf8")).toBe("half-typed in an editor"); + }); + + test("an invalid manifest blocks publishing with diagnostics", async () => { + const { store } = await courseWithEdit(); + const m = (await store.readFile("c1", ada.id, "stcourse.yaml"))!; + await store.saveFiles("c1", ada, [{ path: "stcourse.yaml", content: m.content.replace("lessons/welcome.md", "lessons/missing.md"), baseHash: m.hash }], "break"); + const r = await store.publish("c1", ada); + expect(r.ok).toBe(false); + if (r.ok || r.reason !== "invalid") throw new Error("expected invalid"); + expect(r.diagnostics[0]!.message).toContain("lessons/missing.md"); + }); +}); + +describe("update draft and conflict resolution", () => { + test("conflicts are shown with both sides, resolved by the author, then publish succeeds", async () => { + const { store, pub } = await courseWithEdit("# Studio version\n"); + writeFileSync(join(pub, "lessons/welcome.md"), "# External version\n"); + await extGit(pub, "commit", "-qam", "External edit"); + const u = await store.updateDraft("c1", ada); + expect(u).toEqual({ ok: false, reason: "conflict", conflicts: ["lessons/welcome.md"] }); + const st = await store.status("c1", ada.id); + expect(st.merging).toBe(true); + expect(st.conflicts).toEqual(["lessons/welcome.md"]); + await expect(store.saveFiles("c1", ada, [{ path: "x.md", content: "x" }], "x")).rejects.toThrow(/merge/); + const [c] = await store.conflicts("c1", ada.id); + expect(c).toMatchObject({ path: "lessons/welcome.md", ours: "# Studio version\n", theirs: "# External version\n" }); + expect(c!.working).toContain("<<<<<<<"); + const res = await store.resolve("c1", ada, [{ path: "lessons/welcome.md", content: "# Both versions\n" }]); + expect(res.remaining).toEqual([]); + expect(res.commit).toBeTruthy(); + expect(await store.publish("c1", ada)).toMatchObject({ ok: true, published: true }); + expect(readFileSync(join(pub, "lessons/welcome.md"), "utf8")).toBe("# Both versions\n"); + }); + + test("abort returns the draft to its pre-merge state", async () => { + const { store, pub } = await courseWithEdit("# Studio version\n"); + writeFileSync(join(pub, "lessons/welcome.md"), "# External version\n"); + await extGit(pub, "commit", "-qam", "External edit"); + await store.updateDraft("c1", ada); + await store.abortMerge("c1", ada.id); + const st = await store.status("c1", ada.id); + expect(st.merging).toBe(false); + expect((await store.readFile("c1", ada.id, "lessons/welcome.md"))!.content).toBe("# Studio version\n"); + }); + + test("a non-conflicting external commit merges cleanly into the draft", async () => { + const { store, pub } = await courseWithEdit(); + writeFileSync(join(pub, "extra.md"), "external\n"); + await extGit(pub, "add", "extra.md"); + await extGit(pub, "commit", "-qm", "External add"); + expect((await store.updateDraft("c1", ada)).ok).toBe(true); + expect((await store.readFile("c1", ada.id, "extra.md"))!.content).toBe("external\n"); + }); +}); + +describe("sync", () => { + test("no remote: an error, nothing happens", async () => { + const { store } = await courseWithEdit(); + expect(await store.sync("c1", ada)).toMatchObject({ ok: false, reason: "no-remote" }); + }); + + test("push to an empty remote, pull a remote commit, report a conflicting divergence", async () => { + const { store, pub } = await courseWithEdit(); + await store.publish("c1", ada); + const remote = join(tempDir(), "remote.git"); + await extGit(tempDir(), "init", "-q", "--bare", "-b", "main", remote); + await store.setRemote("c1", remote); + expect(await store.remote("c1")).toBe(remote); + expect(await store.sync("c1", ada)).toMatchObject({ ok: true, pushed: true, pulled: false }); + expect(await extGit(remote, "rev-parse", "main")).toBe(await extGit(pub, "rev-parse", "HEAD")); + + // someone else pushes to the remote + const other = join(tempDir(), "other"); + await extGit(tempDir(), "clone", "-q", remote, other); + writeFileSync(join(other, "lessons/welcome.md"), "# Remote version\n"); + await extGit(other, "commit", "-qam", "Remote edit"); + await extGit(other, "push", "-q", "origin", "main"); + expect(await store.sync("c1", ada)).toMatchObject({ ok: true, pulled: true, pushed: false }); + expect(readFileSync(join(pub, "lessons/welcome.md"), "utf8")).toBe("# Remote version\n"); + + // diverge with a conflict + writeFileSync(join(other, "lessons/welcome.md"), "# Remote again\n"); + await extGit(other, "commit", "-qam", "Remote edit 2"); + await extGit(other, "push", "-q", "origin", "main"); + await store.updateDraft("c1", ada); + const cur = (await store.readFile("c1", ada.id, "lessons/welcome.md"))!; + await store.saveFiles("c1", ada, [{ path: "lessons/welcome.md", content: "# Local again\n", baseHash: cur.hash }], "local"); + expect((await store.publish("c1", ada)).ok).toBe(true); + const head = await extGit(pub, "rev-parse", "HEAD"); + expect(await store.sync("c1", ada)).toEqual({ ok: false, reason: "conflict", conflicts: ["lessons/welcome.md"] }); + expect(await extGit(pub, "rev-parse", "HEAD")).toBe(head); + expect(readFileSync(join(pub, "lessons/welcome.md"), "utf8")).toBe("# Local again\n"); + + // the author resolves via the draft, then Sync goes through + expect(await store.updateDraft("c1", ada, "remote")).toMatchObject({ ok: false, reason: "conflict" }); + await store.resolve("c1", ada, [{ path: "lessons/welcome.md", content: "# Merged\n" }]); + expect((await store.publish("c1", ada)).ok).toBe(true); + expect(await store.sync("c1", ada)).toMatchObject({ ok: true, pushed: true }); + expect(await extGit(remote, "show", "main:lessons/welcome.md")).toBe("# Merged"); + void bob; + }); +}); From 3c2220145bb48e466aa9b24d26bf6118e112ddf2 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 03:15:33 +0100 Subject: [PATCH 10/62] feat(ai): Anthropic provider over fetch with streaming tool use Co-Authored-By: Claude Opus 5.5 --- .../feature-ai/src/providers/anthropic.ts | 139 ++++++++++++++++++ packages/feature-ai/src/providers/sse.ts | 44 ++++++ packages/feature-ai/test/anthropic.test.ts | 116 +++++++++++++++ packages/feature-ai/test/helpers.ts | 53 +++++++ 4 files changed, 352 insertions(+) create mode 100644 packages/feature-ai/src/providers/anthropic.ts create mode 100644 packages/feature-ai/src/providers/sse.ts create mode 100644 packages/feature-ai/test/anthropic.test.ts create mode 100644 packages/feature-ai/test/helpers.ts diff --git a/packages/feature-ai/src/providers/anthropic.ts b/packages/feature-ai/src/providers/anthropic.ts new file mode 100644 index 0000000..1576bf3 --- /dev/null +++ b/packages/feature-ai/src/providers/anthropic.ts @@ -0,0 +1,139 @@ +import type { AiProvider, ChatMessage, ToolSpec } from "@stc/contracts"; +import type { Effort } from "../settings"; +import { errorMessage, readSse } from "./sse"; + +export const DEFAULT_ANTHROPIC_MODEL = "claude-opus-5-5"; +const FIRST_PARTY = "https://api.anthropic.com"; + +export interface AnthropicOptions { + apiKey: string; + model?: string; + baseUrl?: string; + effort?: Effort; + maxTokens?: number; + fetch?: typeof fetch; +} + +type Block = Record & { type: string }; +type ApiMessage = { role: "user" | "assistant"; content: Block[] }; + +/** + * Anthropic Messages API over plain fetch with streaming and tool use. + * - Thinking is left to the model default (adaptive on current models); no temperature or forced tool_choice. + * - The frozen ChatMessage contract has no slot for thinking blocks, so the provider remembers every + * response's raw content blocks by tool_use id and replays them unchanged when the caller sends back + * the matching assistant tool-call message (required for thinking + tool use). + * - On the first-party endpoint, server-side refusal fallbacks are on ("default" routing). + */ +export function createAnthropicProvider(o: AnthropicOptions): AiProvider { + const doFetch = o.fetch ?? fetch; + const base = (o.baseUrl || FIRST_PARTY).replace(/\/+$/, ""); + const firstParty = base === FIRST_PARTY; + const rawByToolId = new Map(); + + const toApi = (messages: ChatMessage[]): { system: string; messages: ApiMessage[] } => { + const system = messages.filter((m) => m.role === "system").map((m) => m.content).join("\n\n"); + const out: ApiMessage[] = []; + const push = (role: "user" | "assistant", block: Block) => { + const last = out[out.length - 1]; + if (last && last.role === role) last.content.push(block); + else out.push({ role, content: [block] }); + }; + for (const m of messages) { + if (m.role === "system") continue; + if (m.role === "tool") push("user", { type: "tool_result", tool_use_id: m.toolCallId ?? "", content: m.content }); + else if (m.role === "assistant" && m.toolCallId) { + let input: unknown = {}; + try { input = JSON.parse(m.content || "{}"); } catch { input = { value: m.content }; } + push("assistant", { type: "tool_use", id: m.toolCallId, name: m.name ?? "", input }); + } else if (m.content) push(m.role === "assistant" ? "assistant" : "user", { type: "text", text: m.content }); + } + // Replace assistant turns we produced with their exact original blocks (thinking signatures intact). + for (const msg of out) { + if (msg.role !== "assistant") continue; + const id = msg.content.find((b) => b.type === "tool_use")?.id as string | undefined; + const raw = id ? rawByToolId.get(id) : undefined; + if (raw) msg.content = structuredClone(raw); + } + return { system, messages: out }; + }; + + return { + id: "anthropic", + async *chat(messages: ChatMessage[], opts: { tools?: ToolSpec[]; signal?: AbortSignal }) { + const { system, messages: apiMessages } = toApi(messages); + const body: Record = { + model: o.model || DEFAULT_ANTHROPIC_MODEL, + max_tokens: o.maxTokens ?? 64000, + stream: true, + messages: apiMessages, + }; + if (system) body.system = system; + if (opts.tools?.length) body.tools = opts.tools.map((t) => ({ name: t.name, description: t.description, input_schema: t.inputSchema })); + if (o.effort) body.output_config = { effort: o.effort }; + const headers: Record = { + "content-type": "application/json", + "x-api-key": o.apiKey, + "anthropic-version": "2023-06-01", + }; + if (firstParty) { + body.fallbacks = "default"; + headers["anthropic-beta"] = "server-side-fallback-2026-07-01"; + } + const res = await doFetch(`${base}/v1/messages`, { method: "POST", headers, body: JSON.stringify(body), signal: opts.signal }); + if (!res.ok || !res.body) throw new Error(`Anthropic API error ${await errorMessage(res)}`); + + const blocks: Block[] = []; + const partialJson = new Map(); + const toolIds: string[] = []; + let stopReason: string | null = null; + for await (const { data } of readSse(res.body)) { + let ev: any; + try { ev = JSON.parse(data); } catch { continue; } + switch (ev.type) { + case "content_block_start": { + const b = { ...ev.content_block } as Block; + blocks[ev.index] = b; + if (b.type === "tool_use") partialJson.set(ev.index, ""); + break; + } + case "content_block_delta": { + const b = blocks[ev.index]; + const d = ev.delta ?? {}; + if (!b) break; + if (d.type === "text_delta") { + b.text = String(b.text ?? "") + d.text; + yield { text: d.text as string }; + } else if (d.type === "input_json_delta") partialJson.set(ev.index, (partialJson.get(ev.index) ?? "") + d.partial_json); + else if (d.type === "thinking_delta") b.thinking = String(b.thinking ?? "") + d.thinking; + else if (d.type === "signature_delta") b.signature = String(b.signature ?? "") + d.signature; + else if (d.type === "citations_delta") b.citations = [...((b.citations as unknown[]) ?? []), d.citation]; + break; + } + case "content_block_stop": { + const b = blocks[ev.index]; + if (b?.type === "tool_use") { + const json = partialJson.get(ev.index) ?? ""; + let input: unknown = {}; + try { input = json ? JSON.parse(json) : (b.input ?? {}); } catch { input = { _invalidJson: json }; } + b.input = input; + toolIds.push(b.id as string); + yield { toolCall: { id: b.id as string, name: b.name as string, input } }; + } + break; + } + case "message_delta": + if (ev.delta?.stop_reason) stopReason = ev.delta.stop_reason; + break; + case "error": + throw new Error(`Anthropic API error: ${ev.error?.message ?? "stream error"}`); + } + } + const content = blocks.filter(Boolean); + for (const id of toolIds) rawByToolId.set(id, content); + if (stopReason === "refusal" && !content.some((b) => b.type === "text" && b.text)) { + yield { text: "The model declined to answer this request." }; + } + }, + }; +} diff --git a/packages/feature-ai/src/providers/sse.ts b/packages/feature-ai/src/providers/sse.ts new file mode 100644 index 0000000..7b06171 --- /dev/null +++ b/packages/feature-ai/src/providers/sse.ts @@ -0,0 +1,44 @@ +/** Minimal Server-Sent Events reader: yields one {event, data} per blank-line-terminated record. */ +export async function* readSse(body: ReadableStream): AsyncIterable<{ event: string; data: string }> { + const decoder = new TextDecoder(); + let buf = ""; + let event = "message"; + let data: string[] = []; + const flushLine = function* (line: string): Generator<{ event: string; data: string }> { + if (line === "") { + if (data.length) yield { event, data: data.join("\n") }; + event = "message"; + data = []; + return; + } + if (line.startsWith(":")) return; + const i = line.indexOf(":"); + const field = i === -1 ? line : line.slice(0, i); + let value = i === -1 ? "" : line.slice(i + 1); + if (value.startsWith(" ")) value = value.slice(1); + if (field === "event") event = value; + else if (field === "data") data.push(value); + }; + for await (const chunk of body as unknown as AsyncIterable) { + buf += decoder.decode(chunk, { stream: true }); + let nl: number; + while ((nl = buf.search(/\r\n|\n|\r/)) !== -1) { + const line = buf.slice(0, nl); + buf = buf.slice(nl + (buf.startsWith("\r\n", nl) ? 2 : 1)); + yield* flushLine(line); + } + } + buf += decoder.decode(); + if (buf) yield* flushLine(buf); + yield* flushLine(""); +} + +export async function errorMessage(res: Response): Promise { + const text = await res.text().catch(() => ""); + try { + const j = JSON.parse(text); + const m = j?.error?.message ?? j?.message ?? j?.error; + if (typeof m === "string") return `${res.status}: ${m}`; + } catch {} + return `${res.status}: ${text.slice(0, 300) || res.statusText}`; +} diff --git a/packages/feature-ai/test/anthropic.test.ts b/packages/feature-ai/test/anthropic.test.ts new file mode 100644 index 0000000..7477b07 --- /dev/null +++ b/packages/feature-ai/test/anthropic.test.ts @@ -0,0 +1,116 @@ +import { expect, test } from "bun:test"; +import type { ChatMessage } from "@stc/contracts"; +import { createAnthropicProvider } from "../src/providers/anthropic"; +import { collect, fakeFetch, sseBody } from "./helpers"; + +const ok = (events: [string | null, unknown][]) => () => new Response(sseBody(events), { headers: { "content-type": "text/event-stream" } }); + +const textTurn: [string, unknown][] = [ + ["message_start", { type: "message_start", message: { id: "m1", content: [] } }], + ["content_block_start", { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }], + ["ping", { type: "ping" }], + ["content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "Hello " } }], + ["content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "world" } }], + ["content_block_stop", { type: "content_block_stop", index: 0 }], + ["message_delta", { type: "message_delta", delta: { stop_reason: "end_turn" } }], + ["message_stop", { type: "message_stop" }], +]; + +const toolTurn: [string, unknown][] = [ + ["message_start", { type: "message_start", message: { id: "m2", content: [] } }], + ["content_block_start", { type: "content_block_start", index: 0, content_block: { type: "thinking", thinking: "", signature: "" } }], + ["content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "thinking_delta", thinking: "" } }], + ["content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "signature_delta", signature: "SIG-abc" } }], + ["content_block_stop", { type: "content_block_stop", index: 0 }], + ["content_block_start", { type: "content_block_start", index: 1, content_block: { type: "text", text: "" } }], + ["content_block_delta", { type: "content_block_delta", index: 1, delta: { type: "text_delta", text: "Let me look." } }], + ["content_block_stop", { type: "content_block_stop", index: 1 }], + ["content_block_start", { type: "content_block_start", index: 2, content_block: { type: "tool_use", id: "toolu_1", name: "search_course", input: {} } }], + ["content_block_delta", { type: "content_block_delta", index: 2, delta: { type: "input_json_delta", partial_json: "{\"que" } }], + ["content_block_delta", { type: "content_block_delta", index: 2, delta: { type: "input_json_delta", partial_json: "ry\": \"add\"}" } }], + ["content_block_stop", { type: "content_block_stop", index: 2 }], + ["message_delta", { type: "message_delta", delta: { stop_reason: "tool_use" } }], + ["message_stop", { type: "message_stop" }], +]; + +test("streams text and sends a well-formed request", async () => { + const ff = fakeFetch([ok(textTurn)]); + const p = createAnthropicProvider({ apiKey: "sk-test", fetch: ff.fetch }); + const msgs: ChatMessage[] = [{ role: "system", content: "You are kind." }, { role: "user", content: "Hi" }]; + const r = await collect(p.chat(msgs, { tools: [{ name: "search_course", description: "Search", inputSchema: { type: "object", properties: {} } }] })); + expect(r.text).toBe("Hello world"); + const c = ff.calls[0]!; + expect(c.url).toBe("https://api.anthropic.com/v1/messages"); + const h = new Headers(c.init.headers); + expect(h.get("x-api-key")).toBe("sk-test"); + expect(h.get("anthropic-version")).toBe("2023-06-01"); + expect(c.body.model).toBe("claude-opus-5-5"); + expect(c.body.stream).toBe(true); + expect(c.body.system).toBe("You are kind."); + expect(c.body.messages).toEqual([{ role: "user", content: [{ type: "text", text: "Hi" }] }]); + expect(c.body.tools).toEqual([{ name: "search_course", description: "Search", input_schema: { type: "object", properties: {} } }]); + expect(c.body.thinking).toBeUndefined(); + expect(c.body.temperature).toBeUndefined(); + expect(c.body.tool_choice).toBeUndefined(); + expect(c.body.fallbacks).toBe("default"); + expect(h.get("anthropic-beta")).toBe("server-side-fallback-2026-07-01"); +}); + +test("assembles tool input and replays thinking blocks unchanged on the next call", async () => { + const ff = fakeFetch([ok(toolTurn), ok(textTurn)]); + const p = createAnthropicProvider({ apiKey: "k", fetch: ff.fetch }); + const msgs: ChatMessage[] = [{ role: "user", content: "How do I add?" }]; + const first = await collect(p.chat(msgs, {})); + expect(first.text).toBe("Let me look."); + expect(first.toolCalls).toEqual([{ id: "toolu_1", name: "search_course", input: { query: "add" } }]); + + msgs.push({ role: "assistant", content: "Let me look." }); + msgs.push({ role: "assistant", content: JSON.stringify({ query: "add" }), toolCallId: "toolu_1", name: "search_course" }); + msgs.push({ role: "tool", content: "lessons/04-exercise.md: ...", toolCallId: "toolu_1", name: "search_course" }); + await collect(p.chat(msgs, {})); + const sent = ff.calls[1]!.body.messages; + expect(sent).toHaveLength(3); + expect(sent[1]).toEqual({ + role: "assistant", + content: [ + { type: "thinking", thinking: "", signature: "SIG-abc" }, + { type: "text", text: "Let me look." }, + { type: "tool_use", id: "toolu_1", name: "search_course", input: { query: "add" } }, + ], + }); + expect(sent[2]).toEqual({ role: "user", content: [{ type: "tool_result", tool_use_id: "toolu_1", content: "lessons/04-exercise.md: ..." }] }); +}); + +test("tool calls it did not produce are mapped from the message fields", async () => { + const ff = fakeFetch([ok(textTurn)]); + const p = createAnthropicProvider({ apiKey: "k", fetch: ff.fetch }); + await collect(p.chat([ + { role: "user", content: "q" }, + { role: "assistant", content: "{\"x\":1}", toolCallId: "t9", name: "list_notes" }, + { role: "tool", content: "none", toolCallId: "t9" }, + ], {})); + expect(ff.calls[0]!.body.messages[1]).toEqual({ role: "assistant", content: [{ type: "tool_use", id: "t9", name: "list_notes", input: { x: 1 } }] }); +}); + +test("errors carry the API message", async () => { + const ff = fakeFetch([() => Response.json({ type: "error", error: { type: "authentication_error", message: "invalid x-api-key" } }, { status: 401 })]); + const p = createAnthropicProvider({ apiKey: "bad", fetch: ff.fetch }); + await expect(collect(p.chat([{ role: "user", content: "x" }], {}))).rejects.toThrow(/invalid x-api-key/); + const ff2 = fakeFetch([ok([["error", { type: "error", error: { type: "overloaded_error", message: "Overloaded" } }]])]); + await expect(collect(createAnthropicProvider({ apiKey: "k", fetch: ff2.fetch }).chat([{ role: "user", content: "x" }], {}))).rejects.toThrow(/Overloaded/); +}); + +test("a refusal yields a plain sentence", async () => { + const ff = fakeFetch([ok([["message_start", { type: "message_start", message: { content: [] } }], ["message_delta", { type: "message_delta", delta: { stop_reason: "refusal" } }], ["message_stop", { type: "message_stop" }]])]); + const r = await collect(createAnthropicProvider({ apiKey: "k", fetch: ff.fetch }).chat([{ role: "user", content: "x" }], {})); + expect(r.text).toContain("declined"); +}); + +test("custom base URL, model and effort", async () => { + const ff = fakeFetch([ok(textTurn)]); + await collect(createAnthropicProvider({ apiKey: "k", baseUrl: "http://proxy.local/", model: "claude-sonnet-5-5", effort: "high", fetch: ff.fetch }).chat([{ role: "user", content: "x" }], {})); + expect(ff.calls[0]!.url).toBe("http://proxy.local/v1/messages"); + expect(ff.calls[0]!.body.model).toBe("claude-sonnet-5-5"); + expect(ff.calls[0]!.body.output_config).toEqual({ effort: "high" }); + expect(ff.calls[0]!.body.fallbacks).toBeUndefined(); +}); diff --git a/packages/feature-ai/test/helpers.ts b/packages/feature-ai/test/helpers.ts new file mode 100644 index 0000000..d54648f --- /dev/null +++ b/packages/feature-ai/test/helpers.ts @@ -0,0 +1,53 @@ +import type { AiProvider, ChatMessage, ToolSpec } from "@stc/contracts"; + +/** SSE body from a list of [event, data] pairs, split into awkward chunks to exercise buffering. */ +export function sseBody(events: [string | null, unknown][], chunk = 7): ReadableStream { + const text = events.map(([e, d]) => `${e ? `event: ${e}\n` : ""}data: ${typeof d === "string" ? d : JSON.stringify(d)}\n\n`).join(""); + const bytes = new TextEncoder().encode(text); + let i = 0; + return new ReadableStream({ + pull(c) { + if (i >= bytes.length) return c.close(); + c.enqueue(bytes.slice(i, i + chunk)); + i += chunk; + }, + }); +} + +export type Captured = { url: string; init: RequestInit; body: any }; +/** A fake fetch that records requests and answers from a queue of responses. */ +export function fakeFetch(responses: (() => Response)[]) { + const calls: Captured[] = []; + const f = (async (url: string | URL | Request, init?: RequestInit) => { + calls.push({ url: String(url), init: init ?? {}, body: init?.body ? JSON.parse(String(init.body)) : null }); + const next = responses.shift(); + if (!next) throw new Error("fakeFetch: no response queued"); + return next(); + }) as unknown as typeof fetch; + return { fetch: f, calls }; +} + +export async function collect(it: AsyncIterable<{ text?: string; toolCall?: { id: string; name: string; input: unknown } }>) { + let text = ""; + const toolCalls: { id: string; name: string; input: unknown }[] = []; + for await (const ev of it) { + if (ev.text) text += ev.text; + if (ev.toolCall) toolCalls.push(ev.toolCall); + } + return { text, toolCalls }; +} + +/** Scripted fake provider: each call to chat() plays the next turn and records the messages it got. */ +export function fakeProvider(turns: ((messages: ChatMessage[], tools?: ToolSpec[]) => { text?: string; toolCalls?: { id: string; name: string; input: unknown }[] })[]) { + const seen: { messages: ChatMessage[]; tools?: ToolSpec[] }[] = []; + const provider: AiProvider = { + id: "fake", + async *chat(messages, opts) { + seen.push({ messages: structuredClone(messages), tools: opts.tools }); + const turn = turns[Math.min(seen.length - 1, turns.length - 1)]!(messages, opts.tools); + if (turn.text) for (const part of turn.text.match(/.{1,8}/gs) ?? []) yield { text: part }; + for (const tc of turn.toolCalls ?? []) yield { toolCall: tc }; + }, + }; + return { provider, seen }; +} From 9738da458d94d224ec8b738ad043ff6022ecbfc9 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 03:15:44 +0100 Subject: [PATCH 11/62] feat(import): video-folder and pdf-deck jobs, deck cache Co-Authored-By: Claude Opus 5.5 --- packages/feature-import/src/deck.ts | 49 ++++++++++++ packages/feature-import/src/jobs/pdf-deck.ts | 26 +++++++ .../feature-import/src/jobs/video-folder.ts | 62 +++++++++++++++ .../test/video-folder-pdf.test.ts | 75 +++++++++++++++++++ 4 files changed, 212 insertions(+) create mode 100644 packages/feature-import/src/deck.ts create mode 100644 packages/feature-import/src/jobs/pdf-deck.ts create mode 100644 packages/feature-import/src/jobs/video-folder.ts create mode 100644 packages/feature-import/test/video-folder-pdf.test.ts diff --git a/packages/feature-import/src/deck.ts b/packages/feature-import/src/deck.ts new file mode 100644 index 0000000..9844060 --- /dev/null +++ b/packages/feature-import/src/deck.ts @@ -0,0 +1,49 @@ +import { basename, extname, join } from "node:path"; +import { copyFile, mkdir } from "node:fs/promises"; +import { coverCacheDir, deckCacheDir, exists, type Deps, type ManagedCourse } from "./context"; +import { putFile } from "./media"; +import { renderPdf } from "./pdf"; +import { slugify, uniqueId } from "./util/text"; + +export interface AttachedDeck { deckId: string; file: string; pages?: number; warning?: string } + +/** All lesson and module ids of a raw manifest, for validating "attach to". */ +export function attachTargets(manifest: Record): Set { + const ids = new Set(); + for (const m of manifest.modules ?? []) { + ids.add(m.id); + for (const l of m.lessons ?? []) ids.add(l.id); + } + return ids; +} + +/** + * Copies a PDF into the course (media/decks/), registers it in the (in-memory) manifest, and builds the derived cache: + * page images + text.json under /cache/decks/// and the first page as a cover candidate. + * A failure of the PDF tool only degrades to a warning: the deck stays attached and the cache can be rebuilt. + * The caller writes the manifest. + */ +export async function attachDeck(d: Deps, course: ManagedCourse, pdfPath: string, attachTo: string, opts: { title?: string; mode?: "copy" | "link"; onProgress?: (fraction: number) => void } = {}): Promise { + if (!attachTarget(course, attachTo)) throw new Error(`attachTo "${attachTo}" is not a lesson or module of course "${course.id}"`); + const decks: any[] = (course.manifest.decks ??= []); + const baseName = basename(pdfPath, extname(pdfPath)); + const deckId = uniqueId(`d-${slugify(baseName)}`, (x) => decks.some((k) => k.id === x)); + let file = `media/decks/${basename(pdfPath)}`; + for (let n = 2; await exists(join(course.dir, file)); n++) file = `media/decks/${baseName}-${n}${extname(pdfPath)}`; + await putFile(pdfPath, join(course.dir, file), opts.mode ?? "copy"); + decks.push({ id: deckId, file, title: opts.title ?? baseName.replace(/[_]+/g, " ").trim(), attachTo }); + + const out: AttachedDeck = { deckId, file }; + try { + const r = await renderPdf(d.python, join(course.dir, file), deckCacheDir(d, course.id, deckId), { onPage: (p, t) => opts.onProgress?.(p / t) }); + out.pages = r.pages; + await mkdir(coverCacheDir(d, course.id), { recursive: true }); + await copyFile(join(deckCacheDir(d, course.id, deckId), "p-001.jpg"), join(coverCacheDir(d, course.id), `deck-${deckId}.jpg`)); + } catch (e) { + out.warning = `deck "${basename(pdfPath)}" is attached, but page images and text could not be built (${e instanceof Error ? e.message.split("\n")[0] : e}); STC_PYTHON must point to Python with PyMuPDF`; + } + return out; +} + +const attachTarget = (c: ManagedCourse, id: string) => attachTargets(c.manifest).has(id); + diff --git a/packages/feature-import/src/jobs/pdf-deck.ts b/packages/feature-import/src/jobs/pdf-deck.ts new file mode 100644 index 0000000..7cfe3e4 --- /dev/null +++ b/packages/feature-import/src/jobs/pdf-deck.ts @@ -0,0 +1,26 @@ +import { stat } from "node:fs/promises"; +import { join } from "node:path"; +import type { ImportResult, JobContext } from "@stc/contracts"; +import { finish, managedCourse, saveManifest, type Deps } from "../context"; +import { attachDeck } from "../deck"; +import { optStr, parsePayload } from "./common"; + +export async function importPdfDeck(d: Deps, payload: unknown, jc: JobContext): Promise { + const { path, options } = parsePayload(payload, "pdf-deck"); + const warnings: string[] = []; + if (!(await stat(path).catch(() => null))?.isFile() || !/\.pdf$/i.test(path)) throw new Error(`not a PDF file: ${path}`); + const course = await managedCourse(d, optStr(options, "courseId")); + const attachTo = optStr(options, "attachTo"); + if (!attachTo) throw new Error("options.attachTo is required: the lesson or module id the deck belongs to"); + await jc.reportProgress(0.05, "Copying the deck"); + const r = await attachDeck(d, course, path, attachTo, { + title: optStr(options, "title"), + mode: options.media === "link" ? "link" : "copy", + onProgress: (f) => void jc.reportProgress(0.1 + 0.8 * f, "Rendering slides"), + }); + if (r.warning) warnings.push(r.warning); + await saveManifest(course.dir, course.manifest); + await finish(d, course.id, warnings); + await jc.reportProgress(1, r.pages ? `Deck attached (${r.pages} slides)` : "Deck attached"); + return { courseId: course.id, manifestPath: join(course.dir, "stcourse.yaml"), warnings }; +} diff --git a/packages/feature-import/src/jobs/video-folder.ts b/packages/feature-import/src/jobs/video-folder.ts new file mode 100644 index 0000000..b4ff4f4 --- /dev/null +++ b/packages/feature-import/src/jobs/video-folder.ts @@ -0,0 +1,62 @@ +import { stat } from "node:fs/promises"; +import { basename, extname, join } from "node:path"; +import type { ImportResult, JobContext } from "@stc/contracts"; +import { finish, frameCandidates, freshCourseId, publishCourse, saveManifest, sourceBlock, stageDir, type Deps } from "../context"; +import { probeDuration, putFile } from "../media"; +import { naturalSort } from "../util/text"; +import { openSource } from "../util/source"; +import { adoptCover, maybeQueueTranscription, optBool, optStr, parsePayload, withStage } from "./common"; + +const VIDEO = /\.(mp4|m4v|webm|mov|mkv|avi)$/i; +const BROWSER_OK = /\.(mp4|m4v|webm|mov)$/i; + +export function lessonTitle(file: string): string { + const base = basename(file, extname(file)); + const t = base.replace(/^\s*\d+\s*[-_.)\s]+(?=\S)/, "").replace(/_/g, " ").replace(/\s+/g, " ").trim(); + return t || base; +} + +export async function importVideoFolder(d: Deps, payload: unknown, jc: JobContext): Promise { + const { path, options } = parsePayload(payload, "video-folder"); + const warnings: string[] = []; + if (!(await stat(path).catch(() => null))?.isDirectory()) throw new Error(`not a folder: ${path}`); + const src = await openSource(path); + const files = naturalSort(src.entries.filter((e) => VIDEO.test(e) && !e.split("/").some((p) => p.startsWith(".")))); + if (files.length === 0) throw new Error(`no video files found in ${path}`); + const title = optStr(options, "title") ?? basename(path).replace(/[_]+/g, " "); + const id = await freshCourseId(d, optStr(options, "courseId"), title); + const mode = options.media === "link" ? "link" : "copy"; + const stage = await stageDir(d, "videos"); + + return withStage(stage, async () => { + const lessons: any[] = []; + const placed: string[] = []; + const used = new Set(); + for (const [i, f] of files.entries()) { + await jc.reportProgress(0.05 + 0.7 * (i / files.length), `Copying ${basename(f)}`); + let rel = `media/${f}`; + if (used.has(rel.toLowerCase())) rel = `media/${i + 1}-${basename(f)}`; + used.add(rel.toLowerCase()); + await putFile(join(path, f), join(stage, rel), mode); + placed.push(join(stage, rel)); + if (!BROWSER_OK.test(f)) warnings.push(`${f} may not play in browsers (${extname(f)}); consider converting it to MP4`); + const duration = await probeDuration(d.tools, join(stage, rel)); + lessons.push({ id: `l${i + 1}`, title: lessonTitle(f), type: "video", file: rel, ...(duration ? { duration } : {}) }); + } + await jc.reportProgress(0.8, "Making cover candidates"); + const covers = await frameCandidates(d, id, placed); + if (covers.length === 0) warnings.push("could not extract cover frames (is ffmpeg available?)"); + const cover = await adoptCover(d, id, stage, covers[0]); + await saveManifest(stage, { + schema: 1, id, title, visibility: "restricted", ...(cover ? { cover } : {}), + source: sourceBlock("video-folder", path), + modules: [{ id: "m1", title: "Videos", lessons }], + }); + await jc.reportProgress(0.92, "Publishing the course"); + const dir = await publishCourse(d, stage, id); + await finish(d, id, warnings); + await maybeQueueTranscription(d, id, optBool(options, "transcribe"), warnings); + await jc.reportProgress(1, `Imported ${files.length} videos`); + return { courseId: id, manifestPath: join(dir, "stcourse.yaml"), warnings }; + }); +} diff --git a/packages/feature-import/test/video-folder-pdf.test.ts b/packages/feature-import/test/video-folder-pdf.test.ts new file mode 100644 index 0000000..1df5d4b --- /dev/null +++ b/packages/feature-import/test/video-folder-pdf.test.ts @@ -0,0 +1,75 @@ +import { expect, test } from "bun:test"; +import { readdir, readFile, stat } from "node:fs/promises"; +import { join } from "node:path"; +import { importPdfDeck } from "../src/jobs/pdf-deck"; +import { importVideoFolder, lessonTitle } from "../src/jobs/video-folder"; +import { makeEnv } from "./harness"; +import { makePdf, makeVideo, tmp } from "./fixtures"; + +test("lessonTitle strips ordering prefixes", () => { + expect(lessonTitle("01 - Intro to Nix.mp4")).toBe("Intro to Nix"); + expect(lessonTitle("2_setup_env.mkv")).toBe("setup env"); + expect(lessonTitle("12.mp4")).toBe("12"); +}); + +async function folderCourse() { + const env = await makeEnv(); + const dir = await tmp(); + const src = join(dir, "My Homelab Videos"); + await makeVideo(join(src, "10 - Last.mp4"), 1); + await makeVideo(join(src, "2 - Second Clip.mp4"), 1); + await makeVideo(join(src, "1 - Ünïcode first.mp4"), 1); + const before = (await readdir(src)).sort(); + const r = await importVideoFolder(env.deps, { kind: "video-folder", path: src }, env.jc); + return { env, src, before, r }; +} + +test("a folder of videos becomes one lesson per file in natural order", async () => { + const { env, src, before, r } = await folderCourse(); + expect(r.courseId).toBe("my-homelab-videos"); + const m = env.deps.ctx.courses.get(r.courseId)!; + expect(m.status).toBe("ok"); + expect(m.manifest!.modules[0]!.lessons.map((l) => l.title)).toEqual(["Ünïcode first", "Second Clip", "Last"]); + expect(m.manifest!.visibility).toBe("restricted"); + expect(m.manifest!.source?.kind).toBe("video-folder"); + expect(env.deps.ctx.courses.resolvePath(r.courseId, "media/1 - Ünïcode first.mp4")).toBeTruthy(); + expect((await readdir(src)).sort()).toEqual(before); +}); + +test("a deck attaches to a lesson, builds page images and text, and leaves the PDF original alone", async () => { + const { env, r } = await folderCourse(); + const d = await tmp(); + await makePdf(join(d, "Slides Deck.pdf"), ["Alpha slide", "Beta slide", "Gamma slide"]); + const before = await readFile(join(d, "Slides Deck.pdf")); + const res = await importPdfDeck(env.deps, { kind: "pdf-deck", path: join(d, "Slides Deck.pdf"), options: { courseId: r.courseId, attachTo: "l2", title: "My deck" } }, env.jc); + expect(res.warnings).toEqual([]); + const c = env.deps.ctx.courses.get(r.courseId)!; + expect(c.status).toBe("ok"); + expect(c.manifest!.decks).toEqual([{ id: "d-slides-deck", file: "media/decks/Slides Deck.pdf", title: "My deck", attachTo: "l2" }]); + const cache = join(env.dataDir, "cache/decks", r.courseId, "d-slides-deck"); + expect((await readdir(cache)).sort()).toEqual(["p-001.jpg", "p-002.jpg", "p-003.jpg", "text.json"]); + expect(JSON.parse(await readFile(join(cache, "text.json"), "utf8"))[2].text).toContain("Gamma slide"); + expect((await stat(join(env.dataDir, "cache/covers", r.courseId, "deck-d-slides-deck.jpg"))).size).toBeGreaterThan(100); + expect(Buffer.compare(await readFile(join(d, "Slides Deck.pdf")), before)).toBe(0); +}); + +test("deck attach validates the target and refuses courses the importer did not create", async () => { + const { env, r } = await folderCourse(); + const d = await tmp(); + await makePdf(join(d, "a.pdf")); + await expect(importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: r.courseId, attachTo: "nope" } }, env.jc)).rejects.toThrow(/not a lesson or module/); + await expect(importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: r.courseId } }, env.jc)).rejects.toThrow(/attachTo is required/); + await expect(importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: "sample", attachTo: "x" } }, env.jc)).rejects.toThrow(/only edits courses it created/); + await expect(importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: "../x", attachTo: "x" } }, env.jc)).rejects.toThrow(/only edits courses it created/); + await expect(importPdfDeck(env.deps, { path: join(d, "missing.pdf"), options: {} }, env.jc)).rejects.toThrow(/not a PDF/); +}); + +test("without PyMuPDF the deck is still attached, with a warning", async () => { + const { env, r } = await folderCourse(); + env.deps.python = "/nonexistent/python"; + const d = await tmp(); + await makePdf(join(d, "a.pdf")); + const res = await importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: r.courseId, attachTo: "m1" } }, env.jc); + expect(res.warnings.join()).toContain("page images and text could not be built"); + expect(env.deps.ctx.courses.get(r.courseId)!.manifest!.decks).toHaveLength(1); +}); From 0b117159cd84d4e27d8c364e112e2e0139c7b127 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 03:15:48 +0100 Subject: [PATCH 12/62] docs: WP-A learner package plan Co-Authored-By: Claude Opus 5.5 --- bun.lock | 125 +++++++++++++++++ docs/superpowers/plans/2026-10-30-learner.md | 136 +++++++++++++++++++ packages/feature-learner/package.json | 9 +- 3 files changed, 269 insertions(+), 1 deletion(-) create mode 100644 docs/superpowers/plans/2026-10-30-learner.md diff --git a/bun.lock b/bun.lock index 15e30fe..fd7707b 100644 --- a/bun.lock +++ b/bun.lock @@ -57,6 +57,13 @@ "packages/feature-learner": { "name": "@stc/feature-learner", "version": "0.0.0", + "dependencies": { + "@better-auth/sso": "^1.7.14", + "better-auth": "^1.7.14", + "dompurify": "^3.4.17", + "marked": "^18.1.2", + "pdfjs-dist": "^6.4.299", + }, }, "packages/feature-studio": { "name": "@stc/feature-studio", @@ -68,14 +75,68 @@ }, }, "packages": { + "@authenio/xml-encryption": ["@authenio/xml-encryption@6.0.2", "", { "dependencies": { "@xmldom/xmldom": "^0.8.13", "escape-html": "^1.0.3" } }, "sha512-/jv5qO935fH4Ma+fxh5rJLRYZvUM57fd/I74IiSoR8cFm+sgjszXFEEn1PkqZhhBD4MVkH2YT+/UDY+pnW0S0Q=="], + + "@better-auth/core": ["@better-auth/core@1.7.14", "", { "dependencies": { "@standard-schema/spec": "^1.1.0", "zod": "^4.5.4" }, "peerDependencies": { "@better-auth/utils": "^0.4.2 || ^0.5.0", "@better-fetch/fetch": "1.3.3", "@opentelemetry/api": "^1.9.0", "better-call": "1.4.4", "jose": "^6.1.0", "kysely": "^0.28.5 || ^0.29.0", "nanostores": "^1.0.1" }, "optionalPeers": ["@opentelemetry/api"] }, "sha512-ByhdrGVwp/el+hoTYKSVOHIH23/lQj9TQ1txdGcWOg22tnfr3hJ6ukzySooeYj/R36XsQgBhkJqIp8mKsNeErA=="], + + "@better-auth/drizzle-adapter": ["@better-auth/drizzle-adapter@1.7.14", "", { "peerDependencies": { "@better-auth/core": "^1.7.14", "@better-auth/utils": "^0.4.2 || ^0.5.0", "drizzle-orm": "^0.45.2 || >=1.0.0-rc.1 <2.0.0" }, "optionalPeers": ["drizzle-orm"] }, "sha512-5RC8Ec5Ur15FWsS4As9/VrlRbB3+9EfShxIpA5A1FcD6IgPxFvzQNkAm2+FYx9LUSqx47NPf9hBTG2GUCMP9Ig=="], + + "@better-auth/kysely-adapter": ["@better-auth/kysely-adapter@1.7.14", "", { "peerDependencies": { "@better-auth/core": "^1.7.14", "@better-auth/utils": "^0.4.2 || ^0.5.0", "kysely": "^0.28.17 || ^0.29.0" }, "optionalPeers": ["kysely"] }, "sha512-4wkqj8a8JVpWDYLlgzfbk7eOaWCFkL6HHQd6CLamFKaC8vNsXew1x4v5dE5O6dGr8j4g5qMISS+BAvOuIvBevg=="], + + "@better-auth/memory-adapter": ["@better-auth/memory-adapter@1.7.14", "", { "peerDependencies": { "@better-auth/core": "^1.7.14", "@better-auth/utils": "^0.4.2 || ^0.5.0" } }, "sha512-Ojv9BcmFyHuTHha9ZM/NUW1ut9J84wLngta+07Es0Ejn8kekqP5aHBpZd5Z8AvADs5iJUOWvQGFmGlPWAKJgBg=="], + + "@better-auth/mongo-adapter": ["@better-auth/mongo-adapter@1.7.14", "", { "peerDependencies": { "@better-auth/core": "^1.7.14", "@better-auth/utils": "^0.4.2 || ^0.5.0", "mongodb": "^6.0.0 || ^7.0.0" }, "optionalPeers": ["mongodb"] }, "sha512-6IA0RSPZFMdRwwnLZ9oWnDIiD0y0o0azTHUfa01BNU2AFJgMDkeiASUV3bKcJBskzQoyQrff74eFYTsTSeeVUQ=="], + + "@better-auth/prisma-adapter": ["@better-auth/prisma-adapter@1.7.14", "", { "peerDependencies": { "@better-auth/core": "^1.7.14", "@better-auth/utils": "^0.4.2 || ^0.5.0", "@prisma/client": "^5.0.0 || ^6.0.0 || ^7.0.0", "prisma": "^5.0.0 || ^6.0.0 || ^7.0.0" }, "optionalPeers": ["@prisma/client", "prisma"] }, "sha512-mF6t5xqeZNWzUe0ErmrSdzQmtslJscmu9X8Mp5QmROlpnnAzvSc70RILu8lHms8CePt7N34+KRH1QvYjGFrVGQ=="], + + "@better-auth/sso": ["@better-auth/sso@1.7.14", "", { "dependencies": { "@xmldom/xmldom": "^0.9.12", "fast-xml-parser": "^5.8.0", "jose": "^6.2.3", "samlify": "^2.13.1", "tldts": "^7.4.3", "zod": "^4.5.4" }, "peerDependencies": { "@better-auth/core": "^1.7.14", "@better-auth/utils": "^0.4.2 || ^0.5.0", "@better-fetch/fetch": "1.3.3", "better-auth": "^1.7.14", "better-call": "1.4.4" } }, "sha512-sfUzpm6GO5l5MSUA17qRO/E4INthqF+KRmymtj4Hhm+Xsx6CoRgr0BVIDfAic3tSqVhObcVYiiiZ38dXXhkCCw=="], + + "@better-auth/telemetry": ["@better-auth/telemetry@1.7.14", "", { "peerDependencies": { "@better-auth/core": "^1.7.14", "@better-auth/utils": "^0.4.2 || ^0.5.0", "@better-fetch/fetch": "1.3.3" } }, "sha512-PtKFzsH7gje/WJuUPDjckOoNFazsq8NMt3AdiXIPzHl7tZULcNGTKAX8K7+yn0rG6cO/4mtMgpOjhtilElOnmg=="], + + "@better-auth/utils": ["@better-auth/utils@0.5.1", "", { "dependencies": { "@noble/hashes": "^2.0.1" } }, "sha512-WETqw5vQ52uQ/xwWR4E8rTgT6fDEr/wnS+otDNkBCa/69281FCF32OKDdQUcahMCnjsvqXBNiEvAQHFB0EVRRg=="], + + "@better-fetch/fetch": ["@better-fetch/fetch@1.3.3", "", {}, "sha512-U6SlDt/GUWtK2JAxS/NhdoYTEim+EL1G5byz2TUj83KjLZBUpYzb5SmX0aXaCnbGkAjFKnV/o/0+cyXMbEcqqg=="], + "@fontsource/ibm-plex-mono": ["@fontsource/ibm-plex-mono@5.3.0", "", {}, "sha512-eTgnZjZEGk1QtD3ZstF+Vclo2HLAni8YMy34/DxllwZvyz1lR/1RF/xTiAquOBO7MvqBx8D2Ig2WCPMVfdZu7Q=="], "@fontsource/ibm-plex-sans": ["@fontsource/ibm-plex-sans@5.3.0", "", {}, "sha512-CbE4CbbEEZJX860XyUiRpsksXIQR8Rp2XDva2VO53NJox9tVNtusrysd2x5YkUEY3ErQ66W1IiiQL8/wihhw5w=="], "@fontsource/literata": ["@fontsource/literata@5.3.0", "", {}, "sha512-9JGMT5+YfBK49thoMq9ASaMRJErHT4V4QuMhAEwKHRLreOsIMP2xP6ZRLhPJ0e6YlET+kjWTJuN7LAxt9/gWGQ=="], + "@napi-rs/canvas": ["@napi-rs/canvas@1.0.13", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "1.0.13", "@napi-rs/canvas-darwin-arm64": "1.0.13", "@napi-rs/canvas-darwin-x64": "1.0.13", "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.13", "@napi-rs/canvas-linux-arm64-gnu": "1.0.13", "@napi-rs/canvas-linux-arm64-musl": "1.0.13", "@napi-rs/canvas-linux-riscv64-gnu": "1.0.13", "@napi-rs/canvas-linux-x64-gnu": "1.0.13", "@napi-rs/canvas-linux-x64-musl": "1.0.13", "@napi-rs/canvas-win32-arm64-msvc": "1.0.13", "@napi-rs/canvas-win32-x64-msvc": "1.0.13" } }, "sha512-b87ugZpQ5ViwqVBkjervxpwa0GmJPsKHLBNoZN6GpjWeHzTbDUGryriq+2DYW2XbB+xCqSIINQysC0/I87nPbw=="], + + "@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@1.0.13", "", { "os": "android", "cpu": "arm64" }, "sha512-trn7AaDzQ+yeeA0ezhkC7cB1ZDPtp4UYk0EvB5F8qtYL9jEVWtVrU30WOpI52DszoTxaTsCgf5PP4ZVaOlOIHQ=="], + + "@napi-rs/canvas-darwin-arm64": ["@napi-rs/canvas-darwin-arm64@1.0.13", "", { "os": "darwin", "cpu": "arm64" }, "sha512-6Y2djEetL8copxfNK6hg7oig/xqTwOchv19hBOm+zBNdKc/02Z3kjZN5wrLzv/l5O04/bAQS6I/AlQpwBlXgPg=="], + + "@napi-rs/canvas-darwin-x64": ["@napi-rs/canvas-darwin-x64@1.0.13", "", { "os": "darwin", "cpu": "x64" }, "sha512-OZoPwwu4TMMg8GTCFBrg8UFExwQTU/55i3uTC/izDat6PHE+ro9dimyQoOboFf91+NY7Pnn70ScR8f8I8+eZXw=="], + + "@napi-rs/canvas-linux-arm-gnueabihf": ["@napi-rs/canvas-linux-arm-gnueabihf@1.0.13", "", { "os": "linux", "cpu": "arm" }, "sha512-dZSYaKROuswU2eUWG5aBKndNVe68elo1YX93xtRax7q658tj8rYE+HwduovWJgs6qwv1r1/CLR60nDeBbyjvyg=="], + + "@napi-rs/canvas-linux-arm64-gnu": ["@napi-rs/canvas-linux-arm64-gnu@1.0.13", "", { "os": "linux", "cpu": "arm64" }, "sha512-W6FzYE8ADHI93In1vi80KBL6btDtAMuj50Ch5ptkWS8xuntyzbR4h5JDUmXoakxZn2yMInksudTEqVSWoFqFGg=="], + + "@napi-rs/canvas-linux-arm64-musl": ["@napi-rs/canvas-linux-arm64-musl@1.0.13", "", { "os": "linux", "cpu": "arm64" }, "sha512-+6fQ+rR49hkAa4voh2stLIwQ+4t+gmoseM8Bc53/DlMXb73IKymDPWK9A/NtWeqTW4ijWppYuyuxS2hPI8tzNA=="], + + "@napi-rs/canvas-linux-riscv64-gnu": ["@napi-rs/canvas-linux-riscv64-gnu@1.0.13", "", { "os": "linux", "cpu": "none" }, "sha512-17XEMO3t9RW/TtHtSFf2AkaSWuCWdzbPeidz7zrOFawHxoLSJCnpTI1hN5aUvrZNnkNNmt/vZVJCh0ryUaik3A=="], + + "@napi-rs/canvas-linux-x64-gnu": ["@napi-rs/canvas-linux-x64-gnu@1.0.13", "", { "os": "linux", "cpu": "x64" }, "sha512-lc48IwknVHBsNzXn9Xs5OkUsRX38Ml3Z1aOybQ51mMEzZckm8+4A0qROlyYHEoBPzr1kL3IU8ZUAqDb7wtdX8w=="], + + "@napi-rs/canvas-linux-x64-musl": ["@napi-rs/canvas-linux-x64-musl@1.0.13", "", { "os": "linux", "cpu": "x64" }, "sha512-bGY1r26S2bs8UNFJcPWcCSNdCFbKep5PATsvtgtC9A5u6SRgr2/jJE/RlhHL5m9ZpixNDhVkLY/5h16Kx5V9YA=="], + + "@napi-rs/canvas-win32-arm64-msvc": ["@napi-rs/canvas-win32-arm64-msvc@1.0.13", "", { "os": "win32", "cpu": "arm64" }, "sha512-2vRPBZoaIvJrUkDhZTE2Hn0YrnpJRCWVM3EJACoBKjuWI6WgPxGtUoRMsxRviCsyks8bKTSFdyw14CellnG2qA=="], + + "@napi-rs/canvas-win32-x64-msvc": ["@napi-rs/canvas-win32-x64-msvc@1.0.13", "", { "os": "win32", "cpu": "x64" }, "sha512-DUlW3lZPz63kYsf8mxgyyZuevhlH0T3/BGP9l+NazhB0F1+gL6D4hqZdnhZpw24DlWmTpATDcyYSZlJOi+DdfQ=="], + + "@noble/ciphers": ["@noble/ciphers@2.4.0", "", {}, "sha512-AnjFn0Jv92laAkvMrghlFZq4qQCIN/4DxFV/eooqtC2YTjB7kBeLMS2T9KJX4Dn+ZVXLOwK0lSgqDtx9gvxtiw=="], + + "@noble/hashes": ["@noble/hashes@2.4.0", "", {}, "sha512-X5XaVWZIBCT7HHZGm5I7ZQXDwLG+bGXuSrMQAW+7Zvl87h1kmc1ZB1VSRJcpUfoUrGQp4Fkoxm5kZ+Ms+aW+eA=="], + + "@nodable/entities": ["@nodable/entities@3.1.0", "", {}, "sha512-LsS/DjHr+uDM647Gru/cA8+J3a3HfhttwCKLyuoyN7yXTFCxKBtSgMQBvrW9yNPa2/zDRUNuGPaH5QUFoa4arQ=="], + "@remix-run/router": ["@remix-run/router@1.23.4", "", {}, "sha512-q7j5geK7xs3UJSdm9/iytUNclBnLmYx1EnSeCFXHPeutdqgIMeFeHtUZgS3EhlKxdBEAu8OwtJCwmLrEzpSs7Q=="], + "@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="], + "@stc/contracts": ["@stc/contracts@workspace:packages/contracts"], "@stc/core": ["@stc/core@workspace:packages/core"], @@ -104,10 +165,46 @@ "@types/react-dom": ["@types/react-dom@19.3.0", "", { "peerDependencies": { "@types/react": "^19.3.0" } }, "sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q=="], + "@types/trusted-types": ["@types/trusted-types@2.0.7", "", {}, "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw=="], + + "@xmldom/is-dom-node": ["@xmldom/is-dom-node@1.0.1", "", {}, "sha512-CJDxIgE5I0FH+ttq/Fxy6nRpxP70+e2O048EPe85J2use3XKdatVM7dDVvFNjQudd9B49NPoZ+8PG49zj4Er8Q=="], + + "@xmldom/xmldom": ["@xmldom/xmldom@0.9.12", "", {}, "sha512-5AXjrcMClTryPe9LgZrygpB1lj7s0S9E0+W+AHaVKAVyHanafK86iPSvG5xHVSp/jC+VH1UXu0TAEmY279xH7A=="], + + "anynum": ["anynum@1.0.1", "", {}, "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A=="], + + "better-auth": ["better-auth@1.7.14", "", { "dependencies": { "@better-auth/core": "1.7.14", "@better-auth/drizzle-adapter": "1.7.14", "@better-auth/kysely-adapter": "1.7.14", "@better-auth/memory-adapter": "1.7.14", "@better-auth/mongo-adapter": "1.7.14", "@better-auth/prisma-adapter": "1.7.14", "@better-auth/telemetry": "1.7.14", "@better-auth/utils": "0.5.1", "@better-fetch/fetch": "1.3.3", "@noble/ciphers": "^2.2.0", "@noble/hashes": "^2.2.0", "better-call": "1.4.4", "defu": "^6.1.4", "jose": "^6.2.3", "kysely": "^0.28.17 || ^0.29.0", "nanostores": "^1.3.0", "zod": "^4.5.4" }, "peerDependencies": { "@lynx-js/react": "*", "@prisma/client": "^5.0.0 || ^6.0.0 || ^7.0.0", "@sveltejs/kit": "^2.0.0 || ^3.0.0", "@tanstack/react-start": "^1.0.0", "@tanstack/solid-start": "^1.0.0", "drizzle-kit": ">=0.31.4 || >=1.0.0-beta.1", "drizzle-orm": "^0.45.2 || >=1.0.0-rc.1 <2.0.0", "mongodb": "^6.0.0 || ^7.0.0", "mysql2": "^3.0.0", "next": "^14.0.0 || ^15.0.0 || ^16.0.0", "pg": "^8.0.0", "prisma": "^5.0.0 || ^6.0.0 || ^7.0.0", "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0", "solid-js": "^1.0.0", "svelte": "^4.0.0 || ^5.0.0", "vitest": "^2.0.0 || ^3.0.0 || ^4.0.0 || ^5.0.0", "vue": "^3.0.0" }, "optionalPeers": ["@lynx-js/react", "@prisma/client", "@sveltejs/kit", "@tanstack/react-start", "@tanstack/solid-start", "drizzle-kit", "drizzle-orm", "mongodb", "mysql2", "next", "pg", "prisma", "react", "react-dom", "solid-js", "svelte", "vitest", "vue"] }, "sha512-QxDCI0pOCCHDYmruisKhKyrvOJ6Icv9xg0RA6ZUysfKFXENYwhNNpwUlkE/NmViH2nDMgkKcymmLr55pnpR9vw=="], + + "better-call": ["better-call@1.4.4", "", { "dependencies": { "@better-auth/utils": "^0.5.0", "@better-fetch/fetch": "^1.3.1", "rou3": "^0.9.1", "set-cookie-parser": "^3.1.2" }, "peerDependencies": { "zod": "^4.0.0" }, "optionalPeers": ["zod"] }, "sha512-Gq4XdGFy4dmUxFEAX2nj0ycFcFbD9ZWh8YROouZoyXeF1M11TpYbplk0pDByTFF8Wz+MYa6W9052c9m7dD+mfw=="], + "bun-types": ["bun-types@1.4.4", "", { "dependencies": { "@types/node": "*", "undici-types": "*" } }, "sha512-yjrJh5oGX7mMPKGd19YXyIjmEuj4G4zNC30bs7PM9Az7NQ3ndF3j6hbeRdvJTRQEhJ/FYQwW0a2moweEi2hMxQ=="], "csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="], + "defu": ["defu@6.1.7", "", {}, "sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ=="], + + "dompurify": ["dompurify@3.4.17", "", { "optionalDependencies": { "@types/trusted-types": "^2.0.7" } }, "sha512-B/pSlYVHQ58kqISD/yDjuJ1yfzI34m1gF4lAod2bwSRhiN4K5Cl3H8Af7hqB2PmTL911tdEudnnXn5FnhRoz0w=="], + + "escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="], + + "fast-xml-builder": ["fast-xml-builder@1.3.1", "", { "dependencies": { "path-expression-matcher": "^1.6.2", "xml-naming": "^0.3.0" } }, "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug=="], + + "fast-xml-parser": ["fast-xml-parser@5.11.2", "", { "dependencies": { "@nodable/entities": "^3.0.1", "fast-xml-builder": "^1.2.0", "is-unsafe": "^2.0.0", "path-expression-matcher": "^1.6.2", "strnum": "^2.4.2", "xml-naming": "^0.3.0" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-R9iDuNrQYeQut46cn2r2wHKn4HYzVDvDm5J1wW+koZewykv0yuO2HChTYeZtrULyHIDM9cj9TUXloCsueCQUog=="], + + "is-unsafe": ["is-unsafe@2.0.2", "", {}, "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ=="], + + "jose": ["jose@6.2.14", "", {}, "sha512-3JasLuki1NRLQvY4wCoC37KZk4qxoBx2k2aFqLxT84IezqW0bkJKnNtxnZj/Fu1xsQivoc/lytxaRHJoGLjJ1A=="], + + "kysely": ["kysely@0.29.7", "", {}, "sha512-fmIq+7kv4WbFm2tcfznRzikqB4cuNpgZ40PSU2fAJ2/OO5XhojqnTPrmgrCjYSRJVwBln03XcRXJGnw9H099IQ=="], + + "marked": ["marked@18.1.2", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-DfOnvLAQ6WYeWxWO1u4RyFOeqyHup6dks5PPqnoFiooU7ipRvidO/2j8plhAHIiZg574E9ZlzSjUiPMLAzFkgg=="], + + "nanostores": ["nanostores@1.5.5", "", {}, "sha512-FixtE239Gl6Gz0ZGCK0WzvNwObNCIwVT+L7zQcf1QgcA6AzBxblzqe9OT88qiucNkhsjSOqiogSaF5jVbJiqSA=="], + + "path-expression-matcher": ["path-expression-matcher@1.6.2", "", {}, "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ=="], + + "pdfjs-dist": ["pdfjs-dist@6.4.299", "", { "optionalDependencies": { "@napi-rs/canvas": "^1.0.10" } }, "sha512-AVl138zALtfaAPvADulE0PZThbYzCBS79nL4pOSL/6Sm/4AH5A21BD9VHt97OlCuzJuCpmeZtAtkinisF4Vb1g=="], + "react": ["react@19.3.1", "", {}, "sha512-QNhWdt7nrgVWWQG58QuGsAR3jv3Hbe4mb4vz7w2csnHavciZJgda1CPNaldF6eVxv90ud3jUXpKlHL69gO38yw=="], "react-dom": ["react-dom@19.3.1", "", { "dependencies": { "scheduler": "^0.28.0" }, "peerDependencies": { "react": "^19.3.1" } }, "sha512-Y0bARz53lhEC1NMCASs0/j+faB/zqt3R8hpvg1e8+61ZVJ4UL0UdC4cBbXfZMcIuhrfgeILTLe+ZeBCavQPtKQ=="], @@ -116,14 +213,42 @@ "react-router-dom": ["react-router-dom@6.30.6", "", { "dependencies": { "@remix-run/router": "1.23.4", "react-router": "6.30.6" }, "peerDependencies": { "react": ">=16.8", "react-dom": ">=16.8" } }, "sha512-0RHKZz7wwffvkU+2MFVT2NnjK44ssLEV+m0CAJaS2Ksmorrwj7WxH00jO0SOCW26/tINUnJHToXblDs33I38YQ=="], + "rou3": ["rou3@0.9.2", "", {}, "sha512-3SOzvaAg8rkHrXtRjpCvCvbyO5to9oOO27Z/XqHEYXfMRVSw/qMIVdmaOk9W2lcRLtR6dlqTjo9hDeJk70QBYQ=="], + + "samlify": ["samlify@2.15.1", "", { "dependencies": { "@authenio/xml-encryption": "^6.0.2", "@xmldom/xmldom": "^0.9.12", "xml": "^1.0.1", "xml-crypto": "^6.4.0", "xml-escape": "^1.1.0", "xpath": "^0.0.35" } }, "sha512-peZitMO2cAnf0pY45H0yw0Drqmby7U91yRovH3AH8TdwYCNRnmidVORXIjcK+21cBk3Mjl8EVz8Nvuw/v20rAw=="], + "scheduler": ["scheduler@0.28.0", "", {}, "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw=="], + "set-cookie-parser": ["set-cookie-parser@3.2.1", "", {}, "sha512-Npd1Pt+4zb3BrSZDPd2qFwRpS6b1EUEasRrO8xS0zMJi0fApXXK5YuzP0minyJnxwnFHoVoJwA55q0A0cu63gQ=="], + + "strnum": ["strnum@2.5.1", "", { "dependencies": { "anynum": "^1.0.1" } }, "sha512-UM4nTLWzQiDBseVxUGxN0wJzIk0UU89Qi7pKXz9qISo+lhviJikTuikBsK/oFRlpvYHvSdz0TVoLyQ+sEpYwxA=="], + + "tldts": ["tldts@7.4.19", "", { "dependencies": { "tldts-core": "^7.4.19" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-48dH6aSpQnpbzDUc4vIQZ1VjB8tkK+e8c1yx05/SZK+rkssgjEOlxDd/yU8/a4vGYQ9vRgJr9dwMq9TJ0b3X2Q=="], + + "tldts-core": ["tldts-core@7.4.20", "", {}, "sha512-C630TWXZCZmLq0F2bW4QqwnaaxCQ/svpAqvJwnC4JvgEM5ohhBnGsEzTLSOXcN+xjz+kEaHS5uoWIw2klnqd0A=="], + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], "undici-types": ["undici-types@8.12.3", "", {}, "sha512-EXE4ydL3S76dFb27UFjNVZHTyv/TLdATj/KUe29ptdcg6qXL48J2tZ3126E5kICj5jn781OR+XlVH1N6Mz8jbA=="], + "xml": ["xml@1.0.1", "", {}, "sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw=="], + + "xml-crypto": ["xml-crypto@6.4.0", "", { "dependencies": { "@xmldom/is-dom-node": "^1.0.1", "@xmldom/xmldom": "^0.8.15", "xpath": "^0.0.33" } }, "sha512-d21ModhBO/+QWWAmOjyDeQJuHNmsR/HtOxyBchiTbKfGNKuHYwQNd+SlLhwyyrwoAGLp4Qc6BQ7nAtnM47dHSQ=="], + + "xml-escape": ["xml-escape@1.1.0", "", {}, "sha512-B/T4sDK8Z6aUh/qNr7mjKAwwncIljFuUP+DO/D5hloYFj+90O88z8Wf7oSucZTHxBAsC1/CTP4rtx/x1Uf72Mg=="], + + "xml-naming": ["xml-naming@0.3.0", "", {}, "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ=="], + + "xpath": ["xpath@0.0.35", "", {}, "sha512-8O+m2N04SwLpGw/1Grz2HD8ejnWDV7M1dMPgKKuOjhYh5W9H4pel/+06kInohIdpS2FeSbL4i8ID8c4tpxwq9Q=="], + "zod": ["zod@4.6.7", "", {}, "sha512-g8W2D2kU6a8untc053fX1uVBKzM7yqAxW/3zwk83+6RyF2+SKOuV+c0wBDhUwzV1PaGapmZH+rmcUzPf+3CGwg=="], + "@authenio/xml-encryption/@xmldom/xmldom": ["@xmldom/xmldom@0.8.15", "", {}, "sha512-/5NV/vDALVFDXgLmfsy9TRCBlKwO2LNBFzpzvb9iIj+jR+eSc6DLYYvVOdivT/jm7MtU6TebYuRmzEOI7w40UA=="], + "@types/node/undici-types": ["undici-types@8.9.0", "", {}, "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg=="], + + "xml-crypto/@xmldom/xmldom": ["@xmldom/xmldom@0.8.15", "", {}, "sha512-/5NV/vDALVFDXgLmfsy9TRCBlKwO2LNBFzpzvb9iIj+jR+eSc6DLYYvVOdivT/jm7MtU6TebYuRmzEOI7w40UA=="], + + "xml-crypto/xpath": ["xpath@0.0.33", "", {}, "sha512-NNXnzrkDrAzalLhIUc01jO2mOzXGXh1JwPgkihcLLzw98c0WgYDmmjSh1Kl3wzaxSVWMuA+fe0WTWOBDWCBmNA=="], } } diff --git a/docs/superpowers/plans/2026-10-30-learner.md b/docs/superpowers/plans/2026-10-30-learner.md new file mode 100644 index 0000000..f5c82be --- /dev/null +++ b/docs/superpowers/plans/2026-10-30-learner.md @@ -0,0 +1,136 @@ +# WP-A Learner App Implementation Plan + +> **For agentic workers:** executed inline by the WP-A agent (no subagents), task by task, TDD, one commit per task. Steps use checkbox (`- [ ]`) syntax. + +**Goal:** Build `packages/feature-learner`: sign-in (Better Auth), roles/groups admin API, the library, the lesson reader for every lesson type, notes, progress, gating, and the in-browser JS/TS exercise runner. It must render `sample`, `pure-snake` and `systemic-design`. + +**Architecture:** +- Server side (`src/server/*`): `learnerFeature(ctx): Feature & { auth: AuthService }` returns the JSON API routes and the `AuthService`. It runs its own migrations (`migrations/0001_learner_auth.sql`) lazily on first use until Phase 2 wires `migrationsDirs` into `migrate()`. +- Auth: Better Auth with a **thin custom adapter over `Bun.SQL`** (`src/server/auth/adapter.ts`). One adapter serves both dialects because `@stc/db` already gives a `Bun.SQL` handle for SQLite and Postgres. +- Web side (`src/web/*`): React routes `/`, `/c/:courseId`, `/c/:courseId/:lessonId`, `/c/:courseId/notes`, `/account`, `/account/admin`. Styles are plain CSS loaded as text and injected once, so the same module graph is safe to import on the server. +- `LessonView` is a presentational lesson renderer that takes a `LessonPayload`. The studio uses it for preview. +- The browser runner (`src/web/runner/*`) runs learner code and tests as ES modules from blob URLs inside a module Web Worker, with a hard timeout (the worker is terminated). TS is transpiled by the server's `Bun.Transpiler` (`POST /api/learner/transpile`). +- Decks render client-side with pdfjs-dist. The worker script is served from `/api/learner/pdf.worker.mjs`. + +**Tech Stack:** Bun 1.3, TypeScript, React 19, React Router 6, better-auth 1.7 (+ `@better-auth/sso` for SAML/OIDC SSO, `generic-oauth` for a configured OIDC provider), marked + DOMPurify, pdfjs-dist. + +**Spec:** sections 4.1–4.4, 6, 7, 12, 13. Visual reference: `docs/design/mockups/platform-v1.html` minus its `[` / `]` / `/` / Ctrl+Enter shortcuts. + +## Global Constraints + +Inherited from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun only, plain CSS, no course content in the DB, both dialects, AGPL, tokens, keyboard rules, commit trailer). Package-specific: +- Owned paths: `packages/feature-learner/**`, `docs/superpowers/plans/2026-10-30-learner.md`, `docs/contract-requests/learner.md`. Shared touches: one line in `apps/web/src/features.ts`, the feature wiring in `apps/server/src/main.ts`. +- Frozen: `packages/contracts`, `packages/course-format`, `packages/db` (src + 0001), `packages/core`, `packages/ui`. +- No new colours: course accent (manifest) and the tokens only. Literata only for lesson titles and reading text. + +## Review Focus + +1. **Auth adapter correctness on both dialects** (where operators, AND/OR grouping, dates and booleans as text/int). Test: Task 2 runs the adapter suite under `forEachDb`. +2. **Owner bootstrap is race-safe**: exactly one owner even with concurrent sign-ups (unique key claim). Test: Task 3. +3. **Visibility**: lesson, outline, code and zip routes all apply `canView` and return 404 for hidden courses; gating returns `locked` without content. Test: Task 5/7. +4. **Zip download** handles spaces and non-ASCII names (UTF-8 flag) and never follows symlinks out of the root. Test: Task 7. +5. **Runner isolation**: learner code runs only inside the worker; an infinite loop is killed by the timeout. Test: Task 8. + +--- + +### Task 1: Package scaffold, migrations runner, test helper + +**Files:** `packages/feature-learner/{package.json,src/index.ts,src/server/migrate.ts,migrations/0001_learner_auth.sql,test/helpers.ts,test/migrate.test.ts}` + +**Interfaces (Produces):** +- `migrationsDirs: string[]` (absolute path of `packages/feature-learner/migrations`). +- `runMigrations(db: Db, dirs?: string[]): Promise` — same bookkeeping table and semantics as `@stc/db` `migrate()`. +- Tables: `auth_user`, `auth_session`, `auth_account`, `auth_verification`, `auth_sso_provider` (Better Auth, snake_case columns, ISO text dates, integer booleans), `learner_meta (key PK, value)`, `learner_group_claims (user_id, group_id)`. +- test helper `forEachDb(name, fn)` = `@stc/db`'s helper plus the learner migrations. + +- [ ] Step 1: failing test: after `runMigrations` the tables exist on both dialects and a second run applies nothing. +- [ ] Step 2: implement; Step 3: green on SQLite and Postgres; commit. + +### Task 2: Better Auth adapter over Bun.SQL + +**Files:** `src/server/auth/adapter.ts`, `test/adapter.test.ts` + +**Interfaces:** `bunSqlAdapter(db: Db)` → Better Auth `DBAdapterInstance`. Model `x` → table `auth_`, field `camelCase` → `snake_case`. `supportsDates/Booleans/JSON: false`, no transactions (SQLite has one connection; a nested query in a transaction would deadlock). +- [ ] Failing tests (both dialects): create/findOne/findMany (eq, ne, in, not_in, contains, starts_with, ends_with, lt/gt, insensitive mode, OR grouping, sort, limit, offset), update, updateMany, delete, deleteMany, count, consumeOne, incrementOne. +- [ ] Implement, green, commit. + +### Task 3: AuthService, owner bootstrap, claim groups, feature wiring + +**Files:** `src/server/auth/index.ts`, `src/server/auth/groups.ts`, `src/server/feature.ts`, `test/auth.test.ts`, `apps/server/src/main.ts` (wiring), `docs/contract-requests/learner.md` + +**Interfaces:** +- `createAuth(ctx, opts?)` → `{ handler(req), service: AuthService, providers(): ProviderInfo[] }`. +- Email/password always on; social providers (GitHub, Google, Microsoft, GitLab, Discord) from `STC_AUTH__ID/SECRET`; one generic OIDC provider from `STC_AUTH_OIDC_*`; SAML and per-domain OIDC through `@better-auth/sso` (owner-only registration). +- First user to sign up becomes `owner` (claimed via `INSERT` into `learner_meta` key `owner`, PK conflict loses); others get `learner`. +- `syncClaimGroups(db, userId, names)` creates groups by name and replaces the user's claim-sourced memberships; manual memberships are untouched. +- `learnerFeature(ctx)` returns `Feature & { auth }`; `startServer` accepts feature factories `(ctx) => Feature` and installs a feature's `auth` (contract request filed; two-line change in `main.ts`). +- [ ] Failing tests: sign-up → session cookie → `userFromRequest` returns roles `[owner]`; second user `[learner]`; concurrent first sign-ups give one owner; sign-out clears; claim sync keeps manual memberships. +- [ ] Implement, green, commit. + +### Task 4: Admin API (roles, groups, course ACL, library curation) + +**Files:** `src/server/admin.ts`, `test/admin.test.ts` +- Routes (owner only, 403 otherwise): `GET /api/admin/users`, `PUT /api/admin/users/:id/roles`, `GET|POST /api/admin/groups`, `DELETE /api/admin/groups/:id`, `PUT /api/admin/groups/:id/members`, `GET|PUT /api/admin/courses/:id/acl`, `GET|PUT /api/admin/library` (featured + collections). +- The last owner cannot drop their own owner role. +- [ ] Tests first; implement; commit. + +### Task 5: Library, outline and lesson API + +**Files:** `src/server/catalog.ts`, `src/server/lesson.ts`, `src/server/gating.ts`, `test/catalog.test.ts`, `test/lesson.test.ts` +- `GET /api/library` → `{ featured, continue[], newest[], collections[], courses[] }` filtered by `canView`. Each course: id, title, pitch, authors, coverUrl, accent, formats (`text|video|slides|exercises`), lessonCount, minutes (video duration, text estimated at 220 wpm, exercises 15 min), collections, progress, status. +- `GET /api/library/search?q=` → courses and lessons matching title/pitch/lesson titles. +- `GET /api/courses/:id` → outline, tracks, gating, `locked` per module, progress, diagnostics (errors only for owner/authors). +- `GET /api/courses/:id/lessons/:lessonId` → `LessonPayload` (markdown, hints, decks, video, transcript, code, attachments, browser exercise, local command, prev/next) or `{ locked: true }`. +- Gating: module *i* locked when any lesson of an earlier module is not done; owners and course authors bypass. +- [ ] Tests against the sample fixture plus temp courses; implement; commit. + +### Task 6: Progress, notes, exercise attempts API + +**Files:** `src/server/userdata.ts`, `test/userdata.test.ts` +- `PUT /api/courses/:id/progress/:lessonId` `{state, positionSeconds?}`; notes CRUD (`GET/POST /api/courses/:id/notes`, `PATCH/DELETE /api/notes/:id`); attempts `GET/PUT /api/courses/:id/lessons/:lessonId/attempt?track=`. A passing attempt marks the lesson done. Anonymous → 401. Hidden course → 404. +- [ ] Tests on both dialects (acceptance: notes persist on SQLite and Postgres); implement; commit. + +### Task 7: Code files and zip download + +**Files:** `src/server/zip.ts`, `src/server/code.ts`, `test/zip.test.ts`, `test/code.test.ts` +- `GET /api/courses/:id/code/:moduleId` → `{ dir, command, files: {path,size}[] }` (skips `.git`, `node_modules`, symlinks escaping root; caps 2000 files). +- `GET /api/courses/:id/code/:moduleId/zip` → `application/zip`, `Content-Disposition` via `contentDisposition(".zip")`. +- `zip(entries)`: store/deflate, CRC32, UTF-8 name flag. +- [ ] Tests: zip round-trip via `unzip -l`, non-ASCII names, symlink escape skipped; commit. + +### Task 8: Browser exercise runner + +**Files:** `src/web/runner/{assert.ts,worker-source.ts,runner.ts}`, `src/server/transpile.ts`, `test/runner.test.ts` +- `createBrowserRunner(language, { transpile })`: `ExerciseRunner`. Worker disables `fetch`, `XMLHttpRequest`, `WebSocket`, `importScripts` before importing code. Results carry expected/actual for failed asserts. +- [ ] Tests (Bun supports module workers and blob URLs): starter fails, solution passes, a syntax error is reported, `while(true)` times out, TS is transpiled. +- [ ] commit. + +### Task 9: Markdown pipeline + +**Files:** `src/web/markdown.ts`, `test/markdown.test.ts` +- `renderLessonHtml(md, { courseId, baseFile, tracks, lessonFiles })` → sanitized HTML: heading ids, code blocks tagged with their track (`data-t`), bold track labels tagged, `
      ` turned into folds; consecutive hint folds become a tier group (`data-tier`, `data-tiers`), relative images/links rewritten to `/api/media/...` or lesson routes. +- [ ] Tests with Pure Snake-style input; commit. + +### Task 10: Web shell and library + +**Files:** `src/web/{routes.tsx,api.ts,styles.css,shell/*,library/*}` +- Header extras (course switcher, search, Reading/Work, account) portalled into the Phase 0 header; course outline sidebar (mobile overlay, Escape closes); library: featured hero, continue row, New row, collection rows of covers, format/length/in-progress filters, search. +- [ ] Component smoke tests with `react-dom/server` render; screenshot check later; commit. + +### Task 11: Lesson views and `LessonView` + +**Files:** `src/web/lesson/*` +- Text (track switcher, Reading/Work, folds, hint tiers), video (range-served `
      • ); } diff --git a/packages/feature-import/src/web/import.css b/packages/feature-import/src/web/import.css index 4fc8d78..6dfc535 100644 --- a/packages/feature-import/src/web/import.css +++ b/packages/feature-import/src/web/import.css @@ -26,6 +26,7 @@ .imp-status[data-s="running"] { color: var(--accent); } .imp-job-bar { display: flex; align-items: center; gap: 12px; } .imp-job-msg { font-size: 12.5px; color: var(--faint); } +.imp-job-bar .imp-job-msg { flex: none; width: 200px; text-align: right; } .imp-job details { font-size: 12.5px; color: var(--muted); } .imp-job summary { cursor: pointer; } .imp-job ul.warn { margin: 6px 0 0; padding-left: 18px; } From 3cec42bba5ad6dc50ed0a22a64610d727c709328 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 03:22:38 +0100 Subject: [PATCH 30/62] feat(ai): house-style generator writing drafts through a DraftSink Co-Authored-By: Claude Opus 5.5 --- .../feature-ai/src/generator/draft-sink.ts | 46 ++++ packages/feature-ai/src/generator/generate.ts | 169 ++++++++++++++ .../feature-ai/src/generator/templates.ts | 209 ++++++++++++++++++ packages/feature-ai/src/index.ts | 3 + packages/feature-ai/test/generator.test.ts | 90 ++++++++ 5 files changed, 517 insertions(+) create mode 100644 packages/feature-ai/src/generator/draft-sink.ts create mode 100644 packages/feature-ai/src/generator/generate.ts create mode 100644 packages/feature-ai/src/generator/templates.ts create mode 100644 packages/feature-ai/test/generator.test.ts diff --git a/packages/feature-ai/src/generator/draft-sink.ts b/packages/feature-ai/src/generator/draft-sink.ts new file mode 100644 index 0000000..720c811 --- /dev/null +++ b/packages/feature-ai/src/generator/draft-sink.ts @@ -0,0 +1,46 @@ +import { mkdir } from "node:fs/promises"; +import { dirname, join, resolve, sep } from "node:path"; + +export interface DraftFile { path: string; content: string } + +/** + * Where generated files go. Same signature as the studio's `createDraftWriter(ctx).writeDraftFiles`, which + * commits onto the author's draft branch; Phase 2 passes that in. Drafts are never published by the AI package. + */ +export interface DraftSink { + writeDraftFiles(courseId: string, authorId: string, files: DraftFile[], message: string): Promise<{ location: string; paths: string[] }>; +} + +export function safeRelPath(p: string): string | null { + const clean = p.trim().replace(/\\/g, "/").replace(/^\.\//, ""); + if (!clean || clean.startsWith("/") || /^[a-zA-Z]:/.test(clean)) return null; + const parts = clean.split("/"); + if (parts.some((s) => s === ".." || s === "" || s === "." || s === ".git")) return null; + return clean; +} + +/** Filesystem fallback: `/drafts///…` plus DRAFT.json. Outside every course folder. */ +export function createFsDraftSink(dataDir: string): DraftSink { + return { + async writeDraftFiles(courseId, authorId, files, message) { + if (!safeRelPath(courseId) || courseId.includes("/")) throw new Error("Invalid course id"); + const runId = `${new Date().toISOString().replace(/[:.]/g, "-")}-${Bun.randomUUIDv7().slice(-8)}`; + const location = resolve(dataDir, "drafts", courseId, runId); + const paths: string[] = []; + for (const f of files) { + const rel = safeRelPath(f.path); + if (!rel) throw new Error(`Unsafe draft path: ${f.path}`); + const abs = resolve(location, rel); + if (!abs.startsWith(location + sep)) throw new Error(`Unsafe draft path: ${f.path}`); + await mkdir(dirname(abs), { recursive: true }); + await Bun.write(abs, f.content); + paths.push(rel); + } + await Bun.write( + join(location, "DRAFT.json"), + JSON.stringify({ courseId, authorId, message, createdAt: new Date().toISOString(), published: false, files: paths }, null, 2), + ); + return { location, paths }; + }, + }; +} diff --git a/packages/feature-ai/src/generator/generate.ts b/packages/feature-ai/src/generator/generate.ts new file mode 100644 index 0000000..c0da188 --- /dev/null +++ b/packages/feature-ai/src/generator/generate.ts @@ -0,0 +1,169 @@ +import { stat } from "node:fs/promises"; +import { join } from "node:path"; +import type { AiProvider } from "@stc/contracts"; +import { resolveInside, type CourseRepository, type LoadedCourse } from "@stc/course-format"; +import { safeRelPath, type DraftFile, type DraftSink } from "./draft-sink"; +import { OUTPUT_FORMAT, REQUIRED, VOICE, styleTemplate, type Kind, type Style } from "./templates"; + +export type GenerateSource = + | { kind: "lesson"; lessonId: string } + | { kind: "file"; path: string } + | { kind: "cache"; path: string } + | { kind: "text"; label: string; text: string }; + +export interface GenerateRequest { + courseId: string; + kind: Kind; + style: Style; + topic: string; + sources: GenerateSource[]; + instructions?: string; +} + +export interface GenerateResult { location: string; paths: string[]; warnings: string[] } + +const MAX_MATERIAL = 400_000; + +export function slugify(s: string): string { + return s.normalize("NFKD").replace(/[̀-ͯ]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60) || "draft"; +} + +export function targetPaths(req: GenerateRequest): string[] { + const slug = slugify(req.topic); + if (req.kind === "exercise") return [`exercises/${slug}/README.md`, `exercises/${slug}/starter.js`, `exercises/${slug}/tests.js`, `exercises/${slug}/solution.js`, `exercises/${slug}/fixtures/${slug}.json`, `exercises/${slug}/manifest-snippet.yaml`]; + if (req.kind === "questions") return [`questions/${slug}.md`]; + return req.style === "systemic-design" ? [`course/${slug}/lesson.md`, `course/${slug}/scenarios.md`] : [`course/${slug}.md`]; +} + +/** Reads the chosen inputs. Authors may use any course file, including solutions; cache text comes from /cache//. */ +export async function gatherMaterial(repo: CourseRepository, course: LoadedCourse, sources: GenerateSource[], dataDir: string): Promise<{ text: string; warnings: string[] }> { + const parts: string[] = []; + const warnings: string[] = []; + let total = 0; + const push = (label: string, body: string) => { + if (total >= MAX_MATERIAL) { warnings.push(`Left out "${label}": the material is already ${MAX_MATERIAL} characters.`); return; } + let text = body; + if (total + text.length > MAX_MATERIAL) { + text = text.slice(0, MAX_MATERIAL - total); + warnings.push(`"${label}" was shortened to fit the material limit.`); + } + total += text.length; + parts.push(`\n${text}\n`); + }; + const readText = async (abs: string | null, label: string) => { + const info = abs ? await stat(abs).catch(() => null) : null; + if (!abs || !info?.isFile()) { warnings.push(`Source not found: ${label}`); return; } + const text = await Bun.file(abs).text(); + if (text.includes("\u0000")) { warnings.push(`Skipped binary file: ${label}`); return; } + push(label, text); + }; + for (const s of sources) { + if (s.kind === "text") push(s.label || "Pasted text", s.text); + else if (s.kind === "lesson") { + const e = course.lessonIndex.get(s.lessonId); + const md = e ? await repo.readLessonMarkdown(course.id, s.lessonId) : null; + if (!e || md === null) warnings.push(`Lesson has no readable text: ${s.lessonId}`); + else push(`${e.lesson.type === "video" ? e.lesson.transcript : e.lesson.file} (lesson ${s.lessonId}: ${e.lesson.title})`, md); + } else if (s.kind === "file") await readText(repo.resolvePath(course.id, s.path), s.path); + else await readText(resolveInside(join(dataDir, "cache", course.id), s.path), `cache/${s.path}`); + } + return { text: parts.join("\n\n"), warnings }; +} + +function outline(course: LoadedCourse): string { + return (course.manifest?.modules ?? []).map((m, i) => `${i + 1}. ${m.title} (${m.lessons.length} lessons)`).join("\n"); +} + +export function buildGeneratorPrompt(req: GenerateRequest, material: string, course: LoadedCourse): { system: string; user: string } { + const what = req.kind === "document" ? "an explanatory lesson document" : req.kind === "exercise" ? "a small exercise with tests and hint tiers" : "check-your-understanding questions"; + const system = [ + `You draft course material for STCourses authors. You write ${what} for the course "${course.manifest?.title ?? course.id}", in the house style below, from the source material the author selected.`, + VOICE, + styleTemplate(req.kind, req.style), + OUTPUT_FORMAT, + ].join("\n\n"); + const tracks = course.manifest?.tracks?.map((t) => `${t.label} (${t.id})`).join(", ") || "one track (JavaScript)"; + const user = [ + `Course: ${course.manifest?.title ?? course.id}`, + `Language tracks: ${tracks}`, + `Existing modules:\n${outline(course) || "(none yet)"}`, + `Topic: ${req.topic}`, + req.instructions?.trim() ? `Author's instructions:\n${req.instructions.trim()}` : "", + `Write these files:\n${targetPaths(req).map((p) => `- ${p}`).join("\n")}`, + `Source material (transcripts, text, slide text and code chosen by the author):\n\n${material || "(none: work from the topic and instructions, and mark facts TODO(author): check)"}`, + ].filter(Boolean).join("\n\n"); + return { system, user }; +} + +/** Reads `=== FILE: path ===` … `=== END FILE ===` blocks. Unsafe paths are dropped with a warning. */ +export function parseFiles(text: string): { files: DraftFile[]; warnings: string[] } { + const files: DraftFile[] = []; + const warnings: string[] = []; + const lines = text.split(/\r?\n/); + let cur: { path: string; lines: string[] } | null = null; + const close = () => { + if (!cur) return; + const rel = safeRelPath(cur.path); + if (!rel) warnings.push(`Dropped a file with an unsafe path: ${cur.path}`); + else files.push({ path: rel, content: cur.lines.join("\n").replace(/\n*$/, "\n") }); + cur = null; + }; + for (const line of lines) { + const open = /^===\s*FILE:\s*(.+?)\s*===\s*$/.exec(line); + if (open) { close(); cur = { path: open[1]!.replace(/^`|`$/g, ""), lines: [] }; continue; } + if (/^===\s*END FILE\s*===\s*$/.test(line)) { close(); continue; } + cur?.lines.push(line); + } + if (cur) { warnings.push(`The last file (${(cur as { path: string }).path}) was not closed; it may be cut off.`); close(); } + if (!files.length) warnings.push("The model returned no files."); + return { files, warnings }; +} + +export function checkHouseStyle(req: GenerateRequest, files: DraftFile[]): string[] { + const warnings: string[] = []; + const md = files.filter((f) => /\.(md|markdown)$/i.test(f.path)).map((f) => f.content).join("\n"); + for (const s of REQUIRED[req.kind][req.style]) if (!md.includes(s)) warnings.push(`House style: missing "${s}".`); + if (req.kind === "exercise") { + for (const name of ["starter.js", "tests.js", "solution.js"]) if (!files.some((f) => f.path.endsWith(`/${name}`))) warnings.push(`Exercise: missing ${name}.`); + const tests = files.find((f) => f.path.endsWith("/tests.js")); + if (tests && !/export\s+const\s+tests\b/.test(tests.content)) warnings.push("Exercise: tests.js does not export `tests`."); + } + return warnings; +} + +const marker = (req: GenerateRequest) => + `\n\n`; + +/** Runs the model once and writes the files it returns through the DraftSink. Never publishes. */ +export async function generateDraft(o: { + provider: AiProvider; + repo: CourseRepository; + req: GenerateRequest; + sink: DraftSink; + authorId: string; + dataDir: string; + signal?: AbortSignal; + onProgress?: (fraction: number, message: string) => Promise | void; +}): Promise { + const course = o.repo.get(o.req.courseId); + if (!course || course.status !== "ok") throw new Error(`Course not available: ${o.req.courseId}`); + await o.onProgress?.(0.05, "Reading sources"); + const material = await gatherMaterial(o.repo, course, o.req.sources, o.dataDir); + const prompt = buildGeneratorPrompt(o.req, material.text, course); + await o.onProgress?.(0.15, "Drafting"); + let text = ""; + let chars = 0; + for await (const ev of o.provider.chat([{ role: "system", content: prompt.system }, { role: "user", content: prompt.user }], { signal: o.signal })) { + if (ev.text) { + text += ev.text; + chars += ev.text.length; + if (chars > 4000) { chars = 0; await o.onProgress?.(Math.min(0.85, 0.15 + text.length / 60_000), "Drafting"); } + } + } + const parsed = parseFiles(text); + if (!parsed.files.length) throw new Error(parsed.warnings.join(" ") || "The model returned no files."); + const files = parsed.files.map((f) => (/\.(md|markdown)$/i.test(f.path) ? { ...f, content: marker(o.req) + f.content } : f)); + await o.onProgress?.(0.9, "Writing drafts"); + const written = await o.sink.writeDraftFiles(course.id, o.authorId, files, `AI draft: ${o.req.kind} "${o.req.topic}" (${o.req.style} style)`); + return { ...written, warnings: [...material.warnings, ...parsed.warnings, ...checkHouseStyle(o.req, parsed.files)] }; +} diff --git a/packages/feature-ai/src/generator/templates.ts b/packages/feature-ai/src/generator/templates.ts new file mode 100644 index 0000000..79bb578 --- /dev/null +++ b/packages/feature-ai/src/generator/templates.ts @@ -0,0 +1,209 @@ +// House-style prompt templates for the studio generator, distilled from the two reference courses: +// - Pure Snake (functional-programming-course): module shape of course/03-pure-game-rules.md and MENTOR.md; +// - Systemic Design (systemic-design-course): the beats of course/01-handles/lesson.md and mentor/SKILL.md. + +export type Style = "pure-snake" | "systemic-design"; +export type Kind = "document" | "exercise" | "questions"; + +export const OUTPUT_FORMAT = `## Output format + +Reply with one or more files and nothing that matters outside them. Write each file exactly like this: + +=== FILE: === + +=== END FILE === + +Use only relative paths inside the course (no leading slash, no ".."). Everything you write is a draft for an author to review; it is never published automatically.`; + +export const VOICE = `## Voice and rules (both house styles) + +- Write for a learner working alone at home with general programming experience. Plain, direct sentences; second person ("you"). No hype, no emoji, no headings in all caps. +- Teach by making the learner think first: a concrete situation or puzzle before the explanation, and the explanation folded so they can try first. +- Every fact about a tool, library or real system must come from the source material you are given. If the material does not say it, leave it out or mark it "TODO(author): check". +- Code in fenced blocks with a language tag. Keep examples small. Exercise solutions never appear in learner-facing text; near-solution hints are skeletons with \`___\` holes. +- When the course has several language tracks, give each track's signature and use one shared fixture file for test cases, so every track tests the same cases with the same names.`; + +export const PURE_SNAKE_DOCUMENT = `## House style: Pure Snake module + +Follow this module shape exactly (headings verbatim, in this order): + +# Module N — + +> **Time:** ~<estimate> · **You'll edit:** <files per track> +> **Tests:** <test command per track> +> **Prerequisites:** <earlier modules> + +## Why this module +Two or three paragraphs: how this is usually done (stateful, the familiar way), what goes wrong, and what this module does instead. End by announcing the puzzles. + +## Learning objectives +"After this module you can:" then 4-6 bullets, each starting with a bold verb (**implement**, **explain**, **reason**, **predict**, **read**). + +## Concepts +### <one H3 per concept> +Explain with a tiny example on a *different* record than the exercise (say so: "(illustration on a different record)"), shown in each track. +Include at least one bridge: +> **From everyday programming:** <how the learner already meets this idea: loops, setters, callbacks, SQL, Git, unit tests, mocks> + +### Puzzle 1: <a question the learner can answer on paper> +A concrete small case (a 5 x 5 board, a list of three items). Then **Before reading on, decide:** <the question>. + +<details><summary>Think about it first, then open the explanation</summary> + +The reasoning, the answer, and the rule it reveals. Name the fixture cases that check it. + +</details> + +(One or two puzzles. Each one targets a subtlety implementations commonly get wrong.) + +## Exercises +### Exercise N.1 — \`functionName\` +The signature per track and a one-line example in a comment. "Fixture: \`fixtures/mNN-<function>.json\` (<case names>)." +(2-4 exercises, each small and testable.) + +### Run +The test command per track. + +## Hints +For EVERY exercise, three folds in this order, labelled exactly: + +<details><summary>Exercise N.1 (functionName) — nudge</summary> + +One question or pointer back to an earlier idea. + +</details> + +<details><summary>Exercise N.1 (functionName) — stronger hint</summary> + +The approach in words, no code. + +</details> + +<details><summary>Exercise N.1 (functionName) — near-solution (try the others first)</summary> + +A skeleton per track with \`___\` holes. Never the full answer. + +</details> + +## Check your understanding +6-8 numbered questions. Include: at least one "Predict:" question with concrete input, at least one "What would break if…", one "Explain to a friend, with a drawing…", and one design question ("Sketch a different model that…"). + +## In production +2-3 bullets linking the idea to real systems, each with a source from the material or marked TODO(author). + +## Further reading +3-5 bullets: "[Title](url) — why it is worth reading." Only links present in the material; otherwise "TODO(author): add reading".`; + +export const SYSTEMIC_DESIGN_DOCUMENT = `## House style: Systemic Design module + +Every module follows the same beats, as H2 headings verbatim and in this order: Observe, Predict, Spec, Build, Name it, Expansion Drop, Further reading. + +# Module N: <Title> +One paragraph: what the engine learns in this module and why later modules stand on it. + +## Observe +A concrete situation from a real game or system, with the exact rule text quoted and its source cited in a link. Distinguish what the source says from the course's own reading ("That example is the course's reading of the quoted rule"). End with the tension the module resolves. + +## Predict +"Write your answers down before you open the discussion, for example in \`notes/NN.md\` in your workspace." +2-3 numbered questions that make the learner commit to a design choice. + +<details><summary>Discussion</summary> + +Answer each question by number. Compare the tempting options and say what goes wrong with each, then name the answer. + +</details> + +## Spec +The contract: harness functions with signatures in each track, data formats, and how tests are named. Point to \`scenarios.md\`: each scenario heading is a test name, identical across tracks, with Setup / Action / Assertions bullets. + +## Build +Where code goes (generic machinery in the engine, named content in content files; the engine must not name specific items) and what not to build yet. + +<details><summary>Hint 1 — nudge</summary> + +A question about who owns the data or when something happens. + +</details> + +<details><summary>Hint 2 — concept</summary> + +The design in bolded terms (**Id**, **Entity record**, …), still no code. + +</details> + +<details><summary>Hint 3 — pseudo-code</summary> + +\`\`\`text +language-neutral pseudo-code, never one track's real code +\`\`\` + +</details> + +## Name it +Name the pattern, then show where it appears in real engines or systems with links from the material, and the one idea to keep. + +## Expansion Drop +New content arrives that the design must absorb without touching the engine: numbered steps (commit, copy the expansion test, implement in harness/content only, run the expansion check). State the test name and what it checks. + +<details><summary>Discussion: what this drop measures</summary> + +What a good design makes trivial here, and what a coupled design would have had to change. + +</details> + +## Further reading +Bullets with links from the material, each with one line on why. + +Also write \`scenarios.md\` next to the lesson: "# Module N scenarios: <Title>", one sentence on where the tests live, then one "### test_NN_<name>" heading per scenario with Setup / Action / Assertions bullets.`; + +export const EXERCISE = (style: Style) => `## House style: exercise + +Write one small exercise that runs in the STCourses browser runner, plus its hint tiers. + +Files (all under the exercise folder you are given): +- \`README.md\`: the task in the course's voice: one paragraph of context, the signature, two worked examples (input → output), and what "done" means. Then the hints as three folds per exercise, labelled ${style === "systemic-design" ? '"Hint 1 — nudge", "Hint 2 — concept", "Hint 3 — pseudo-code"' : '"Exercise N.M (name) — nudge", "… — stronger hint", "… — near-solution (try the others first)"'}. The near-solution tier is a skeleton with \`___\` holes (stronger hint: the approach in words; nudge: one question). The tiers in plain words are always nudge, stronger hint, near-solution. +- \`starter.js\`: an ES module exporting the function(s) with the right signature and a body that returns a wrong but type-correct value (or throws "TODO"). A doc comment states the contract. +- \`tests.js\`: an ES module \`export const tests = [ { name: "…", run(mod, assert) { assert.equal(mod.fn(…), …) } }, … ]\`. Use only \`assert.equal\`, \`assert.deepEqual\` and \`assert.ok\`. 5-8 tests including edge cases; test names read like fixture case names (\`sum-of-empty-list-is-zero\`). No imports: the runner loads this one module. +- \`solution.js\`: the reference solution (authors only; learners never see it). +- \`fixtures/<slug>.json\`: the same test cases as data (\`[{ "name", "input", "expected" }]\`), shared across tracks so other language tracks test identical cases. +- \`manifest-snippet.yaml\`: the lesson entry an author would paste into the manifest (type: exercise, browser: { language: js, starter, tests, solution }, hints: { file, match }).`; + +export const QUESTIONS = `## House style: knowledge check + +Write a "## Check your understanding" section as a numbered list of 6-8 questions about the material, in the reference courses' style: +- at least one "Predict:" question with concrete input the learner works out on paper; +- at least one "What would break if…" question; +- one "Explain to a friend, with a drawing…" question; +- one design question ("Sketch a different model that…"); +- the rest test the main idea, not trivia. No multiple choice. + +After the list, add one fold per question for authors: + +<details><summary>Answer notes for question N (authors)</summary> + +What a good answer contains and the misconception it probes. + +</details>`; + +export function styleTemplate(kind: Kind, style: Style): string { + if (kind === "exercise") return EXERCISE(style); + if (kind === "questions") return QUESTIONS; + return style === "systemic-design" ? SYSTEMIC_DESIGN_DOCUMENT : PURE_SNAKE_DOCUMENT; +} + +/** Headings and labels the house-style check looks for. */ +export const REQUIRED: Record<Kind, Record<Style, string[]>> = { + document: { + "pure-snake": ["## Why this module", "## Learning objectives", "## Concepts", "Think about it first", "## Exercises", "## Hints", "— nudge", "— stronger hint", "— near-solution", "## Check your understanding", "## Further reading"], + "systemic-design": ["## Observe", "## Predict", "## Spec", "## Build", "Hint 1 — nudge", "Hint 2 — concept", "Hint 3 — pseudo-code", "## Name it", "## Expansion Drop", "## Further reading"], + }, + exercise: { + "pure-snake": ["nudge", "stronger hint", "near-solution"], + "systemic-design": ["Hint 1 — nudge", "Hint 2 — concept", "Hint 3 — pseudo-code"], + }, + questions: { + "pure-snake": ["## Check your understanding"], + "systemic-design": ["## Check your understanding"], + }, +}; diff --git a/packages/feature-ai/src/index.ts b/packages/feature-ai/src/index.ts index 1fa0c30..75a46aa 100644 --- a/packages/feature-ai/src/index.ts +++ b/packages/feature-ai/src/index.ts @@ -10,3 +10,6 @@ export * from "./mcp/client"; export * from "./assistant/prompt"; export * from "./assistant/tools"; export * from "./assistant/loop"; +export * from "./generator/draft-sink"; +export * from "./generator/generate"; +export * from "./generator/templates"; diff --git a/packages/feature-ai/test/generator.test.ts b/packages/feature-ai/test/generator.test.ts new file mode 100644 index 0000000..394a39e --- /dev/null +++ b/packages/feature-ai/test/generator.test.ts @@ -0,0 +1,90 @@ +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createCourseRepository, type CourseRepository } from "@stc/course-format"; +import { buildGeneratorPrompt, gatherMaterial, generateDraft, parseFiles, checkHouseStyle, type GenerateRequest } from "../src/generator/generate"; +import { createFsDraftSink } from "../src/generator/draft-sink"; +import { fakeProvider } from "./helpers"; + +const FIXTURES = join(import.meta.dir, "../../../fixtures/courses"); +let repo: CourseRepository; +let dataDir: string; +beforeAll(async () => { + repo = await createCourseRepository([FIXTURES]); + dataDir = mkdtempSync(join(tmpdir(), "stc-ai-gen-")); +}); +afterAll(() => rmSync(dataDir, { recursive: true, force: true })); + +const req = (over: Partial<GenerateRequest> = {}): GenerateRequest => ({ + courseId: "sample", kind: "document", style: "pure-snake", topic: "Adding many numbers", + sources: [{ kind: "lesson", lessonId: "m2-text" }, { kind: "text", label: "Author notes", text: "Learners confuse sum and count." }], ...over, +}); + +test("Pure Snake style prompt carries the module shape and hint tiers", async () => { + const course = repo.get("sample")!; + const material = await gatherMaterial(repo, course, req().sources, dataDir); + const p = buildGeneratorPrompt(req(), material.text, course); + for (const s of ["## Why this module", "## Learning objectives", "## Concepts", "Think about it first, then open the explanation", "From everyday programming", "— nudge", "— stronger hint", "— near-solution", "## Check your understanding", "## In production", "## Further reading", "=== FILE:"]) { + expect(p.system).toContain(s); + } + expect(p.user).toContain("Functions take values and return values."); + expect(p.user).toContain("Learners confuse sum and count."); + expect(p.user).toContain("course/adding-many-numbers.md"); +}); + +test("Systemic Design style prompt carries the beats", () => { + const course = repo.get("sample")!; + const p = buildGeneratorPrompt(req({ style: "systemic-design" }), "material", course); + const beats = ["## Observe", "## Predict", "## Spec", "## Build", "## Name it", "## Expansion Drop", "## Further reading"]; + let last = -1; + for (const b of beats) { const i = p.system.indexOf(b); expect(i).toBeGreaterThan(last); last = i; } + for (const s of ["Hint 1 — nudge", "Hint 2 — concept", "Hint 3 — pseudo-code", "what this drop measures", "scenarios.md"]) expect(p.system).toContain(s); +}); + +test("exercise and question prompts", () => { + const course = repo.get("sample")!; + const ex = buildGeneratorPrompt(req({ kind: "exercise" }), "m", course); + for (const s of ["starter.js", "tests.js", "solution.js", "export const tests", "fixtures/", "nudge", "stronger hint", "near-solution"]) expect(ex.system + ex.user).toContain(s); + const q = buildGeneratorPrompt(req({ kind: "questions" }), "m", course); + expect(q.system).toContain("Check your understanding"); + expect(q.system).toMatch(/predict/i); + expect(q.system).toMatch(/what would break if/i); +}); + +test("parseFiles reads file blocks and rejects unsafe paths", () => { + const out = parseFiles("intro\n=== FILE: course/a.md ===\n# A\n=== END FILE ===\n=== FILE: ../escape.md ===\nx\n=== END FILE ===\n=== FILE: /abs.md ===\ny\n=== END FILE ===\n=== FILE: exercises/b/starter.js ===\nexport const b = 1\n=== END FILE ==="); + expect(out.files).toEqual([{ path: "course/a.md", content: "# A\n" }, { path: "exercises/b/starter.js", content: "export const b = 1\n" }]); + expect(out.warnings.join("\n")).toMatch(/escape\.md/); + expect(out.warnings.join("\n")).toMatch(/abs\.md/); + expect(parseFiles("no files here").warnings[0]).toMatch(/no files/i); +}); + +test("house style check flags missing tiers", () => { + const w = checkHouseStyle(req(), [{ path: "course/x.md", content: "# M\n## Why this module\n" }]); + expect(w.some((x) => x.includes("Hints"))).toBe(true); + expect(w.some((x) => x.includes("near-solution"))).toBe(true); +}); + +test("generation writes drafts through the sink and never touches the course", async () => { + const before = readdirSync(join(FIXTURES, "sample"), { recursive: true }).length; + const doc = [ + "# Module 3 — Adding many numbers", "## Why this module", "x", "## Learning objectives", "## Concepts", "<details><summary>Think about it first, then open the explanation</summary>", "y", "</details>", + "## Exercises", "## Hints", "<details><summary>Exercise 3.1 (sum) — nudge</summary></details>", "<details><summary>Exercise 3.1 (sum) — stronger hint</summary></details>", + "<details><summary>Exercise 3.1 (sum) — near-solution (try the others first)</summary></details>", "## Check your understanding", "1. Predict…", "## In production", "## Further reading", + ].join("\n"); + const { provider, seen } = fakeProvider([() => ({ text: `Here you go.\n=== FILE: course/adding-many-numbers.md ===\n${doc}\n=== END FILE ===\n` })]); + const sink = createFsDraftSink(dataDir); + const r = await generateDraft({ provider, repo, req: req(), sink, authorId: "author-1", dataDir }); + expect(seen[0]!.tools).toBeUndefined(); + expect(r.paths).toEqual(["course/adding-many-numbers.md"]); + expect(r.location.startsWith(join(dataDir, "drafts", "sample"))).toBe(true); + expect(r.warnings).toEqual([]); + const written = readFileSync(join(r.location, "course/adding-many-numbers.md"), "utf8"); + expect(written.startsWith("<!-- Draft generated by STCourses AI")).toBe(true); + expect(written).toContain("## Why this module"); + const meta = JSON.parse(readFileSync(join(r.location, "DRAFT.json"), "utf8")); + expect(meta).toMatchObject({ courseId: "sample", authorId: "author-1", published: false }); + expect(readdirSync(join(FIXTURES, "sample"), { recursive: true }).length).toBe(before); + expect(existsSync(join(FIXTURES, "sample", "course"))).toBe(false); +}); From 5f85ce1d4b62f66ccde338335f4426324d68214d Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 03:23:30 +0100 Subject: [PATCH 31/62] feat(learner): library, search, course outline with gating, lesson payload API Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- packages/feature-learner/src/server/access.ts | 44 ++++ .../feature-learner/src/server/catalog.ts | 216 ++++++++++++++++++ .../feature-learner/src/server/feature.ts | 12 +- packages/feature-learner/src/server/gating.ts | 15 ++ packages/feature-learner/src/server/lesson.ts | 115 ++++++++++ packages/feature-learner/src/shared/types.ts | 121 ++++++++++ packages/feature-learner/test/catalog.test.ts | 61 +++++ packages/feature-learner/test/courses.ts | 48 ++++ packages/feature-learner/test/gating.test.ts | 21 ++ packages/feature-learner/test/lesson.test.ts | 47 ++++ 10 files changed, 699 insertions(+), 1 deletion(-) create mode 100644 packages/feature-learner/src/server/access.ts create mode 100644 packages/feature-learner/src/server/catalog.ts create mode 100644 packages/feature-learner/src/server/gating.ts create mode 100644 packages/feature-learner/src/server/lesson.ts create mode 100644 packages/feature-learner/src/shared/types.ts create mode 100644 packages/feature-learner/test/catalog.test.ts create mode 100644 packages/feature-learner/test/courses.ts create mode 100644 packages/feature-learner/test/gating.test.ts create mode 100644 packages/feature-learner/test/lesson.test.ts diff --git a/packages/feature-learner/src/server/access.ts b/packages/feature-learner/src/server/access.ts new file mode 100644 index 0000000..9ad4ea2 --- /dev/null +++ b/packages/feature-learner/src/server/access.ts @@ -0,0 +1,44 @@ +import type { AuthUser, ServerContext } from "@stc/contracts"; +import { canView } from "@stc/core"; +import type { LoadedCourse } from "@stc/course-format"; +import type { Db } from "@stc/db"; + +export interface AccessIndex { + acl: Map<string, { principal_type: string; principal_id: string }[]>; + authorOf: Set<string>; +} + +/** All ACL rows plus the courses the user co-authors, fetched once per request. */ +export async function accessIndex(db: Db, user: AuthUser | null): Promise<AccessIndex> { + const acl = new Map<string, { principal_type: string; principal_id: string }[]>(); + const authorOf = new Set<string>(); + if (!user) return { acl, authorOf }; + for (const r of await db.sql`SELECT course_id, principal_type, principal_id FROM course_acl`) { + const list = acl.get(r.course_id) ?? []; + list.push({ principal_type: r.principal_type, principal_id: r.principal_id }); + acl.set(r.course_id, list); + } + for (const r of await db.sql`SELECT course_id FROM course_authors WHERE user_id = ${user.id}`) authorOf.add(r.course_id); + return { acl, authorOf }; +} + +export const isOwner = (u: AuthUser | null): boolean => !!u?.roles.includes("owner"); + +export function canEdit(user: AuthUser | null, courseId: string, idx: AccessIndex): boolean { + return !!user && (user.roles.includes("owner") || user.roles.includes("author") || idx.authorOf.has(courseId)); +} + +/** Learners see courses with a manifest that `canView` allows; owners also see broken ones. */ +export function visible(user: AuthUser | null, c: LoadedCourse, idx: AccessIndex): boolean { + if (!c.manifest) return isOwner(user); + return canView(user, c, idx.acl.get(c.id) ?? [], canEdit(user, c.id, idx)); +} + +/** The course if this user may read it, else null (callers answer 404, never 403, so hidden courses stay hidden). */ +export async function courseFor(ctx: ServerContext, user: AuthUser | null, courseId: string) { + const course = ctx.courses.get(courseId); + if (!course || !course.manifest) return null; + const idx = await accessIndex(ctx.db, user); + if (!visible(user, course, idx)) return null; + return { course, manifest: course.manifest, idx, canEdit: canEdit(user, courseId, idx) }; +} diff --git a/packages/feature-learner/src/server/catalog.ts b/packages/feature-learner/src/server/catalog.ts new file mode 100644 index 0000000..e2395b7 --- /dev/null +++ b/packages/feature-learner/src/server/catalog.ts @@ -0,0 +1,216 @@ +import { statSync } from "node:fs"; +import type { AuthUser, Feature, ServerContext } from "@stc/contracts"; +import type { LoadedCourse, Manifest } from "@stc/course-format"; +import { libraryRepo, progressRepo } from "@stc/db"; +import type { + ContinueItem, CourseCard, CoursePayload, Format, LessonProgress, LibraryPayload, OutlineModule, SearchHit, +} from "../shared/types"; +import { mediaUrl } from "../shared/types"; +import { accessIndex, courseFor, isOwner, visible } from "./access"; +import { moduleLocks } from "./gating"; +import { json, notFound, param } from "./http"; + +const WPM = 220; +const EXERCISE_MINUTES = 15; + +/** Lesson length estimates (minutes), cached per course and manifest mtime. */ +export function createEstimator(ctx: ServerContext) { + const cache = new Map<string, { key: string; minutes: Map<string, number | null> }>(); + return async function minutesFor(c: LoadedCourse): Promise<Map<string, number | null>> { + let key = c.manifestPath; + try { key += `@${statSync(c.manifestPath).mtimeMs}`; } catch { /* keep path */ } + const hit = cache.get(c.id); + if (hit && hit.key === key) return hit.minutes; + const minutes = new Map<string, number | null>(); + for (const mod of c.manifest?.modules ?? []) { + for (const l of mod.lessons) { + if (l.type === "video") { + minutes.set(l.id, l.duration ? l.duration / 60 : null); + continue; + } + let words = 0; + try { + const md = await ctx.courses.readLessonMarkdown(c.id, l.id); + words = md ? md.split(/\s+/).filter(Boolean).length : 0; + } catch { /* unreadable: no estimate */ } + const read = words / WPM; + minutes.set(l.id, l.type === "exercise" ? Math.max(EXERCISE_MINUTES, read) : words ? Math.max(1, read) : null); + } + } + cache.set(c.id, { key, minutes }); + return minutes; + }; +} + +export function formatsOf(m: Manifest): Format[] { + const types = new Set(m.modules.flatMap((x) => x.lessons.map((l) => l.type))); + const f: Format[] = []; + if (types.has("text")) f.push("text"); + if (types.has("video")) f.push("video"); + if (m.decks?.length) f.push("slides"); + if (types.has("exercise")) f.push("exercises"); + return f; +} + +export function addedAt(c: LoadedCourse): string { + const imported = c.manifest?.source?.importedAt; + if (imported && !Number.isNaN(Date.parse(imported))) return new Date(imported).toISOString(); + try { return statSync(c.manifestPath).mtime.toISOString(); } catch { return new Date(0).toISOString(); } +} + +const slug = (s: string) => s.trim().toLowerCase().normalize("NFKD").replace(/[^\p{L}\p{N}]+/gu, "-").replace(/^-|-$/g, ""); +const fold = (s: string) => s.normalize("NFKD").replace(/\p{M}/gu, "").toLowerCase(); + +type ProgressByCourse = Map<string, Record<string, LessonProgress>>; + +async function allProgress(ctx: ServerContext, user: AuthUser | null): Promise<ProgressByCourse> { + const out: ProgressByCourse = new Map(); + if (!user) return out; + const rows = await ctx.db.sql`SELECT course_id, lesson_id, state, position_seconds, updated_at FROM progress WHERE user_id = ${user.id}`; + for (const r of rows) { + const rec = out.get(r.course_id) ?? {}; + rec[r.lesson_id] = { state: r.state, positionSeconds: r.position_seconds === null ? null : Number(r.position_seconds), updatedAt: r.updated_at }; + out.set(r.course_id, rec); + } + return out; +} + +export function libraryRoutes(ctx: ServerContext): Feature["routes"] { + const minutesFor = createEstimator(ctx); + const lib = libraryRepo(ctx.db); + + async function card(c: LoadedCourse, progress: Record<string, LessonProgress>): Promise<CourseCard> { + const m = c.manifest; + if (!m) { + return { + id: c.id, title: c.id, pitch: c.diagnostics[0]?.message ?? null, authors: [], coverUrl: null, accent: null, formats: [], + lessonCount: 0, minutes: 0, collections: [], tracks: [], source: null, addedAt: addedAt(c), status: "error", progress: 0, doneCount: 0, started: false, + }; + } + const lessons = m.modules.flatMap((x) => x.lessons); + const est = await minutesFor(c); + const minutes = Math.round(lessons.reduce((s, l) => s + (est.get(l.id) ?? 0), 0)); + const doneCount = lessons.filter((l) => progress[l.id]?.state === "done").length; + return { + id: m.id, title: m.title, pitch: m.pitch ?? null, authors: m.authors ?? [], + coverUrl: m.cover && ctx.courses.resolvePath(m.id, m.cover) ? mediaUrl(m.id, m.cover) : null, + accent: m.accent ?? null, formats: formatsOf(m), lessonCount: lessons.length, minutes, + collections: m.collections ?? [], tracks: (m.tracks ?? []).map((t) => t.label), source: m.source?.kind ?? null, + addedAt: addedAt(c), status: c.status, progress: lessons.length ? doneCount / lessons.length : 0, doneCount, + started: lessons.some((l) => progress[l.id]), + }; + } + + function continueItem(c: LoadedCourse, progress: Record<string, LessonProgress>, bypass: boolean): ContinueItem | null { + const m = c.manifest!; + const lessons = m.modules.flatMap((x) => x.lessons); + const done = lessons.filter((l) => progress[l.id]?.state === "done").length; + if (!lessons.some((l) => progress[l.id]) || done === lessons.length) return null; + const locks = moduleLocks(m, progress, bypass); + for (const [i, mod] of m.modules.entries()) { + if (locks[i]) break; + const next = mod.lessons.find((l) => progress[l.id]?.state !== "done"); + if (!next) continue; + return { + courseId: m.id, lessonId: next.id, lessonTitle: next.title, moduleTitle: mod.title, moduleIndex: i, + moduleDone: mod.lessons.filter((l) => progress[l.id]?.state === "done").length, moduleTotal: mod.lessons.length, + progress: done / lessons.length, + }; + } + return null; + } + + return { + "/api/library": { + GET: async (_req, rc) => { + const user = rc.user; + const idx = await accessIndex(ctx.db, user); + const courses = ctx.courses.list().filter((c) => visible(user, c, idx)); + const progress = await allProgress(ctx, user); + const cards = await Promise.all(courses.map((c) => card(c, progress.get(c.id) ?? {}))); + const byId = new Map(cards.map((c) => [c.id, c])); + + const cont: (ContinueItem & { at: string })[] = []; + for (const c of courses) { + if (!c.manifest) continue; + const p = progress.get(c.id) ?? {}; + const item = continueItem(c, p, isOwner(user)); + if (item) cont.push({ ...item, at: Object.values(p).map((x) => x.updatedAt).sort().pop() ?? "" }); + } + cont.sort((a, b) => (a.at < b.at ? 1 : -1)); + + const newest = [...cards].filter((c) => c.status === "ok" || c.lessonCount).sort((a, b) => (a.addedAt < b.addedAt ? 1 : a.addedAt > b.addedAt ? -1 : a.title.localeCompare(b.title))).map((c) => c.id); + + const collections: LibraryPayload["collections"] = []; + for (const col of await lib.listCollections()) { + collections.push({ id: col.id, title: col.title, courseIds: col.courseIds.filter((id) => byId.has(id)) }); + } + for (const c of cards) { + for (const name of c.collections) { + const id = slug(name); + let col = collections.find((x) => x.id === id || x.title.toLowerCase() === name.toLowerCase()); + if (!col) collections.push((col = { id, title: name, courseIds: [] })); + if (!col.courseIds.includes(c.id)) col.courseIds.push(c.id); + } + } + + const featuredId = await lib.getFeatured(); + const featured = (featuredId && byId.get(featuredId)) || (cont[0] && byId.get(cont[0].courseId)) || byId.get(newest[0] ?? "") || null; + const payload: LibraryPayload = { + signedIn: !!user, + featured, + continue: cont.map(({ at: _at, ...rest }) => rest), + newest, + collections: collections.filter((c) => c.courseIds.length > 0), + courses: cards.sort((a, b) => a.title.localeCompare(b.title)), + }; + return json(payload); + }, + }, + "/api/library/search": { + GET: async (_req, rc) => { + const q = fold(rc.url.searchParams.get("q") ?? "").trim(); + if (!q) return json([]); + const terms = q.split(/\s+/); + const hit = (s: string | undefined) => !!s && terms.every((t) => fold(s).includes(t)); + const idx = await accessIndex(ctx.db, rc.user); + const out: SearchHit[] = []; + for (const c of ctx.courses.list()) { + const m = c.manifest; + if (!m || !visible(rc.user, c, idx)) continue; + const course = hit(`${m.title} ${m.pitch ?? ""} ${(m.authors ?? []).join(" ")} ${(m.collections ?? []).join(" ")}`); + const lessons = m.modules.flatMap((mod) => mod.lessons.filter((l) => hit(l.title) || hit(`${mod.title} ${l.title}`)).map((l) => ({ id: l.id, title: l.title, moduleTitle: mod.title }))); + if (course || lessons.length) out.push({ courseId: m.id, courseTitle: m.title, course, lessons: lessons.slice(0, 8) }); + } + out.sort((a, b) => Number(b.course) - Number(a.course) || b.lessons.length - a.lessons.length); + return json(out.slice(0, 50)); + }, + }, + "/api/courses/:courseId": { + GET: async (_req, rc) => { + const found = await courseFor(ctx, rc.user, param(rc, "courseId")); + if (!found) return notFound(); + const { course, manifest: m } = found; + const progress = rc.user ? await progressRepo(ctx.db).get(rc.user.id, m.id) : {}; + const est = await minutesFor(course); + const locks = moduleLocks(m, progress, found.canEdit); + const deckTargets = new Set((m.decks ?? []).map((d) => d.attachTo)); + const modules: OutlineModule[] = m.modules.map((mod, i) => ({ + id: mod.id, title: mod.title, index: i, locked: locks[i]!, hasCode: !!mod.code, + lessons: mod.lessons.map((l, li) => ({ + id: l.id, title: l.title, type: l.type, minutes: est.get(l.id) ?? null, + hasDeck: deckTargets.has(l.id) || (li === 0 && deckTargets.has(mod.id)), browser: !!l.browser, + })), + })); + const payload: CoursePayload = { + id: m.id, title: m.title, pitch: m.pitch ?? null, authors: m.authors ?? [], accent: m.accent ?? null, + coverUrl: m.cover && ctx.courses.resolvePath(m.id, m.cover) ? mediaUrl(m.id, m.cover) : null, + tracks: m.tracks ?? [], gating: m.gating, source: m.source?.kind ?? null, status: course.status, + modules, progress, canEdit: found.canEdit, signedIn: !!rc.user, + diagnostics: found.canEdit ? course.diagnostics.map((d) => ({ level: d.level, lessonId: d.lessonId, path: d.path, message: d.message })) : [], + }; + return json(payload); + }, + }, + }; +} diff --git a/packages/feature-learner/src/server/feature.ts b/packages/feature-learner/src/server/feature.ts index 954e50c..c01bf11 100644 --- a/packages/feature-learner/src/server/feature.ts +++ b/packages/feature-learner/src/server/feature.ts @@ -1,8 +1,10 @@ -import type { AuthService, Feature, ServerContext } from "@stc/contracts"; +import type { AuthService, Feature, RouteHandler, ServerContext } from "@stc/contracts"; import { createAuth } from "./auth"; import { runMigrations } from "./migrate"; import { json } from "./http"; import { adminRoutes } from "./admin"; +import { libraryRoutes } from "./catalog"; +import { lessonRoutes } from "./lesson"; /** * The learner feature: auth, library, reader, notes, progress and exercises. @@ -22,6 +24,14 @@ export function learnerFeature(ctx: ServerContext): Feature & { auth: AuthServic GET: async () => json({ emailPassword: true, providers: auth.providers(), sso: await auth.hasSso() }), }, ...adminRoutes(ctx, auth.service.invalidate), + ...libraryRoutes(ctx), + ...lessonRoutes(ctx), }; + // Every handler waits for the learner migrations (a no-op once Phase 2 runs them in migrate()). + for (const methods of Object.values(routes)) { + for (const [m, h] of Object.entries(methods)) { + (methods as Record<string, RouteHandler>)[m] = async (req, rc) => { await ready; return h!(req, rc); }; + } + } return { name: "learner", routes, auth: auth.service }; } diff --git a/packages/feature-learner/src/server/gating.ts b/packages/feature-learner/src/server/gating.ts new file mode 100644 index 0000000..ff4de01 --- /dev/null +++ b/packages/feature-learner/src/server/gating.ts @@ -0,0 +1,15 @@ +import type { Manifest } from "@stc/course-format"; + +/** + * Sequential gating (spec section 12): with `gating: sequential`, module i is locked until every + * lesson of every earlier module is done. `bypass` (owners and course authors) unlocks everything. + */ +export function moduleLocks(m: Manifest, progress: Record<string, { state: string }>, bypass = false): boolean[] { + if (m.gating !== "sequential" || bypass) return m.modules.map(() => false); + let open = true; + return m.modules.map((mod) => { + const locked = !open; + if (!mod.lessons.every((l) => progress[l.id]?.state === "done")) open = false; + return locked; + }); +} diff --git a/packages/feature-learner/src/server/lesson.ts b/packages/feature-learner/src/server/lesson.ts new file mode 100644 index 0000000..fb9cafc --- /dev/null +++ b/packages/feature-learner/src/server/lesson.ts @@ -0,0 +1,115 @@ +import { statSync } from "node:fs"; +import type { Feature, ServerContext } from "@stc/contracts"; +import { extractHints, extractSection, type Lesson, type Manifest } from "@stc/course-format"; +import { progressRepo } from "@stc/db"; +import type { FileRef, LessonPayload } from "../shared/types"; +import { mediaUrl } from "../shared/types"; +import { courseFor } from "./access"; +import { createEstimator } from "./catalog"; +import { moduleLocks } from "./gating"; +import { json, notFound, param } from "./http"; + +const isMarkdown = (p: string) => /\.(md|markdown)$/i.test(p); + +export function lessonFiles(m: Manifest): Record<string, string> { + const out: Record<string, string> = {}; + for (const mod of m.modules) for (const l of mod.lessons) if (l.file && !l.anchor && isMarkdown(l.file) && !(l.file in out)) out[l.file] = l.id; + return out; +} + +export function lessonRoutes(ctx: ServerContext): Feature["routes"] { + const minutesFor = createEstimator(ctx); + + const read = async (courseId: string, rel: string): Promise<string | null> => { + const abs = ctx.courses.resolvePath(courseId, rel); + if (!abs) return null; + try { return await Bun.file(abs).text(); } catch { return null; } + }; + const fileRef = (courseId: string, file: string, title?: string): FileRef => { + const abs = ctx.courses.resolvePath(courseId, file); + let size: number | null = null; + try { if (abs) size = statSync(abs).size; } catch { /* missing */ } + return { title: title ?? file.split("/").pop()!, file, url: mediaUrl(courseId, file), size }; + }; + + return { + "/api/courses/:courseId/lessons/:lessonId": { + GET: async (_req, rc) => { + const found = await courseFor(ctx, rc.user, param(rc, "courseId")); + if (!found) return notFound(); + const { course, manifest: m } = found; + const lessonId = param(rc, "lessonId"); + const entry = course.lessonIndex.get(lessonId); + if (!entry) return notFound(); + const { lesson: l } = entry; + const modIndex = m.modules.findIndex((x) => x.id === entry.moduleId); + const mod = m.modules[modIndex]!; + const flat: { l: Lesson; mi: number }[] = m.modules.flatMap((x, mi) => x.lessons.map((ll) => ({ l: ll, mi }))); + const pos = flat.findIndex((x) => x.l.id === l.id); + const progress = rc.user ? await progressRepo(ctx.db).get(rc.user.id, m.id) : {}; + const locks = moduleLocks(m, progress, found.canEdit); + const link = (x: { l: Lesson; mi: number } | undefined) => (x ? { id: x.l.id, title: x.l.title } : null); + const est = await minutesFor(course); + + const base: LessonPayload = { + courseId: m.id, courseTitle: m.title, + lesson: { id: l.id, title: l.title, type: l.type, minutes: est.get(l.id) ?? null }, + module: { id: mod.id, title: mod.title, index: modIndex }, + locked: locks[modIndex]!, + markdown: null, baseFile: null, trackMarkdown: {}, tracks: m.tracks ?? [], + video: null, transcript: null, hints: [], decks: [], code: null, attachments: [], browser: null, + localCommand: null, + prev: link(flat[pos - 1]), next: link(flat[pos + 1]), + lessonFiles: lessonFiles(m), + }; + if (base.locked) return json(base); + + if (l.type === "video") { + if (l.file) base.video = { url: mediaUrl(m.id, l.file), file: l.file, duration: l.duration ?? null }; + if (l.transcript && isMarkdown(l.transcript)) { + base.transcript = await read(m.id, l.transcript); + base.baseFile = l.transcript; + } + } else if (l.file && isMarkdown(l.file)) { + const text = await read(m.id, l.file); + base.markdown = text === null ? null : l.anchor ? extractSection(text, l.anchor) : text; + base.baseFile = l.file; + } + for (const [track, t] of Object.entries(l.tracks ?? {})) { + if (t.file && isMarkdown(t.file)) { + const text = await read(m.id, t.file); + if (text !== null) base.trackMarkdown[track] = l.anchor ? extractSection(text, l.anchor) ?? text : text; + } + } + if (l.hints) { + const text = await read(m.id, l.hints.file); + if (text !== null) base.hints = extractHints(text, l.hints.match); + } + const first = mod.lessons[0]?.id === l.id; + base.decks = (m.decks ?? []) + .filter((d) => d.attachTo === l.id || (first && d.attachTo === mod.id)) + .filter((d) => ctx.courses.resolvePath(m.id, d.file)) + .map((d) => ({ id: d.id, title: d.title ?? d.file.split("/").pop()!.replace(/\.pdf$/i, ""), url: mediaUrl(m.id, d.file), attachTo: d.attachTo })); + if (mod.code) base.code = { moduleId: mod.id, dir: mod.code.dir, command: mod.code.command ?? null }; + base.attachments = (m.attachments ?? []) + .filter((a) => !a.module || a.module === mod.id) + .filter((a) => ctx.courses.resolvePath(m.id, a.file)) + .map((a) => fileRef(m.id, a.file, a.title)); + if (l.browser) { + const starter = await read(m.id, l.browser.starter); + const tests = await read(m.id, l.browser.tests); + if (starter !== null && tests !== null) { + base.browser = { + language: l.browser.language, starter, tests, starterFile: l.browser.starter.split("/").pop()!, + // the solution is visible to authors only (spec section 5) + solution: found.canEdit && l.browser.solution ? await read(m.id, l.browser.solution) : null, + }; + } + } + const trackCommands = Object.fromEntries(Object.entries(l.tracks ?? {}).filter(([, t]) => t.command).map(([k, t]) => [k, t.command!])); + base.localCommand = l.localCommand ?? (Object.keys(trackCommands).length ? trackCommands : null); + return json(base); + }, + }, + }; +} diff --git a/packages/feature-learner/src/shared/types.ts b/packages/feature-learner/src/shared/types.ts new file mode 100644 index 0000000..9eb4330 --- /dev/null +++ b/packages/feature-learner/src/shared/types.ts @@ -0,0 +1,121 @@ +// JSON shapes shared by the learner API and the web app. + +export type LessonType = "text" | "video" | "exercise"; +export type Format = "text" | "video" | "slides" | "exercises"; +export type ProgressState = "not-started" | "in-progress" | "done"; + +export interface LessonProgress { state: ProgressState; positionSeconds: number | null; updatedAt: string } + +export interface CourseCard { + id: string; + title: string; + pitch: string | null; + authors: string[]; + coverUrl: string | null; + accent: string | null; + formats: Format[]; + lessonCount: number; + minutes: number; + collections: string[]; + tracks: string[]; + source: string | null; + addedAt: string; + status: "ok" | "error"; + /** 0..1 share of lessons done; 0 when signed out. */ + progress: number; + doneCount: number; + started: boolean; +} + +export interface ContinueItem { + courseId: string; + lessonId: string; + lessonTitle: string; + moduleTitle: string; + moduleIndex: number; + moduleDone: number; + moduleTotal: number; + progress: number; +} + +export interface LibraryPayload { + signedIn: boolean; + featured: CourseCard | null; + continue: ContinueItem[]; + newest: string[]; + collections: { id: string; title: string; courseIds: string[] }[]; + courses: CourseCard[]; +} + +export interface SearchHit { courseId: string; courseTitle: string; course: boolean; lessons: { id: string; title: string; moduleTitle: string }[] } + +export interface OutlineLesson { id: string; title: string; type: LessonType; minutes: number | null; hasDeck: boolean; browser: boolean } +export interface OutlineModule { id: string; title: string; index: number; locked: boolean; hasCode: boolean; lessons: OutlineLesson[] } + +export interface Diagnostic { level: "error" | "warning"; lessonId?: string; path?: string; message: string } + +export interface CoursePayload { + id: string; + title: string; + pitch: string | null; + authors: string[]; + accent: string | null; + coverUrl: string | null; + tracks: { id: string; label: string }[]; + gating: "none" | "sequential"; + source: string | null; + status: "ok" | "error"; + modules: OutlineModule[]; + progress: Record<string, LessonProgress>; + canEdit: boolean; + signedIn: boolean; + diagnostics: Diagnostic[]; +} + +export interface DeckRef { id: string; title: string; url: string; attachTo: string } +export interface FileRef { title: string; file: string; url: string; size: number | null } +export interface LessonLink { id: string; title: string } + +export interface LessonPayload { + courseId: string; + courseTitle: string; + lesson: { id: string; title: string; type: LessonType; minutes: number | null }; + module: { id: string; title: string; index: number }; + locked: boolean; + /** Lesson body (text: the file or its anchored section; exercise: the anchored section; video: null). */ + markdown: string | null; + /** Course-relative path the markdown came from, for resolving relative links and images. */ + baseFile: string | null; + /** Per-track replacement bodies, when the manifest gives track files. */ + trackMarkdown: Record<string, string>; + tracks: { id: string; label: string }[]; + video: { url: string; file: string; duration: number | null } | null; + transcript: string | null; + hints: { label: string; body: string }[]; + decks: DeckRef[]; + code: { moduleId: string; dir: string; command: string | null } | null; + attachments: FileRef[]; + browser: { language: "js" | "ts"; starter: string; tests: string; starterFile: string; solution: string | null } | null; + localCommand: string | Record<string, string> | null; + prev: LessonLink | null; + next: LessonLink | null; + /** Course file → lesson id, for turning links between lesson files into reader links. */ + lessonFiles: Record<string, string>; +} + +export interface Note { + id: string; + courseId: string; + lessonId: string; + anchorKind: "none" | "video" | "slide"; + anchorValue: string | null; + deckId: string | null; + body: string; + createdAt: string; + updatedAt: string; +} + +export interface CodeListing { moduleId: string; dir: string; command: string | null; files: { path: string; size: number }[]; truncated: boolean } + +export const mediaUrl = (courseId: string, rel: string): string => + `/api/media/${encodeURIComponent(courseId)}/${rel.split("/").map(encodeURIComponent).join("/")}`; diff --git a/packages/feature-learner/test/catalog.test.ts b/packages/feature-learner/test/catalog.test.ts new file mode 100644 index 0000000..62fb789 --- /dev/null +++ b/packages/feature-learner/test/catalog.test.ts @@ -0,0 +1,61 @@ +import { expect } from "bun:test"; +import { forEachDb } from "./helpers"; +import { FIXTURES, makeApp } from "./app"; +import { extraCourses } from "./courses"; + +forEachDb("library shows only visible courses, with formats and lengths", async (db) => { + const t = await makeApp(db, [FIXTURES, await extraCourses()]); + const anon = await (await t.call("GET", "/api/library")).json(); + expect(anon.signedIn).toBe(false); + expect(anon.courses.map((c: { id: string }) => c.id)).toEqual(["sample"]); + const sample = anon.courses[0]; + expect(sample.formats).toEqual(["text", "video", "slides", "exercises"]); + expect(sample.lessonCount).toBe(5); + expect(sample.minutes).toBeGreaterThan(14); + expect(anon.featured.id).toBe("sample"); + + const owner = await t.signUp("owner@example.com"); + const learner = await t.signUp("l@example.com"); + const asOwner = await (await t.call("GET", "/api/library", { cookie: owner })).json(); + expect(asOwner.courses.map((c: { id: string }) => c.id).sort()).toEqual(["gated", "sample", "secret"]); + const asLearner = await (await t.call("GET", "/api/library", { cookie: learner })).json(); + expect(asLearner.courses.map((c: { id: string }) => c.id).sort()).toEqual(["gated", "sample"]); + // manifest collections become rows; restricted courses do not leak into them + expect(asLearner.collections).toEqual([ + { id: "devops", title: "DevOps", courseIds: ["gated"] }, + { id: "functional-programming", title: "Functional programming", courseIds: ["gated"] }, + ]); + expect((await t.call("GET", "/api/courses/secret", { cookie: learner })).status).toBe(404); + expect((await t.call("GET", "/api/courses/secret/lessons/s1", { cookie: learner })).status).toBe(404); + + // continue row: after touching a lesson + await db.sql`INSERT INTO progress (user_id, course_id, lesson_id, state, position_seconds, updated_at) + SELECT id, 'gated', 'g1a', 'done', NULL, '2026-10-30T00:00:00Z' FROM auth_user WHERE email = 'l@example.com'`; + const again = await (await t.call("GET", "/api/library", { cookie: learner })).json(); + expect(again.continue).toEqual([{ courseId: "gated", lessonId: "g1b", lessonTitle: "Then this", moduleTitle: "First", moduleIndex: 0, moduleDone: 1, moduleTotal: 2, progress: 1 / 3 }]); + expect(again.featured.id).toBe("gated"); +}, 30_000); + +forEachDb("search matches titles, pitch and lesson titles", async (db) => { + const t = await makeApp(db, [FIXTURES, await extraCourses()]); + const hits = await (await t.call("GET", "/api/library/search?q=clip")).json(); + expect(hits).toEqual([{ courseId: "sample", courseTitle: "Sample course", course: false, lessons: [{ id: "m1-video", title: "Watch the clip", moduleTitle: "First steps" }] }]); + const tiny = await (await t.call("GET", "/api/library/search?q=TINY")).json(); + expect(tiny[0].course).toBe(true); + expect(await (await t.call("GET", "/api/library/search?q=")).json()).toEqual([]); +}, 30_000); + +forEachDb("course outline carries gating locks and progress", async (db) => { + const t = await makeApp(db, [FIXTURES, await extraCourses()]); + await t.signUp("owner@example.com"); + const learner = await t.signUp("l@example.com"); + const c = await (await t.call("GET", "/api/courses/gated", { cookie: learner })).json(); + expect(c.modules.map((m: { locked: boolean }) => m.locked)).toEqual([false, true]); + expect(c.modules[1].lessons[0].hasDeck).toBe(true); + expect(c.canEdit).toBe(false); + expect(c.diagnostics).toEqual([]); + expect((await t.call("GET", "/api/courses/gated")).status).toBe(404); // signed-in only + const locked = await (await t.call("GET", "/api/courses/gated/lessons/g2a", { cookie: learner })).json(); + expect(locked.locked).toBe(true); + expect(locked.markdown).toBeNull(); +}, 30_000); diff --git a/packages/feature-learner/test/courses.ts b/packages/feature-learner/test/courses.ts new file mode 100644 index 0000000..5540b14 --- /dev/null +++ b/packages/feature-learner/test/courses.ts @@ -0,0 +1,48 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +/** Writes extra test courses to a temp dir: a restricted one and a gated one with a deck on a module. */ +export async function extraCourses(): Promise<string> { + const dir = mkdtempSync(join(tmpdir(), "stc-courses-")); + await Bun.write(join(dir, "secret/stcourse.yaml"), `schema: 1 +id: secret +title: Secret course +visibility: restricted +collections: [DevOps] +modules: + - id: m1 + title: Only module + lessons: + - { id: s1, title: Hidden lesson, type: text, file: a.md } +`); + await Bun.write(join(dir, "secret/a.md"), "# Hidden\n\nwords here"); + await Bun.write(join(dir, "gated/stcourse.yaml"), `schema: 1 +id: gated +title: Gated course +pitch: Learn things in order +visibility: signed-in +gating: sequential +collections: [DevOps, Functional programming] +decks: + - { id: dk, file: "Æble slides/deck one.pdf", title: Deck one, attachTo: g2 } +attachments: + - { file: "Æble slides/cheat sheet.pdf", title: Cheat sheet, module: g1 } +modules: + - id: g1 + title: First + lessons: + - { id: g1a, title: Start here, type: text, file: "lessons/one.md" } + - { id: g1b, title: Then this, type: text, file: "lessons/two.md" } + - id: g2 + title: Second + lessons: + - { id: g2a, title: Locked lesson, type: text, file: "lessons/three.md" } +`); + await Bun.write(join(dir, "gated/lessons/one.md"), "# One\n\nSee [two](two.md).\n"); + await Bun.write(join(dir, "gated/lessons/two.md"), "# Two\n"); + await Bun.write(join(dir, "gated/lessons/three.md"), "# Three\n\nsecret content"); + await Bun.write(join(dir, "gated/Æble slides/deck one.pdf"), "%PDF-1.4\n"); + await Bun.write(join(dir, "gated/Æble slides/cheat sheet.pdf"), "%PDF-1.4\ncheat"); + return dir; +} diff --git a/packages/feature-learner/test/gating.test.ts b/packages/feature-learner/test/gating.test.ts new file mode 100644 index 0000000..cf67e3b --- /dev/null +++ b/packages/feature-learner/test/gating.test.ts @@ -0,0 +1,21 @@ +import { expect, test } from "bun:test"; +import { parseManifest } from "@stc/course-format"; +import { moduleLocks } from "../src/server/gating"; + +const m = (gating: string) => { + const r = parseManifest(`schema: 1\nid: g\ntitle: G\ngating: ${gating}\nmodules:\n - { id: a, title: A, lessons: [{ id: a1, title: A1, type: exercise }, { id: a2, title: A2, type: exercise }] }\n - { id: b, title: B, lessons: [{ id: b1, title: B1, type: exercise }] }\n - { id: c, title: C, lessons: [{ id: c1, title: C1, type: exercise }] }\n`); + if (!r.ok) throw new Error(r.errors.join()); + return r.manifest; +}; + +test("no gating unlocks everything", () => { + expect(moduleLocks(m("none"), {})).toEqual([false, false, false]); +}); +test("sequential gating locks until the previous module is done", () => { + expect(moduleLocks(m("sequential"), {})).toEqual([false, true, true]); + expect(moduleLocks(m("sequential"), { a1: { state: "done" } })).toEqual([false, true, true]); + expect(moduleLocks(m("sequential"), { a1: { state: "done" }, a2: { state: "done" } })).toEqual([false, false, true]); + // done later modules do not unlock past an unfinished one + expect(moduleLocks(m("sequential"), { a1: { state: "done" }, a2: { state: "done" }, c1: { state: "done" } })).toEqual([false, false, true]); + expect(moduleLocks(m("sequential"), {}, true)).toEqual([false, false, false]); +}); diff --git a/packages/feature-learner/test/lesson.test.ts b/packages/feature-learner/test/lesson.test.ts new file mode 100644 index 0000000..dba956e --- /dev/null +++ b/packages/feature-learner/test/lesson.test.ts @@ -0,0 +1,47 @@ +import { expect } from "bun:test"; +import { forEachDb } from "./helpers"; +import { FIXTURES, makeApp } from "./app"; +import { extraCourses } from "./courses"; + +forEachDb("lesson payloads for every sample lesson type", async (db) => { + const t = await makeApp(db); + const get = async (id: string, cookie?: string) => (await t.call("GET", `/api/courses/sample/lessons/${id}`, { cookie })).json(); + + const intro = await get("m1-intro"); + expect(intro.markdown).toContain("Welcome to the sample course"); + expect(intro.prev).toBeNull(); + expect(intro.next).toEqual({ id: "m1-video", title: "Watch the clip" }); + expect(intro.code).toEqual({ moduleId: "m1", dir: "code/S01 - First steps", command: "sh hello.sh" }); + expect(intro.lessonFiles["lessons/01-intro.md"]).toBe("m1-intro"); + + const video = await get("m1-video"); + expect(video.video).toEqual({ url: "/api/media/sample/media/clip.mp4", file: "media/clip.mp4", duration: 2 }); + expect(video.transcript).toContain("test pattern"); + expect(video.decks).toEqual([{ id: "d1", title: "deck", url: "/api/media/sample/media/deck.pdf", attachTo: "m1-video" }]); + + const ex = await get("m2-ex-add"); + expect(ex.markdown?.startsWith("### Exercise 2.1")).toBe(true); + expect(ex.markdown).not.toContain("Hints"); + expect(ex.hints.map((h: { label: string }) => h.label)).toEqual(["nudge", "near-solution"]); + expect(ex.browser.language).toBe("js"); + expect(ex.browser.starter).toContain("a - b"); + expect(ex.browser.tests).toContain("export const tests"); + expect(ex.browser.solution).toBeNull(); // not an author + expect(ex.localCommand).toBe("node --test"); + + const owner = await t.signUp("owner@example.com"); + expect((await get("m2-ex-add", owner)).browser.solution).toContain("a + b"); + expect((await t.call("GET", "/api/courses/sample/lessons/nope")).status).toBe(404); +}, 30_000); + +forEachDb("module decks and attachments with non-ASCII paths", async (db) => { + const t = await makeApp(db, [FIXTURES, await extraCourses()]); + const owner = await t.signUp("owner@example.com"); + const l = await (await t.call("GET", "/api/courses/gated/lessons/g2a", { cookie: owner })).json(); + expect(l.locked).toBe(false); // owners bypass gating + expect(l.decks[0].url).toBe("/api/media/gated/%C3%86ble%20slides/deck%20one.pdf"); + const first = await (await t.call("GET", "/api/courses/gated/lessons/g1a", { cookie: owner })).json(); + expect(first.attachments).toEqual([{ title: "Cheat sheet", file: "Æble slides/cheat sheet.pdf", url: "/api/media/gated/%C3%86ble%20slides/cheat%20sheet.pdf", size: 14 }]); + const media = await t.call("GET", first.attachments[0].url, { cookie: owner }); + expect(media.status).toBe(200); +}, 30_000); From 96c147e3a2bdd7b28ead7c3c2c15b32b88bb8b2e Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 03:24:16 +0100 Subject: [PATCH 32/62] feat(learner): progress, notes and exercise attempt API Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .../feature-learner/src/server/feature.ts | 2 + .../feature-learner/src/server/userdata.ts | 117 ++++++++++++++++++ .../feature-learner/test/userdata.test.ts | 65 ++++++++++ 3 files changed, 184 insertions(+) create mode 100644 packages/feature-learner/src/server/userdata.ts create mode 100644 packages/feature-learner/test/userdata.test.ts diff --git a/packages/feature-learner/src/server/feature.ts b/packages/feature-learner/src/server/feature.ts index c01bf11..0a857c0 100644 --- a/packages/feature-learner/src/server/feature.ts +++ b/packages/feature-learner/src/server/feature.ts @@ -5,6 +5,7 @@ import { json } from "./http"; import { adminRoutes } from "./admin"; import { libraryRoutes } from "./catalog"; import { lessonRoutes } from "./lesson"; +import { userDataRoutes } from "./userdata"; /** * The learner feature: auth, library, reader, notes, progress and exercises. @@ -26,6 +27,7 @@ export function learnerFeature(ctx: ServerContext): Feature & { auth: AuthServic ...adminRoutes(ctx, auth.service.invalidate), ...libraryRoutes(ctx), ...lessonRoutes(ctx), + ...userDataRoutes(ctx), }; // Every handler waits for the learner migrations (a no-op once Phase 2 runs them in migrate()). for (const methods of Object.values(routes)) { diff --git a/packages/feature-learner/src/server/userdata.ts b/packages/feature-learner/src/server/userdata.ts new file mode 100644 index 0000000..050b243 --- /dev/null +++ b/packages/feature-learner/src/server/userdata.ts @@ -0,0 +1,117 @@ +import type { Feature, ServerContext } from "@stc/contracts"; +import { attemptsRepo, notesRepo, progressRepo, type Note as DbNote } from "@stc/db"; +import type { Note, ProgressState } from "../shared/types"; +import { courseFor } from "./access"; +import { moduleLocks } from "./gating"; +import { badRequest, body, json, notFound, param, signedIn } from "./http"; + +const STATES: ProgressState[] = ["not-started", "in-progress", "done"]; +const KINDS = ["none", "video", "slide"] as const; +const MAX_NOTE = 20_000; +const MAX_CODE = 200_000; + +const publicNote = ({ userId: _u, ...n }: DbNote): Note => n; + +/** Progress, notes and exercise attempts: per user, signed-in only, scoped to courses the user can see. */ +export function userDataRoutes(ctx: ServerContext): Feature["routes"] { + const progress = progressRepo(ctx.db); + const notes = notesRepo(ctx.db); + const attempts = attemptsRepo(ctx.db); + + /** Course and lesson the user may write to, or an error response. */ + async function target(userId: string, user: Parameters<typeof courseFor>[1], courseId: string, lessonId: string) { + const found = await courseFor(ctx, user, courseId); + if (!found) return { error: notFound() }; + const entry = found.course.lessonIndex.get(lessonId); + if (!entry) return { error: notFound() }; + const p = await progress.get(userId, courseId); + const mi = found.manifest.modules.findIndex((m) => m.id === entry.moduleId); + if (moduleLocks(found.manifest, p, found.canEdit)[mi]) return { error: json({ error: "this module is locked until the previous one is done" }, 403) }; + return { found, entry, progress: p }; + } + + return { + "/api/courses/:courseId/progress": { + GET: signedIn(async (_req, rc, user) => { + const courseId = param(rc, "courseId"); + if (!(await courseFor(ctx, user, courseId))) return notFound(); + return json(await progress.get(user.id, courseId)); + }), + }, + "/api/courses/:courseId/progress/:lessonId": { + PUT: signedIn(async (req, rc, user) => { + const courseId = param(rc, "courseId"), lessonId = param(rc, "lessonId"); + const b = await body<{ state?: ProgressState; positionSeconds?: number }>(req); + if (!b || !STATES.includes(b.state as ProgressState)) return badRequest("state must be not-started, in-progress or done"); + if (b.positionSeconds !== undefined && (typeof b.positionSeconds !== "number" || !Number.isFinite(b.positionSeconds) || b.positionSeconds < 0)) return badRequest("positionSeconds must be a non-negative number"); + const t = await target(user.id, user, courseId, lessonId); + if ("error" in t) return t.error; + const prior = t.progress[lessonId]; + // A position report ("in-progress") never undoes "done"; only an explicit not-started does. + const state = prior?.state === "done" && b.state === "in-progress" ? "done" : b.state!; + const pos = b.positionSeconds ?? prior?.positionSeconds ?? undefined; + await progress.set(user.id, courseId, lessonId, state, pos ?? undefined); + return json((await progress.get(user.id, courseId))[lessonId]); + }), + }, + "/api/courses/:courseId/notes": { + GET: signedIn(async (_req, rc, user) => { + const courseId = param(rc, "courseId"); + if (!(await courseFor(ctx, user, courseId))) return notFound(); + const lesson = rc.url.searchParams.get("lesson") ?? undefined; + return json((await notes.list(user.id, courseId, lesson)).map(publicNote)); + }), + POST: signedIn(async (req, rc, user) => { + const courseId = param(rc, "courseId"); + const b = await body<{ lessonId?: string; anchorKind?: string; anchorValue?: string | number | null; deckId?: string | null; body?: string }>(req); + if (!b || typeof b.lessonId !== "string" || typeof b.body !== "string") return badRequest("lessonId and body are required"); + if (b.body.length > MAX_NOTE) return badRequest("note is too long"); + const kind = (b.anchorKind ?? "none") as (typeof KINDS)[number]; + if (!KINDS.includes(kind)) return badRequest("anchorKind must be none, video or slide"); + if (kind !== "none" && (b.anchorValue === null || b.anchorValue === undefined || Number.isNaN(Number(b.anchorValue)))) return badRequest("video and slide notes need a numeric anchorValue"); + const found = await courseFor(ctx, user, courseId); + if (!found || !found.course.lessonIndex.has(b.lessonId)) return notFound(); + if (kind === "slide" && !(found.manifest.decks ?? []).some((d) => d.id === b.deckId)) return badRequest("slide notes need the deckId of a deck in this course"); + const n = await notes.create({ + userId: user.id, courseId, lessonId: b.lessonId, anchorKind: kind, + anchorValue: kind === "none" ? null : String(b.anchorValue), deckId: kind === "slide" ? b.deckId! : null, body: b.body, + }); + return json(publicNote(n), 201); + }), + }, + "/api/notes/:id": { + PATCH: signedIn(async (req, rc, user) => { + const b = await body<{ body?: string }>(req); + if (!b || typeof b.body !== "string") return badRequest("body is required"); + if (b.body.length > MAX_NOTE) return badRequest("note is too long"); + const n = await notes.update(param(rc, "id"), user.id, b.body); + return n ? json(publicNote(n)) : notFound(); + }), + DELETE: signedIn(async (_req, rc, user) => ((await notes.remove(param(rc, "id"), user.id)) ? json({ ok: true }) : notFound())), + }, + "/api/courses/:courseId/lessons/:lessonId/attempt": { + GET: signedIn(async (_req, rc, user) => { + const courseId = param(rc, "courseId"), lessonId = param(rc, "lessonId"); + const found = await courseFor(ctx, user, courseId); + if (!found || !found.course.lessonIndex.has(lessonId)) return notFound(); + const a = await attempts.get(user.id, courseId, lessonId, rc.url.searchParams.get("track") ?? "default"); + return json(a ? { code: a.code, result: a.result, passed: a.passed, updatedAt: a.updatedAt } : null); + }), + PUT: signedIn(async (req, rc, user) => { + const courseId = param(rc, "courseId"), lessonId = param(rc, "lessonId"); + const b = await body<{ track?: string; code?: string; result?: unknown; passed?: boolean }>(req); + if (!b || typeof b.code !== "string") return badRequest("code is required"); + if (b.code.length > MAX_CODE) return badRequest("code is too long"); + const t = await target(user.id, user, courseId, lessonId); + if ("error" in t) return t.error; + if (t.entry.lesson.type !== "exercise") return badRequest("not an exercise"); + const passed = b.passed === true; + await attempts.save({ userId: user.id, courseId, lessonId, track: b.track || "default", code: b.code, result: b.result ?? null, passed, updatedAt: new Date().toISOString() }); + // An exercise counts as done when all its tests pass (spec 4.4). + if (passed) await progress.set(user.id, courseId, lessonId, "done"); + else if (!t.progress[lessonId]) await progress.set(user.id, courseId, lessonId, "in-progress"); + return json({ ok: true, passed, state: (await progress.get(user.id, courseId))[lessonId]?.state }); + }), + }, + }; +} diff --git a/packages/feature-learner/test/userdata.test.ts b/packages/feature-learner/test/userdata.test.ts new file mode 100644 index 0000000..2148b64 --- /dev/null +++ b/packages/feature-learner/test/userdata.test.ts @@ -0,0 +1,65 @@ +import { expect } from "bun:test"; +import { forEachDb } from "./helpers"; +import { FIXTURES, makeApp } from "./app"; +import { extraCourses } from "./courses"; + +forEachDb("progress: mark done, positions never undo done, gating blocks writes", async (db) => { + const t = await makeApp(db, [FIXTURES, await extraCourses()]); + await t.signUp("owner@example.com"); + const me = await t.signUp("l@example.com"); + const put = (course: string, lesson: string, body: unknown, cookie = me) => t.call("PUT", `/api/courses/${course}/progress/${lesson}`, { cookie, body }); + + expect((await put("sample", "m1-intro", { state: "done" }, "")).status).toBe(401); + expect((await put("sample", "m1-intro", { state: "finished" })).status).toBe(400); + expect((await (await put("sample", "m1-video", { state: "in-progress", positionSeconds: 1.5 })).json()).positionSeconds).toBe(1.5); + await put("sample", "m1-video", { state: "done" }); + const after = await (await put("sample", "m1-video", { state: "in-progress", positionSeconds: 0.5 })).json(); + expect(after).toMatchObject({ state: "done", positionSeconds: 0.5 }); + expect((await put("sample", "nope", { state: "done" })).status).toBe(404); + + expect((await put("gated", "g2a", { state: "done" })).status).toBe(403); + await put("gated", "g1a", { state: "done" }); + await put("gated", "g1b", { state: "done" }); + expect((await put("gated", "g2a", { state: "done" })).status).toBe(200); + const all = await (await t.call("GET", "/api/courses/gated/progress", { cookie: me })).json(); + expect(Object.keys(all).sort()).toEqual(["g1a", "g1b", "g2a"]); +}, 30_000); + +forEachDb("notes persist per lesson with slide and video anchors", async (db) => { + const t = await makeApp(db); + const me = await t.signUp("l@example.com"); + const other = await t.signUp("o@example.com"); + const post = (b: unknown, cookie = me) => t.call("POST", "/api/courses/sample/notes", { cookie, body: b }); + expect((await post({ lessonId: "m1-video", anchorKind: "slide", anchorValue: 2, body: "x" })).status).toBe(400); // no deck id + expect((await post({ lessonId: "m1-video", anchorKind: "video", body: "x" })).status).toBe(400); + const s10 = await (await post({ lessonId: "m1-video", anchorKind: "slide", anchorValue: 10, deckId: "d1", body: "slide ten" })).json(); + await post({ lessonId: "m1-video", anchorKind: "slide", anchorValue: 2, deckId: "d1", body: "slide two" }); + await post({ lessonId: "m1-video", anchorKind: "video", anchorValue: 1.25, body: "at 1.25s" }); + await post({ lessonId: "m1-intro", body: "general" }); + const list = await (await t.call("GET", "/api/courses/sample/notes?lesson=m1-video", { cookie: me })).json(); + expect(list.map((n: { body: string }) => n.body)).toEqual(["slide two", "slide ten", "at 1.25s"]); + expect(list[0]).toMatchObject({ anchorKind: "slide", anchorValue: "2", deckId: "d1" }); + expect(list[0].userId).toBeUndefined(); + expect((await (await t.call("GET", "/api/courses/sample/notes", { cookie: me })).json()).length).toBe(4); + expect(await (await t.call("GET", "/api/courses/sample/notes", { cookie: other })).json()).toEqual([]); + + expect((await t.call("PATCH", `/api/notes/${s10.id}`, { cookie: other, body: { body: "hijack" } })).status).toBe(404); + expect((await (await t.call("PATCH", `/api/notes/${s10.id}`, { cookie: me, body: { body: "edited" } })).json()).body).toBe("edited"); + expect((await t.call("DELETE", `/api/notes/${s10.id}`, { cookie: other })).status).toBe(404); + expect((await t.call("DELETE", `/api/notes/${s10.id}`, { cookie: me })).status).toBe(200); + expect((await (await t.call("GET", "/api/courses/sample/notes?lesson=m1-video", { cookie: me })).json()).length).toBe(2); +}, 30_000); + +forEachDb("exercise attempts save code and a pass marks the lesson done", async (db) => { + const t = await makeApp(db); + const me = await t.signUp("l@example.com"); + const url = "/api/courses/sample/lessons/m2-ex-add/attempt"; + expect(await (await t.call("GET", `${url}?track=js`, { cookie: me })).json()).toBeNull(); + const r1 = await (await t.call("PUT", url, { cookie: me, body: { track: "js", code: "draft", passed: false, result: [{ name: "a", passed: false }] } })).json(); + expect(r1).toEqual({ ok: true, passed: false, state: "in-progress" }); + const r2 = await (await t.call("PUT", url, { cookie: me, body: { track: "js", code: "export const add=(a,b)=>a+b", passed: true, result: [] } })).json(); + expect(r2.state).toBe("done"); + const got = await (await t.call("GET", `${url}?track=js`, { cookie: me })).json(); + expect(got).toMatchObject({ code: "export const add=(a,b)=>a+b", passed: true }); + expect((await t.call("PUT", "/api/courses/sample/lessons/m1-intro/attempt", { cookie: me, body: { code: "x" } })).status).toBe(400); +}, 30_000); From 478ebeb255478af2b5b3f5d9dc3a5d8de2b7aaf5 Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 03:24:28 +0100 Subject: [PATCH 33/62] fix(learner): narrow lesson target result for the type checker Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- packages/feature-learner/src/server/userdata.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/feature-learner/src/server/userdata.ts b/packages/feature-learner/src/server/userdata.ts index 050b243..f2f21f2 100644 --- a/packages/feature-learner/src/server/userdata.ts +++ b/packages/feature-learner/src/server/userdata.ts @@ -45,7 +45,7 @@ export function userDataRoutes(ctx: ServerContext): Feature["routes"] { if (!b || !STATES.includes(b.state as ProgressState)) return badRequest("state must be not-started, in-progress or done"); if (b.positionSeconds !== undefined && (typeof b.positionSeconds !== "number" || !Number.isFinite(b.positionSeconds) || b.positionSeconds < 0)) return badRequest("positionSeconds must be a non-negative number"); const t = await target(user.id, user, courseId, lessonId); - if ("error" in t) return t.error; + if (t.error) return t.error; const prior = t.progress[lessonId]; // A position report ("in-progress") never undoes "done"; only an explicit not-started does. const state = prior?.state === "done" && b.state === "in-progress" ? "done" : b.state!; @@ -103,7 +103,7 @@ export function userDataRoutes(ctx: ServerContext): Feature["routes"] { if (!b || typeof b.code !== "string") return badRequest("code is required"); if (b.code.length > MAX_CODE) return badRequest("code is too long"); const t = await target(user.id, user, courseId, lessonId); - if ("error" in t) return t.error; + if (t.error) return t.error; if (t.entry.lesson.type !== "exercise") return badRequest("not an exercise"); const passed = b.passed === true; await attempts.save({ userId: user.id, courseId, lessonId, track: b.track || "default", code: b.code, result: b.result ?? null, passed, updatedAt: new Date().toISOString() }); From a57222ecc33e318e75d417d6ba49c376c694be42 Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 03:25:10 +0100 Subject: [PATCH 34/62] feat(ai): HTTP routes, chat streaming and generation job Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- packages/feature-ai/src/feature.ts | 425 ++++++++++++++++++ packages/feature-ai/src/index.ts | 1 + .../feature-ai/src/retrieval/course-index.ts | 7 +- packages/feature-ai/test/feature.test.ts | 156 +++++++ packages/feature-ai/test/retrieval.test.ts | 5 + 5 files changed, 592 insertions(+), 2 deletions(-) create mode 100644 packages/feature-ai/src/feature.ts create mode 100644 packages/feature-ai/test/feature.test.ts diff --git a/packages/feature-ai/src/feature.ts b/packages/feature-ai/src/feature.ts new file mode 100644 index 0000000..02f59e2 --- /dev/null +++ b/packages/feature-ai/src/feature.ts @@ -0,0 +1,425 @@ +import { readdir } from "node:fs/promises"; +import { extname, join, relative, sep } from "node:path"; +import type { AiProvider, AuthUser, ChatMessage, Feature, JobContext, RequestContext, ServerContext } from "@stc/contracts"; +import { canView } from "@stc/core"; +import type { LoadedCourse } from "@stc/course-format"; +import { aclRepo, notesRepo } from "@stc/db"; +import { buildSystemPrompt, citationsFor, readMentorText } from "./assistant/prompt"; +import { combineTools, courseTools, mcpTools, type ToolSet } from "./assistant/tools"; +import { runAssistant } from "./assistant/loop"; +import { createSecretBox } from "./crypto"; +import { createFsDraftSink, type DraftSink } from "./generator/draft-sink"; +import { generateDraft, type GenerateRequest, type GenerateSource } from "./generator/generate"; +import { connectMcp, mcpServersFor } from "./mcp/client"; +import { createProvider } from "./providers"; +import { getCourseIndex, noteChunks } from "./retrieval/course-index"; +import { aiSettingsRepo, type McpServerConfig, type ProviderConfig } from "./settings"; + +export interface AiFeatureOptions { + /** Used by providers and HTTP MCP servers. Tests pass a fake; production uses global fetch. */ + fetch?: typeof fetch; + /** Where generated drafts go. Phase 2 passes the studio's createDraftWriter(ctx); default writes under <dataDir>/drafts. */ + draftSink?: DraftSink; + /** Overrides provider construction (tests use a fake AiProvider). */ + providerFactory?: (cfg: ProviderConfig, apiKey: string | null) => AiProvider; + mcpTimeoutMs?: number; +} + +const json = (body: unknown, status = 200) => Response.json(body, { status }); +const err = (status: number, message: string) => json({ error: message }, status); +const KINDS = new Set(["anthropic", "openai"]); +const EFFORTS = new Set(["low", "medium", "high", "xhigh", "max"]); +const HISTORY_TURNS = 20; +const MAX_MESSAGE = 8_000; + +async function body(req: Request): Promise<Record<string, unknown> | null> { + try { + const b = await req.json(); + return b && typeof b === "object" && !Array.isArray(b) ? (b as Record<string, unknown>) : null; + } catch { return null; } +} +const str = (v: unknown): string | undefined => (typeof v === "string" && v.trim() ? v.trim() : undefined); + +function parseProvider(v: unknown): (ProviderConfig & { apiKey?: string; clearApiKey?: boolean }) | null | "invalid" { + if (v === null || v === undefined) return null; + if (typeof v !== "object") return "invalid"; + const p = v as Record<string, unknown>; + if (!KINDS.has(p.kind as string)) return "invalid"; + if (p.effort !== undefined && p.effort !== "" && !EFFORTS.has(p.effort as string)) return "invalid"; + const out: ProviderConfig & { apiKey?: string; clearApiKey?: boolean } = { kind: p.kind as ProviderConfig["kind"] }; + if (str(p.model)) out.model = str(p.model); + if (str(p.baseUrl)) { + try { new URL(str(p.baseUrl)!); } catch { return "invalid"; } + out.baseUrl = str(p.baseUrl); + } + if (str(p.effort)) out.effort = str(p.effort) as ProviderConfig["effort"]; + if (str(p.apiKey)) out.apiKey = str(p.apiKey); + if (p.clearApiKey === true) out.clearApiKey = true; + if (out.kind === "openai" && (!out.baseUrl || !out.model)) return "invalid"; + return out; +} + +function parseMcp(v: unknown): McpServerConfig[] | "invalid" { + if (v === undefined || v === null) return []; + if (!Array.isArray(v)) return "invalid"; + const out: McpServerConfig[] = []; + const ids = new Set<string>(); + for (const raw of v) { + if (!raw || typeof raw !== "object") return "invalid"; + const s = raw as Record<string, unknown>; + const id = str(s.id); + if (!id || !/^[a-zA-Z0-9_-]{1,40}$/.test(id) || ids.has(id)) return "invalid"; + ids.add(id); + const transport = s.transport === "stdio" || s.transport === "http" ? s.transport : null; + if (!transport) return "invalid"; + const cfg: McpServerConfig = { id, name: str(s.name) ?? id, transport }; + if (transport === "stdio") { + if (!str(s.command)) return "invalid"; + cfg.command = str(s.command); + if (Array.isArray(s.args)) cfg.args = s.args.map(String); + if (s.env && typeof s.env === "object") cfg.env = Object.fromEntries(Object.entries(s.env).map(([k, val]) => [k, String(val)])); + } else { + const url = str(s.url); + if (!url) return "invalid"; + try { new URL(url); } catch { return "invalid"; } + cfg.url = url; + if (s.headers && typeof s.headers === "object") cfg.headers = Object.fromEntries(Object.entries(s.headers).map(([k, val]) => [k, String(val)])); + } + if (Array.isArray(s.courseIds)) cfg.courseIds = s.courseIds.map(String).filter(Boolean); + out.push(cfg); + } + return out; +} + +function parseGenerate(b: Record<string, unknown>): GenerateRequest | null { + const courseId = str(b.courseId); + const topic = str(b.topic); + if (!courseId || !topic) return null; + if (!["document", "exercise", "questions"].includes(b.kind as string)) return null; + if (!["pure-snake", "systemic-design"].includes(b.style as string)) return null; + if (!Array.isArray(b.sources)) return null; + const sources: GenerateSource[] = []; + for (const s of b.sources) { + if (!s || typeof s !== "object") return null; + const x = s as Record<string, unknown>; + if (x.kind === "lesson" && str(x.lessonId)) sources.push({ kind: "lesson", lessonId: str(x.lessonId)! }); + else if (x.kind === "file" && str(x.path)) sources.push({ kind: "file", path: str(x.path)! }); + else if (x.kind === "cache" && str(x.path)) sources.push({ kind: "cache", path: str(x.path)! }); + else if (x.kind === "text" && typeof x.text === "string" && x.text.trim()) sources.push({ kind: "text", label: str(x.label) ?? "Pasted text", text: x.text }); + else return null; + } + return { courseId, topic: topic.slice(0, 200), kind: b.kind as GenerateRequest["kind"], style: b.style as GenerateRequest["style"], sources, instructions: typeof b.instructions === "string" ? b.instructions.slice(0, 8_000) : undefined }; +} + +const LISTABLE = new Set([".md", ".markdown", ".txt", ".js", ".mjs", ".ts", ".tsx", ".jsx", ".py", ".rs", ".scala", ".fs", ".go", ".java", ".sh", ".json", ".yaml", ".yml", ".sql", ".html", ".css", ".toml", ".vtt", ".srt"]); +async function listFiles(root: string, limit = 800, keep: (rel: string) => boolean = () => true): Promise<string[]> { + const out: string[] = []; + const walk = async (dir: string, depth: number) => { + if (depth > 8 || out.length >= limit) return; + let entries; + try { entries = await readdir(dir, { withFileTypes: true }); } catch { return; } + entries.sort((a, b) => a.name.localeCompare(b.name)); + for (const e of entries) { + if (out.length >= limit) return; + if (e.name.startsWith(".") || e.name === "node_modules" || e.name === "target" || e.name === "dist") continue; + const abs = join(dir, e.name); + if (e.isDirectory()) await walk(abs, depth + 1); + else if (e.isFile() && LISTABLE.has(extname(e.name).toLowerCase())) { + const rel = relative(root, abs).split(sep).join("/"); + if (keep(rel)) out.push(rel); + } + } + }; + await walk(root, 0); + return out; +} + +export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feature { + const settings = aiSettingsRepo(ctx.db, createSecretBox(ctx.config.secret)); + const acl = aclRepo(ctx.db); + const notes = notesRepo(ctx.db); + const sink = opts.draftSink ?? createFsDraftSink(ctx.config.dataDir); + const makeProvider = opts.providerFactory ?? ((cfg: ProviderConfig, key: string | null) => createProvider(cfg, key, opts.fetch)); + + /** BYOK first ("used only for their own requests"), then the server provider. */ + const resolveProvider = async (userId: string | null): Promise<{ source: "byok" | "server"; cfg: ProviderConfig; key: string | null } | null> => { + if (userId) { + const u = await settings.getUser(userId); + if (u.byok) { + const key = await settings.userApiKey(userId).catch(() => null); + if (key) return { source: "byok", cfg: u.byok, key }; + } + } + const s = await settings.getServer(); + if (!s.provider) return null; + if (s.provider.kind === "anthropic" && !s.provider.hasApiKey) return null; + const key = s.provider.hasApiKey ? await settings.serverApiKey().catch(() => null) : null; + if (s.provider.kind === "anthropic" && !key) return null; + return { source: "server", cfg: s.provider, key }; + }; + + const viewable = async (user: AuthUser | null, courseId: string): Promise<LoadedCourse | null> => { + const c = ctx.courses.get(courseId); + if (!c || c.status !== "ok") return null; + const ok = canView(user, c, await acl.listForCourse(c.id), user ? await acl.isAuthor(c.id, user.id) : false); + return ok ? c : null; + }; + const canAuthor = async (user: AuthUser, courseId: string) => + user.roles.includes("owner") || user.roles.includes("author") || (await acl.isAuthor(courseId, user.id)); + + const conversationOf = async (id: string, userId: string) => { + const rows = await ctx.db.sql`SELECT id, user_id, course_id, title, created_at FROM ai_conversations WHERE id = ${id} AND user_id = ${userId}`; + return rows[0] as { id: string; user_id: string; course_id: string; title: string | null; created_at: string } | undefined; + }; + const addMessage = async (conversationId: string, role: "user" | "assistant", content: string) => { + await ctx.db.sql`INSERT INTO ai_messages (id, conversation_id, role, content, created_at) VALUES (${Bun.randomUUIDv7()}, ${conversationId}, ${role}, ${content}, ${new Date().toISOString()})`; + }; + const messagesOf = async (conversationId: string) => + (await ctx.db.sql`SELECT id, role, content, created_at FROM ai_messages WHERE conversation_id = ${conversationId}`) + .map((r: { id: string; role: string; content: string; created_at: string }) => ({ id: r.id, role: r.role, content: r.content, createdAt: r.created_at })) + .sort((a: { id: string }, b: { id: string }) => (a.id < b.id ? -1 : 1)) as { id: string; role: "user" | "assistant"; content: string; createdAt: string }[]; + + const signedIn = (rc: RequestContext) => (rc.user ? null : err(401, "Sign in to use the assistant.")); + const ownerOnly = (rc: RequestContext) => (!rc.user ? err(401, "Sign in first.") : rc.user.roles.includes("owner") ? null : err(403, "Only the owner can change AI settings.")); + + const chat = async (req: Request, rc: RequestContext): Promise<Response> => { + const deny = signedIn(rc); + if (deny) return deny; + const user = rc.user!; + const course = await viewable(user, rc.params.courseId ?? ""); + if (!course) return err(404, "Course not found."); + const b = await body(req); + const message = typeof b?.message === "string" ? b.message.trim() : ""; + if (!message) return err(400, "Write a message."); + if (message.length > MAX_MESSAGE) return err(400, "That message is too long."); + const resolved = await resolveProvider(user.id); + if (!resolved) return err(409, "AI is not set up. An admin can configure a server provider, or you can add your own key under Account, AI."); + let provider: AiProvider; + try { provider = makeProvider(resolved.cfg, resolved.key); } catch (e) { return err(409, (e as Error).message); } + + let conversationId = str(b?.conversationId); + if (conversationId) { + const conv = await conversationOf(conversationId, user.id); + if (!conv || conv.course_id !== course.id) return err(404, "Conversation not found."); + } else { + conversationId = Bun.randomUUIDv7(); + await ctx.db.sql`INSERT INTO ai_conversations (id, user_id, course_id, title, created_at) VALUES (${conversationId}, ${user.id}, ${course.id}, ${message.slice(0, 80)}, ${new Date().toISOString()})`; + } + const history = (await messagesOf(conversationId)).slice(-HISTORY_TURNS); + await addMessage(conversationId, "user", message); + + const lessonId = str(b?.lessonId); + const lesson = lessonId ? course.lessonIndex.get(lessonId)?.lesson : undefined; + const index = await getCourseIndex(course, ctx.config.dataDir); + const userNotes = await notes.list(user.id, course.id); + const sources = index.search(lesson ? `${message} ${lesson.title}` : message, 6, noteChunks(course, userNotes)); + const mentorText = await readMentorText(ctx.courses, course); + const mentorOn = !(await settings.isMentorOff(user.id, course.id)); + const convId = conversationId; + + const stream = new ReadableStream<Uint8Array>({ + async start(controller) { + const enc = new TextEncoder(); + const send = (ev: unknown) => { try { controller.enqueue(enc.encode(JSON.stringify(ev) + "\n")); } catch {} }; + send({ type: "meta", conversationId: convId, mentorOn, mentorAvailable: !!mentorText, source: resolved.source, citations: citationsFor(sources) }); + let tools: ToolSet | null = null; + try { + const course_ = courseTools({ repo: ctx.courses, course, index, listNotes: (lid) => notes.list(user.id, course.id, lid) }); + const servers = mcpServersFor(course.id, (await settings.getServer()).mcpServers); + const mcp = servers.length ? await mcpTools(servers, { fetch: opts.fetch, timeoutMs: opts.mcpTimeoutMs }) : null; + for (const w of mcp?.warnings ?? []) send({ type: "notice", message: w }); + tools = combineTools(mcp ? [course_, mcp] : [course_]); + const messages: ChatMessage[] = [ + { role: "system", content: buildSystemPrompt({ course, mentorText, mentorOn, sources, lesson: lesson ? { id: lesson.id, title: lesson.title } : null, extraTools: mcp?.names }) }, + ...history.map((m) => ({ role: m.role, content: m.content }) as ChatMessage), + { role: "user", content: message }, + ]; + let answer = ""; + for await (const ev of runAssistant({ provider, messages, tools, signal: req.signal })) { + if (ev.type === "done") { + answer = ev.text; + send({ type: "done", text: ev.text, citations: citationsFor([...sources, ...tools.sources]) }); + } else send(ev); + } + if (answer) await addMessage(convId, "assistant", answer); + } catch (e) { + send({ type: "error", message: (e as Error).message }); + } finally { + await tools?.close?.().catch(() => {}); + try { controller.close(); } catch {} + } + }, + }); + return new Response(stream, { headers: { "content-type": "application/x-ndjson; charset=utf-8", "cache-control": "no-store" } }); + }; + + const generateJob = async (payload: unknown, jc: JobContext) => { + const p = payload as { req: GenerateRequest; userId: string }; + const resolved = await resolveProvider(p.userId); + if (!resolved) throw new Error("AI is not set up for this user."); + const provider = makeProvider(resolved.cfg, resolved.key); + return generateDraft({ provider, repo: ctx.courses, req: p.req, sink, authorId: p.userId, dataDir: ctx.config.dataDir, signal: jc.signal, onProgress: (f, m) => jc.reportProgress(f, m) }); + }; + + return { + name: "ai", + jobs: { "ai.generate": generateJob }, + routes: { + "/api/ai/status": { + GET: async (_req, rc) => { + const resolved = rc.user ? await resolveProvider(rc.user.id) : null; + const out: Record<string, unknown> = { enabled: !!resolved, source: resolved?.source ?? null, signedIn: !!rc.user }; + const courseId = rc.url.searchParams.get("courseId"); + if (courseId) { + const course = await viewable(rc.user, courseId); + if (course) { + out.course = { id: course.id, title: course.manifest?.title ?? course.id }; + out.mentor = { available: !!course.manifest?.mentor, on: rc.user ? !(await settings.isMentorOff(rc.user.id, course.id)) : true }; + } + } + return json(out); + }, + }, + "/api/ai/admin/config": { + GET: async (_req, rc) => ownerOnly(rc) ?? json(await settings.getServer()), + PUT: async (req, rc) => { + const deny = ownerOnly(rc); + if (deny) return deny; + const b = await body(req); + if (!b) return err(400, "Invalid JSON."); + const provider = parseProvider(b.provider); + const mcpServers = parseMcp(b.mcpServers); + if (provider === "invalid") return err(400, "Invalid provider. OpenAI-compatible endpoints need a base URL and a model."); + if (mcpServers === "invalid") return err(400, "Invalid MCP server list."); + await settings.saveServer({ provider, mcpServers }); + return json(await settings.getServer()); + }, + }, + "/api/ai/admin/mcp/test": { + POST: async (req, rc) => { + const deny = ownerOnly(rc); + if (deny) return deny; + const b = await body(req); + const parsed = parseMcp(b ? [b] : null); + if (parsed === "invalid" || !parsed.length) return err(400, "Invalid MCP server."); + try { + const c = await connectMcp(parsed[0]!, { fetch: opts.fetch, timeoutMs: opts.mcpTimeoutMs ?? 15_000 }); + try { return json({ ok: true, tools: (await c.tools()).map((t) => ({ name: t.name, description: t.description })) }); } finally { await c.close(); } + } catch (e) { + return json({ ok: false, error: (e as Error).message }); + } + }, + }, + "/api/ai/account": { + GET: async (_req, rc) => { + if (!rc.user) return err(401, "Sign in first."); + const u = await settings.getUser(rc.user.id); + const s = await settings.getServer(); + return json({ byok: u.byok, serverProvider: s.provider ? { kind: s.provider.kind, model: s.provider.model ?? null } : null }); + }, + }, + "/api/ai/account/key": { + PUT: async (req, rc) => { + if (!rc.user) return err(401, "Sign in first."); + const b = await body(req); + const p = b ? parseProvider(b) : "invalid"; + if (!p || p === "invalid" || !p.apiKey) return err(400, "Choose a provider and paste your key. OpenAI-compatible endpoints also need a base URL and a model."); + await settings.saveUserKey(rc.user.id, { ...p, apiKey: p.apiKey }); + return json((await settings.getUser(rc.user.id)).byok); + }, + DELETE: async (_req, rc) => { + if (!rc.user) return err(401, "Sign in first."); + await settings.removeUserKey(rc.user.id); + return json({ ok: true }); + }, + }, + "/api/ai/courses/:courseId/mentor": { + PUT: async (req, rc) => { + const deny = signedIn(rc); + if (deny) return deny; + const course = await viewable(rc.user, rc.params.courseId ?? ""); + if (!course) return err(404, "Course not found."); + const b = await body(req); + if (typeof b?.off !== "boolean") return err(400, "Send { off: true | false }."); + await settings.setMentorOff(rc.user!.id, course.id, b.off); + return json({ mentor: { available: !!course.manifest?.mentor, on: !b.off } }); + }, + }, + "/api/ai/courses/:courseId/chat": { POST: chat }, + "/api/ai/courses/:courseId/conversations": { + GET: async (_req, rc) => { + const deny = signedIn(rc); + if (deny) return deny; + const rows = await ctx.db.sql`SELECT id, title, created_at FROM ai_conversations WHERE user_id = ${rc.user!.id} AND course_id = ${rc.params.courseId ?? ""}`; + return json(rows.map((r: { id: string; title: string | null; created_at: string }) => ({ id: r.id, title: r.title, createdAt: r.created_at })).sort((a: { createdAt: string }, b: { createdAt: string }) => (a.createdAt < b.createdAt ? 1 : -1))); + }, + }, + "/api/ai/conversations/:id": { + GET: async (_req, rc) => { + const deny = signedIn(rc); + if (deny) return deny; + const conv = await conversationOf(rc.params.id ?? "", rc.user!.id); + if (!conv) return err(404, "Conversation not found."); + return json({ id: conv.id, courseId: conv.course_id, title: conv.title, messages: await messagesOf(conv.id) }); + }, + DELETE: async (_req, rc) => { + const deny = signedIn(rc); + if (deny) return deny; + const conv = await conversationOf(rc.params.id ?? "", rc.user!.id); + if (!conv) return err(404, "Conversation not found."); + await ctx.db.sql.begin(async (tx) => { + await tx`DELETE FROM ai_messages WHERE conversation_id = ${conv.id}`; + await tx`DELETE FROM ai_conversations WHERE id = ${conv.id}`; + }); + return json({ ok: true }); + }, + }, + "/api/ai/studio/courses": { + GET: async (_req, rc) => { + if (!rc.user) return err(401, "Sign in first."); + const out = []; + for (const c of ctx.courses.list()) { + if (c.status === "ok" && (await canAuthor(rc.user, c.id))) out.push({ id: c.id, title: c.manifest?.title ?? c.id, tracks: c.manifest?.tracks ?? [] }); + } + return json(out); + }, + }, + "/api/ai/studio/sources": { + GET: async (_req, rc) => { + if (!rc.user) return err(401, "Sign in first."); + const courseId = rc.url.searchParams.get("courseId") ?? ""; + const course = ctx.courses.get(courseId); + if (!course || course.status !== "ok") return err(404, "Course not found."); + if (!(await canAuthor(rc.user, course.id))) return err(403, "Only authors can generate drafts."); + const lessons = (course.manifest?.modules ?? []).flatMap((m) => m.lessons.map((l) => ({ id: l.id, title: l.title, type: l.type, module: m.title, hasText: l.type !== "video" || !!l.transcript }))); + const files = await listFiles(course.root, 800); // authors may use any course file, solutions included + const cache = await listFiles(join(ctx.config.dataDir, "cache", course.id), 400); + return json({ lessons, files, cache }); + }, + }, + "/api/ai/studio/generate": { + POST: async (req, rc) => { + if (!rc.user) return err(401, "Sign in first."); + const b = await body(req); + const g = b ? parseGenerate(b) : null; + if (!g) return err(400, "Choose a course, a kind (document, exercise or questions), a style and a topic."); + const course = ctx.courses.get(g.courseId); + if (!course || course.status !== "ok") return err(404, "Course not found."); + if (!(await canAuthor(rc.user, course.id))) return err(403, "Only authors can generate drafts."); + if (!(await resolveProvider(rc.user.id))) return err(409, "AI is not set up."); + const jobId = await ctx.jobs.enqueue("ai.generate", { req: g, userId: rc.user.id }); + return json({ jobId }, 202); + }, + }, + "/api/ai/studio/jobs/:id": { + GET: async (_req, rc) => { + if (!rc.user) return err(401, "Sign in first."); + const job = await ctx.jobs.get(rc.params.id ?? ""); + const owner = (job?.payload as { userId?: string } | null)?.userId; + if (!job || job.kind !== "ai.generate" || (owner !== rc.user.id && !rc.user.roles.includes("owner"))) return err(404, "Job not found."); + return json({ id: job.id, status: job.status, progress: job.progress, message: job.message, result: job.result, error: job.error }); + }, + }, + }, + }; +} diff --git a/packages/feature-ai/src/index.ts b/packages/feature-ai/src/index.ts index 75a46aa..62f5df7 100644 --- a/packages/feature-ai/src/index.ts +++ b/packages/feature-ai/src/index.ts @@ -13,3 +13,4 @@ export * from "./assistant/loop"; export * from "./generator/draft-sink"; export * from "./generator/generate"; export * from "./generator/templates"; +export { aiFeature, type AiFeatureOptions } from "./feature"; diff --git a/packages/feature-ai/src/retrieval/course-index.ts b/packages/feature-ai/src/retrieval/course-index.ts index 616c498..57af7fb 100644 --- a/packages/feature-ai/src/retrieval/course-index.ts +++ b/packages/feature-ai/src/retrieval/course-index.ts @@ -19,7 +19,9 @@ const MD = new Set([".md", ".markdown"]); const CODE = new Set([".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx", ".py", ".rs", ".scala", ".sc", ".fs", ".fsx", ".go", ".java", ".kt", ".rb", ".php", ".c", ".h", ".cpp", ".cs", ".sh", ".bash", ".ps1", ".sql", ".yaml", ".yml", ".toml", ".json", ".tf", ".html", ".css"]); const CODE_NAMES = new Set(["Dockerfile", "Makefile", "docker-compose.yml", "Containerfile"]); const CACHE_TEXT = new Set([".txt", ".md", ".vtt", ".srt"]); -const SKIP_FILES = new Set(["package-lock.json", "bun.lock", "yarn.lock", "pnpm-lock.yaml", "Cargo.lock"]); +// Lock files, the manifest, and instructions written for AI agents (the mentor file goes into the system +// prompt only while mentor mode is on, so it must not leak back in through retrieval). +const SKIP_FILES = new Set(["package-lock.json", "bun.lock", "yarn.lock", "pnpm-lock.yaml", "Cargo.lock", "stcourse.yaml", "AGENTS.md", "CLAUDE.md", "GEMINI.md"]); const MAX_FILE = 200_000; const MAX_FILES = 2_000; const MAX_CHUNK = 1_500; @@ -130,6 +132,7 @@ export async function buildCourseIndex(course: LoadedCourse, dataDir: string): P const bm = new Bm25<Chunk>(); const seen = new Set<string>(); const refs = lessonRefs(course); + const mentor = course.manifest?.mentor ? posix(course.manifest.mentor).replace(/^\.\//, "") : null; const codeDirs = (course.manifest?.modules ?? []).flatMap((m) => (m.code ? [posix(m.code.dir).replace(/\/$/, "") + "/"] : [])); const add = (c: Chunk) => { const key = `${c.kind}\u0000${c.text}`; @@ -146,7 +149,7 @@ export async function buildCourseIndex(course: LoadedCourse, dataDir: string): P const name = basename(rel); const isMd = MD.has(ext); const isCode = !isMd && (CODE.has(ext) || CODE_NAMES.has(name) || codeDirs.some((d) => rel.startsWith(d))); - if ((!isMd && !isCode) || SKIP_FILES.has(name) || isSolutionPath(course, rel)) return true; + if ((!isMd && !isCode) || SKIP_FILES.has(name) || rel === mentor || isSolutionPath(course, rel)) return true; let size = 0; try { size = (await stat(abs)).size; } catch { return true; } if (size > MAX_FILE || size === 0) return true; diff --git a/packages/feature-ai/test/feature.test.ts b/packages/feature-ai/test/feature.test.ts new file mode 100644 index 0000000..3912dfc --- /dev/null +++ b/packages/feature-ai/test/feature.test.ts @@ -0,0 +1,156 @@ +import { expect } from "bun:test"; +import { cpSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import type { AuthUser, JobRecord, ServerContext } from "@stc/contracts"; +import { createApp, createJobQueue } from "@stc/core"; +import { createCourseRepository } from "@stc/course-format"; +import type { Db } from "@stc/db"; +import { forEachDb } from "../../db/test/helpers"; +import { aiFeature } from "../src/feature"; +import { fakeProvider } from "./helpers"; + +const SAMPLE = resolve(import.meta.dir, "../../../fixtures/courses/sample"); +const USERS: Record<string, AuthUser> = { + owner: { id: "owner", email: "o@x", name: "Owner", roles: ["owner"], groups: [] }, + ann: { id: "ann", email: "a@x", name: "Ann", roles: ["learner"], groups: [] }, + bob: { id: "bob", email: "b@x", name: "Bob", roles: ["learner"], groups: [] }, + auth: { id: "auth", email: "w@x", name: "Writer", roles: ["author"], groups: [] }, +}; + +async function setup(db: Db, turns: Parameters<typeof fakeProvider>[0] = [() => ({ text: "ok" })]) { + const dir = mkdtempSync(join(tmpdir(), "stc-ai-feat-")); + cpSync(SAMPLE, join(dir, "courses/sample"), { recursive: true }); + cpSync(SAMPLE, join(dir, "courses/secret"), { recursive: true }); + const m = readFileSync(join(SAMPLE, "stcourse.yaml"), "utf8"); + writeFileSync(join(dir, "courses/secret/stcourse.yaml"), m.replace("id: sample", "id: secret").replace("visibility: public", "visibility: restricted")); + const courses = await createCourseRepository([join(dir, "courses")]); + const fake = fakeProvider(turns); + const keys: (string | null)[] = []; + const ctx = { + db, courses, + config: { port: 0, databaseUrl: "", dataDir: join(dir, "data"), courseDirs: [], secret: "test-secret", publicUrl: "" }, + auth: { userFromRequest: async (req: Request) => USERS[req.headers.get("x-user") ?? ""] ?? null }, + } as unknown as ServerContext; + const feature = aiFeature(ctx, { providerFactory: (_cfg, key) => { keys.push(key); return fake.provider; } }); + const jobs = createJobQueue(db, feature.jobs!); + ctx.jobs = jobs; + const app = createApp(ctx, [feature]); + const call = async (method: string, path: string, user?: string, body?: unknown) => { + const headers: Record<string, string> = {}; + if (user) headers["x-user"] = user; + if (body !== undefined) headers["content-type"] = "application/json"; + return app.fetch(new Request(`http://x${path}`, { method, headers, body: body === undefined ? undefined : JSON.stringify(body) })); + }; + return { ctx, dir, call, fake, keys, feature }; +} + +const ndjson = async (r: Response) => (await r.text()).trim().split("\n").map((l) => JSON.parse(l)); + +forEachDb("AI is off by default and admin config is owner-only and hides keys", async (db) => { + const { call } = await setup(db); + expect(await (await call("GET", "/api/ai/status?courseId=sample", "ann")).json()).toMatchObject({ enabled: false, source: null }); + expect((await call("POST", "/api/ai/courses/sample/chat", "ann", { message: "hi" })).status).toBe(409); + expect((await call("GET", "/api/ai/admin/config", "ann")).status).toBe(403); + expect((await call("GET", "/api/ai/admin/config")).status).toBe(401); + const put = await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "anthropic", apiKey: "sk-ant-SERVER" }, mcpServers: [] }); + expect(put.status).toBe(200); + const cfg = await (await call("GET", "/api/ai/admin/config", "owner")).text(); + expect(cfg).not.toContain("SERVER"); + expect(JSON.parse(cfg).provider).toEqual({ kind: "anthropic", hasApiKey: true }); + expect((await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "nope" }, mcpServers: [] })).status).toBe(400); + expect(await (await call("GET", "/api/ai/status?courseId=sample", "ann")).json()).toMatchObject({ enabled: true, source: "server", mentor: { available: true, on: true } }); +}); + +forEachDb("BYOK enables AI for that user only and is used before the server key", async (db) => { + const { call, keys } = await setup(db); + expect((await call("PUT", "/api/ai/account/key", "ann", { kind: "anthropic", apiKey: "sk-ann" })).status).toBe(200); + const acct = await (await call("GET", "/api/ai/account", "ann")).text(); + expect(acct).not.toContain("sk-ann"); + expect(JSON.parse(acct).byok).toEqual({ kind: "anthropic", hasApiKey: true }); + expect(await (await call("GET", "/api/ai/status?courseId=sample", "ann")).json()).toMatchObject({ enabled: true, source: "byok" }); + expect(await (await call("GET", "/api/ai/status?courseId=sample", "bob")).json()).toMatchObject({ enabled: false }); + await call("POST", "/api/ai/courses/sample/chat", "ann", { message: "hello" }).then((r) => r.text()); + expect(keys).toEqual(["sk-ann"]); + expect((await call("DELETE", "/api/ai/account/key", "ann")).status).toBe(200); + expect(await (await call("GET", "/api/ai/status?courseId=sample", "ann")).json()).toMatchObject({ enabled: false }); + expect((await call("PUT", "/api/ai/account/key", "ann", { kind: "anthropic" })).status).toBe(400); +}); + +forEachDb("chat cites course files, applies mentor rules, and stores the conversation", async (db) => { + const { call, fake } = await setup(db, [ + () => ({ toolCalls: [{ id: "t1", name: "read_lesson", input: { lessonId: "m2-ex-add" } }] }), + () => ({ text: "Which operator combines two numbers? See [lessons/04-exercise.md · Adding numbers]." }), + ]); + await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "anthropic", apiKey: "k" }, mcpServers: [] }); + const r = await call("POST", "/api/ai/courses/sample/chat", "ann", { message: "How do I return the sum of its arguments?" }); + expect(r.status).toBe(200); + expect(r.headers.get("content-type")).toContain("application/x-ndjson"); + const events = await ndjson(r); + expect(events[0]).toMatchObject({ type: "meta", mentorOn: true }); + expect(events[0].citations.map((c: { path: string }) => c.path)).toContain("lessons/04-exercise.md"); + expect(events.some((e) => e.type === "tool" && e.name === "read_lesson")).toBe(true); + const done = events.find((e) => e.type === "done"); + expect(done.text).toContain("Which operator"); + expect(done.citations.some((c: { path: string; lessonId?: string }) => c.path === "lessons/04-exercise.md")).toBe(true); + const system = fake.seen[0]!.messages[0]!; + expect(system.role).toBe("system"); + expect(system.content).toContain("Give hints, never full solutions."); + + const convs = await (await call("GET", "/api/ai/courses/sample/conversations", "ann")).json(); + expect(convs).toHaveLength(1); + const conv = await (await call("GET", `/api/ai/conversations/${convs[0].id}`, "ann")).json(); + expect(conv.messages.map((m: { role: string }) => m.role)).toEqual(["user", "assistant"]); + expect((await call("GET", `/api/ai/conversations/${convs[0].id}`, "bob")).status).toBe(404); + + // Learner switches mentor mode off for this course: rules leave the prompt. + expect((await call("PUT", "/api/ai/courses/sample/mentor", "ann", { off: true })).status).toBe(200); + expect(await (await call("GET", "/api/ai/status?courseId=sample", "ann")).json()).toMatchObject({ mentor: { available: true, on: false } }); + const r2 = await ndjson(await call("POST", "/api/ai/courses/sample/chat", "ann", { message: "Show me the full solution", conversationId: convs[0].id })); + expect(r2[0]).toMatchObject({ type: "meta", mentorOn: false, conversationId: convs[0].id }); + const sys2 = fake.seen.at(-1)!.messages[0]!.content; + expect(sys2).not.toContain("never full solutions"); + expect(sys2).toContain("Mentor mode is off"); + // history is replayed + expect(fake.seen.at(-1)!.messages.some((m) => m.role === "assistant" && m.content.includes("Which operator"))).toBe(true); + // Bob still has mentor mode on + expect(await (await call("GET", "/api/ai/status?courseId=sample", "bob")).json()).toMatchObject({ mentor: { on: true } }); + + expect((await call("DELETE", `/api/ai/conversations/${convs[0].id}`, "ann")).status).toBe(200); + expect(await (await call("GET", "/api/ai/courses/sample/conversations", "ann")).json()).toEqual([]); +}); + +forEachDb("course visibility and sign-in are enforced", async (db) => { + const { call } = await setup(db); + await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "anthropic", apiKey: "k" }, mcpServers: [] }); + expect((await call("POST", "/api/ai/courses/secret/chat", "ann", { message: "hi" })).status).toBe(404); + expect((await call("POST", "/api/ai/courses/sample/chat", undefined, { message: "hi" })).status).toBe(401); + expect((await call("POST", "/api/ai/courses/nope/chat", "ann", { message: "hi" })).status).toBe(404); + expect((await call("POST", "/api/ai/courses/secret/chat", "owner", { message: "hi" })).status).toBe(200); +}); + +forEachDb("generation is for authors and writes drafts, not course files", async (db) => { + const doc = "=== FILE: questions/adding.md ===\n## Check your understanding\n1. Predict: add(2, 3)?\n=== END FILE ==="; + const { call, ctx, dir } = await setup(db, [() => ({ text: doc })]); + await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "anthropic", apiKey: "k" }, mcpServers: [] }); + const body = { courseId: "sample", kind: "questions", style: "pure-snake", topic: "Adding", sources: [{ kind: "lesson", lessonId: "m2-text" }] }; + expect((await call("POST", "/api/ai/studio/generate", "ann", body)).status).toBe(403); + expect((await call("GET", "/api/ai/studio/sources?courseId=sample", "ann")).status).toBe(403); + const src = await (await call("GET", "/api/ai/studio/sources?courseId=sample", "auth")).json(); + expect(src.lessons.map((l: { id: string }) => l.id)).toContain("m2-text"); + expect(src.files).toContain("code/S01 - First steps/hello.sh"); + expect((await call("POST", "/api/ai/studio/generate", "auth", { ...body, kind: "poem" })).status).toBe(400); + const g = await call("POST", "/api/ai/studio/generate", "auth", body); + expect(g.status).toBe(202); + const { jobId } = await g.json(); + // run the queued job by hand (the worker is not started in tests) + const job = (await ctx.jobs.get(jobId)) as JobRecord; + expect(job.kind).toBe("ai.generate"); + const feature = aiFeature(ctx, { providerFactory: () => fakeProvider([() => ({ text: doc })]).provider }); + const result = (await feature.jobs!["ai.generate"]!(job.payload, { reportProgress: async () => {}, signal: new AbortController().signal })) as { location: string; paths: string[] }; + expect(result.paths).toEqual(["questions/adding.md"]); + expect(result.location.startsWith(join(dir, "data", "drafts", "sample"))).toBe(true); + expect(await Bun.file(join(dir, "courses/sample/questions/adding.md")).exists()).toBe(false); + expect((await call("GET", `/api/ai/studio/jobs/${jobId}`, "auth")).status).toBe(200); + expect((await call("GET", `/api/ai/studio/jobs/${jobId}`, "bob")).status).toBe(404); +}); diff --git a/packages/feature-ai/test/retrieval.test.ts b/packages/feature-ai/test/retrieval.test.ts index de15aef..96f51b9 100644 --- a/packages/feature-ai/test/retrieval.test.ts +++ b/packages/feature-ai/test/retrieval.test.ts @@ -69,6 +69,11 @@ test("solution files are never indexed", async () => { expect(isSolutionPath(course, "exercises/add/starter.js")).toBe(false); }); +test("the mentor file and the manifest are not retrievable", async () => { + const idx = await buildCourseIndex(course, dataDir); + expect(idx.search("mentor hints never full solutions", 50).some((c) => c.path === "MENTOR.md" || c.path === "stcourse.yaml")).toBe(false); +}); + test("notes are searched alongside the course", async () => { const idx = await getCourseIndex(course, dataDir); expect(await getCourseIndex(course, dataDir)).toBe(idx); From 34826762a80e240476689c3bdee2bea94412fd9d Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 03:25:18 +0100 Subject: [PATCH 35/62] feat(learner): section code listing and zip download Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- packages/feature-learner/src/server/code.ts | 81 +++++++++++++++++++ .../feature-learner/src/server/feature.ts | 2 + packages/feature-learner/src/server/zip.ts | 70 ++++++++++++++++ packages/feature-learner/test/code.test.ts | 58 +++++++++++++ 4 files changed, 211 insertions(+) create mode 100644 packages/feature-learner/src/server/code.ts create mode 100644 packages/feature-learner/src/server/zip.ts create mode 100644 packages/feature-learner/test/code.test.ts diff --git a/packages/feature-learner/src/server/code.ts b/packages/feature-learner/src/server/code.ts new file mode 100644 index 0000000..5d70353 --- /dev/null +++ b/packages/feature-learner/src/server/code.ts @@ -0,0 +1,81 @@ +import { lstatSync, readdirSync, readFileSync, realpathSync } from "node:fs"; +import { join, sep } from "node:path"; +import type { Feature, RequestContext, ServerContext } from "@stc/contracts"; +import { contentDisposition } from "@stc/core"; +import type { CodeListing } from "../shared/types"; +import { courseFor } from "./access"; +import { json, notFound, param } from "./http"; +import { zip } from "./zip"; + +const SKIP = new Set([".git", "node_modules", "__pycache__", ".DS_Store", ".venv", "target", "dist", ".idea", ".vscode"]); +export const MAX_FILES = 2000; +export const MAX_ZIP_BYTES = 200 * 1024 * 1024; + +/** + * Regular files under `dirAbs` (relative paths, sorted). Symlinks are followed only when their target is a + * regular file inside `rootReal`; symlinked directories are skipped, so there are no cycles and no escapes. + */ +export function listFiles(rootReal: string, dirAbs: string, max = MAX_FILES): { files: { path: string; size: number; abs: string }[]; truncated: boolean } { + const files: { path: string; size: number; abs: string }[] = []; + let truncated = false; + const walk = (abs: string, rel: string) => { + let entries; + try { entries = readdirSync(abs, { withFileTypes: true }); } catch { return; } + entries.sort((a, b) => a.name.localeCompare(b.name)); + for (const e of entries) { + if (SKIP.has(e.name)) continue; + if (files.length >= max) { truncated = true; return; } + const childAbs = join(abs, e.name), childRel = rel ? `${rel}/${e.name}` : e.name; + if (e.isDirectory()) { walk(childAbs, childRel); continue; } + let real = childAbs; + if (e.isSymbolicLink()) { + try { real = realpathSync(childAbs); } catch { continue; } + if (!(real === rootReal || real.startsWith(rootReal + sep))) continue; + } + try { + const st = lstatSync(real); + if (st.isFile()) files.push({ path: childRel, size: st.size, abs: real }); + } catch { /* vanished */ } + } + }; + walk(dirAbs, ""); + return { files, truncated }; +} + +export function codeRoutes(ctx: ServerContext): Feature["routes"] { + async function resolve(rc: RequestContext) { + const found = await courseFor(ctx, rc.user, param(rc, "courseId")); + if (!found) return null; + const mod = found.manifest.modules.find((m) => m.id === param(rc, "moduleId")); + if (!mod?.code) return null; + const dirAbs = ctx.courses.resolvePath(found.manifest.id, mod.code.dir); + if (!dirAbs) return null; + return { found, mod, code: mod.code, dirAbs, rootReal: realpathSync(found.course.root) }; + } + + return { + "/api/courses/:courseId/code/:moduleId": { + GET: async (_req, rc) => { + const r = await resolve(rc); + if (!r) return notFound(); + const { files, truncated } = listFiles(r.rootReal, r.dirAbs); + const listing: CodeListing = { moduleId: r.mod.id, dir: r.code.dir, command: r.code.command ?? null, files: files.map(({ path, size }) => ({ path, size })), truncated }; + return json(listing); + }, + }, + "/api/courses/:courseId/code/:moduleId/zip": { + GET: async (_req, rc) => { + const r = await resolve(rc); + if (!r) return notFound(); + const { files } = listFiles(r.rootReal, r.dirAbs, 20_000); + const total = files.reduce((s, f) => s + f.size, 0); + if (total > MAX_ZIP_BYTES) return json({ error: "this folder is too large to download as one zip" }, 413); + const folder = r.code.dir.split("/").filter(Boolean).pop() ?? r.mod.id; + const data = zip(files.map((f) => ({ name: `${folder}/${f.path}`, data: readFileSync(f.abs), mtime: lstatSync(f.abs).mtime }))); + return new Response(data, { + headers: { "Content-Type": "application/zip", "Content-Length": String(data.length), "Content-Disposition": contentDisposition(`${folder}.zip`) }, + }); + }, + }, + }; +} diff --git a/packages/feature-learner/src/server/feature.ts b/packages/feature-learner/src/server/feature.ts index 0a857c0..a13c159 100644 --- a/packages/feature-learner/src/server/feature.ts +++ b/packages/feature-learner/src/server/feature.ts @@ -6,6 +6,7 @@ import { adminRoutes } from "./admin"; import { libraryRoutes } from "./catalog"; import { lessonRoutes } from "./lesson"; import { userDataRoutes } from "./userdata"; +import { codeRoutes } from "./code"; /** * The learner feature: auth, library, reader, notes, progress and exercises. @@ -28,6 +29,7 @@ export function learnerFeature(ctx: ServerContext): Feature & { auth: AuthServic ...libraryRoutes(ctx), ...lessonRoutes(ctx), ...userDataRoutes(ctx), + ...codeRoutes(ctx), }; // Every handler waits for the learner migrations (a no-op once Phase 2 runs them in migrate()). for (const methods of Object.values(routes)) { diff --git a/packages/feature-learner/src/server/zip.ts b/packages/feature-learner/src/server/zip.ts new file mode 100644 index 0000000..42f997e --- /dev/null +++ b/packages/feature-learner/src/server/zip.ts @@ -0,0 +1,70 @@ +/** Minimal ZIP writer (PKWARE APPNOTE 6.3): deflate or store per entry, CRC-32, UTF-8 names (flag bit 11). No ZIP64. */ +export interface ZipEntry { name: string; data: Uint8Array; mtime?: Date } + +function dosTime(d: Date): { time: number; date: number } { + const y = Math.max(1980, d.getFullYear()); + return { + time: (d.getHours() << 11) | (d.getMinutes() << 5) | Math.floor(d.getSeconds() / 2), + date: ((y - 1980) << 9) | ((d.getMonth() + 1) << 5) | d.getDate(), + }; +} + +export function zip(entries: ZipEntry[]): Uint8Array { + const enc = new TextEncoder(); + const locals: Uint8Array[] = []; + const centrals: Uint8Array[] = []; + let offset = 0; + for (const e of entries) { + const name = enc.encode(e.name); + const crc = Bun.hash.crc32(e.data) >>> 0; + const deflated = Bun.deflateSync(e.data); + const useDeflate = deflated.length < e.data.length; + const body = useDeflate ? deflated : e.data; + const { time, date } = dosTime(e.mtime ?? new Date()); + const local = new Uint8Array(30 + name.length); + const lv = new DataView(local.buffer); + lv.setUint32(0, 0x04034b50, true); + lv.setUint16(4, 20, true); + lv.setUint16(6, 0x0800, true); // UTF-8 names + lv.setUint16(8, useDeflate ? 8 : 0, true); + lv.setUint16(10, time, true); + lv.setUint16(12, date, true); + lv.setUint32(14, crc, true); + lv.setUint32(18, body.length, true); + lv.setUint32(22, e.data.length, true); + lv.setUint16(26, name.length, true); + lv.setUint16(28, 0, true); + local.set(name, 30); + const central = new Uint8Array(46 + name.length); + const cv = new DataView(central.buffer); + cv.setUint32(0, 0x02014b50, true); + cv.setUint16(4, 0x0314, true); // made by: unix, 2.0 + cv.setUint16(6, 20, true); + cv.setUint16(8, 0x0800, true); + cv.setUint16(10, useDeflate ? 8 : 0, true); + cv.setUint16(12, time, true); + cv.setUint16(14, date, true); + cv.setUint32(16, crc, true); + cv.setUint32(20, body.length, true); + cv.setUint32(24, e.data.length, true); + cv.setUint16(28, name.length, true); + cv.setUint32(38, (0o100644 << 16) >>> 0, true); // external attrs: regular file rw-r--r-- + cv.setUint32(42, offset, true); + central.set(name, 46); + locals.push(local, body); + centrals.push(central); + offset += local.length + body.length; + } + const cdSize = centrals.reduce((s, c) => s + c.length, 0); + const end = new Uint8Array(22); + const ev = new DataView(end.buffer); + ev.setUint32(0, 0x06054b50, true); + ev.setUint16(8, entries.length, true); + ev.setUint16(10, entries.length, true); + ev.setUint32(12, cdSize, true); + ev.setUint32(16, offset, true); + const out = new Uint8Array(offset + cdSize + 22); + let p = 0; + for (const part of [...locals, ...centrals, end]) { out.set(part, p); p += part.length; } + return out; +} diff --git a/packages/feature-learner/test/code.test.ts b/packages/feature-learner/test/code.test.ts new file mode 100644 index 0000000..858a06f --- /dev/null +++ b/packages/feature-learner/test/code.test.ts @@ -0,0 +1,58 @@ +import { expect, test } from "bun:test"; +import { mkdtempSync, symlinkSync, writeFileSync, mkdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { zip } from "../src/server/zip"; +import { forEachDb } from "./helpers"; +import { FIXTURES, makeApp } from "./app"; + +async function unzipList(bytes: Uint8Array): Promise<string> { + const dir = mkdtempSync(join(tmpdir(), "stc-zip-")); + await Bun.write(join(dir, "a.zip"), bytes); + const out = await Bun.$`unzip -l ${join(dir, "a.zip")}`.text(); + await Bun.$`unzip -o -q ${join(dir, "a.zip")} -d ${join(dir, "x")}`; + return out + "\n" + (await Bun.$`cd ${join(dir, "x")} && find . -type f -exec cat {} +`.text()); +} + +test("zip round-trips through unzip with non-ASCII names", async () => { + const big = new TextEncoder().encode("abc".repeat(1000)); + const listing = await unzipList(zip([ + { name: "S08 - Containerizing Applications/Æble notes.txt", data: new TextEncoder().encode("hej") }, + { name: "S08 - Containerizing Applications/big.txt", data: big }, + ])); + expect(listing).toContain("S08 - Containerizing Applications/Æble notes.txt"); + expect(listing).toContain("hej"); + expect(listing).toContain("abcabc"); +}); + +forEachDb("code listing and zip for a module, skipping escaping symlinks", async (db) => { + const root = mkdtempSync(join(tmpdir(), "stc-code-")); + const outside = mkdtempSync(join(tmpdir(), "stc-outside-")); + writeFileSync(join(outside, "secret.txt"), "TOP SECRET"); + mkdirSync(join(root, "c/S01 - Ünïcode/sub"), { recursive: true }); + mkdirSync(join(root, "c/S01 - Ünïcode/node_modules"), { recursive: true }); + writeFileSync(join(root, "c/S01 - Ünïcode/run.sh"), "echo hi"); + writeFileSync(join(root, "c/S01 - Ünïcode/sub/app.py"), "print(1)"); + writeFileSync(join(root, "c/S01 - Ünïcode/node_modules/x.js"), "x"); + symlinkSync(join(outside, "secret.txt"), join(root, "c/S01 - Ünïcode/leak.txt")); + symlinkSync(join(root, "c/S01 - Ünïcode/run.sh"), join(root, "c/S01 - Ünïcode/alias.sh")); + writeFileSync(join(root, "stcourse.yaml"), `schema: 1\nid: codey\ntitle: Codey\nvisibility: public\nmodules:\n - id: m1\n title: M\n code: { dir: "c/S01 - Ünïcode", command: "sh run.sh" }\n lessons: [{ id: l1, title: L, type: exercise }]\n`); + const t = await makeApp(db, [FIXTURES, root]); + + const list = await (await t.call("GET", "/api/courses/codey/code/m1")).json(); + expect(list.files.map((f: { path: string }) => f.path)).toEqual(["alias.sh", "run.sh", "sub/app.py"]); + expect(list.command).toBe("sh run.sh"); + + const r = await t.call("GET", "/api/courses/codey/code/m1/zip"); + expect(r.status).toBe(200); + expect(r.headers.get("content-type")).toBe("application/zip"); + expect(r.headers.get("content-disposition")).toContain("filename*=UTF-8''S01%20-%20%C3%9Cn%C3%AFcode.zip"); + const listing = await unzipList(new Uint8Array(await r.arrayBuffer())); + expect(listing).toContain("S01 - Ünïcode/sub/app.py"); + expect(listing).not.toContain("TOP SECRET"); + expect(listing).not.toContain("node_modules"); + + expect((await t.call("GET", "/api/courses/codey/code/nope")).status).toBe(404); + expect((await t.call("GET", "/api/courses/sample/code/m1")).status).toBe(200); + expect((await t.call("GET", "/api/courses/sample/code/m2")).status).toBe(404); +}, 30_000); From 752e44696f7d7f0ed6d8ffa942dbb92a3c74a123 Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 03:25:30 +0100 Subject: [PATCH 36/62] fix(learner): zip buffer types Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- packages/feature-learner/src/server/code.ts | 2 +- packages/feature-learner/src/server/zip.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/feature-learner/src/server/code.ts b/packages/feature-learner/src/server/code.ts index 5d70353..27c64a3 100644 --- a/packages/feature-learner/src/server/code.ts +++ b/packages/feature-learner/src/server/code.ts @@ -71,7 +71,7 @@ export function codeRoutes(ctx: ServerContext): Feature["routes"] { const total = files.reduce((s, f) => s + f.size, 0); if (total > MAX_ZIP_BYTES) return json({ error: "this folder is too large to download as one zip" }, 413); const folder = r.code.dir.split("/").filter(Boolean).pop() ?? r.mod.id; - const data = zip(files.map((f) => ({ name: `${folder}/${f.path}`, data: readFileSync(f.abs), mtime: lstatSync(f.abs).mtime }))); + const data = zip(files.map((f) => ({ name: `${folder}/${f.path}`, data: new Uint8Array(readFileSync(f.abs)), mtime: lstatSync(f.abs).mtime }))); return new Response(data, { headers: { "Content-Type": "application/zip", "Content-Length": String(data.length), "Content-Disposition": contentDisposition(`${folder}.zip`) }, }); diff --git a/packages/feature-learner/src/server/zip.ts b/packages/feature-learner/src/server/zip.ts index 42f997e..d69671b 100644 --- a/packages/feature-learner/src/server/zip.ts +++ b/packages/feature-learner/src/server/zip.ts @@ -1,5 +1,5 @@ /** Minimal ZIP writer (PKWARE APPNOTE 6.3): deflate or store per entry, CRC-32, UTF-8 names (flag bit 11). No ZIP64. */ -export interface ZipEntry { name: string; data: Uint8Array; mtime?: Date } +export interface ZipEntry { name: string; data: Uint8Array<ArrayBuffer>; mtime?: Date } function dosTime(d: Date): { time: number; date: number } { const y = Math.max(1980, d.getFullYear()); @@ -9,9 +9,9 @@ function dosTime(d: Date): { time: number; date: number } { }; } -export function zip(entries: ZipEntry[]): Uint8Array { +export function zip(entries: ZipEntry[]): Uint8Array<ArrayBuffer> { const enc = new TextEncoder(); - const locals: Uint8Array[] = []; + const locals: Uint8Array<ArrayBuffer>[] = []; const centrals: Uint8Array[] = []; let offset = 0; for (const e of entries) { From 0992df7257d7c2466635a0e8fffce9a9da5cf2bd Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 03:26:18 +0100 Subject: [PATCH 37/62] feat(studio): studio web UI (course list, manifest, lessons with live preview, decks, attachments, exercises, conflicts, sync) and dev entry Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- apps/web/src/features.ts | 1 + packages/feature-studio/dev/serve.ts | 35 ++++ .../feature-studio/src/ui/CourseStudio.tsx | 163 +++++++++++++++++ packages/feature-studio/src/ui/StudioHome.tsx | 119 +++++++++++++ packages/feature-studio/src/ui/api.ts | 58 ++++++ .../src/ui/editors/CourseSettings.tsx | 85 +++++++++ .../src/ui/editors/GitPanels.tsx | 123 +++++++++++++ .../src/ui/editors/LessonEditor.tsx | 168 ++++++++++++++++++ .../feature-studio/src/ui/editors/Media.tsx | 87 +++++++++ .../feature-studio/src/ui/editors/common.tsx | 131 ++++++++++++++ packages/feature-studio/src/ui/markdown.ts | 129 ++++++++++++++ packages/feature-studio/src/ui/preview.tsx | 28 +++ packages/feature-studio/src/ui/studio.css | 98 ++++++++++ packages/feature-studio/src/ui/workspace.ts | 110 ++++++++++++ packages/feature-studio/src/web.ts | 12 ++ packages/feature-studio/test/app-helpers.ts | 17 +- packages/feature-studio/test/feature.test.ts | 6 +- packages/feature-studio/test/markdown.test.ts | 38 ++++ 18 files changed, 1403 insertions(+), 5 deletions(-) create mode 100644 packages/feature-studio/dev/serve.ts create mode 100644 packages/feature-studio/src/ui/CourseStudio.tsx create mode 100644 packages/feature-studio/src/ui/StudioHome.tsx create mode 100644 packages/feature-studio/src/ui/api.ts create mode 100644 packages/feature-studio/src/ui/editors/CourseSettings.tsx create mode 100644 packages/feature-studio/src/ui/editors/GitPanels.tsx create mode 100644 packages/feature-studio/src/ui/editors/LessonEditor.tsx create mode 100644 packages/feature-studio/src/ui/editors/Media.tsx create mode 100644 packages/feature-studio/src/ui/editors/common.tsx create mode 100644 packages/feature-studio/src/ui/markdown.ts create mode 100644 packages/feature-studio/src/ui/preview.tsx create mode 100644 packages/feature-studio/src/ui/studio.css create mode 100644 packages/feature-studio/src/ui/workspace.ts create mode 100644 packages/feature-studio/src/web.ts create mode 100644 packages/feature-studio/test/markdown.test.ts diff --git a/apps/web/src/features.ts b/apps/web/src/features.ts index 8b077c9..cc95268 100644 --- a/apps/web/src/features.ts +++ b/apps/web/src/features.ts @@ -2,3 +2,4 @@ import type { WebFeature } from "@stc/contracts"; // Phase 1 packages append one import line each and push into this list. export const webFeatures: WebFeature[] = []; +import { studioWeb } from "../../../packages/feature-studio/src/web"; webFeatures.push(studioWeb); diff --git a/packages/feature-studio/dev/serve.ts b/packages/feature-studio/dev/serve.ts new file mode 100644 index 0000000..ef20f0e --- /dev/null +++ b/packages/feature-studio/dev/serve.ts @@ -0,0 +1,35 @@ +/** + * Dev server for the studio until Phase 2 lets apps/server compose ctx-dependent features + * (docs/contract-requests/studio.md). Mirrors startServer, adds the studio feature and a dev-only user. + * + * STC_PORT=3463 STC_DATA_DIR=/tmp/stc-studio-data STC_DEV_USER=author bun packages/feature-studio/dev/serve.ts + * + * STC_DEV_USER: "author" | "owner" | "learner" | unset (anonymous). Never use this entry in production. + */ +import type { AuthUser, ServerContext } from "@stc/contracts"; +import { createApp, createJobQueue, loadConfig } from "@stc/core"; +import { createCourseRepository } from "@stc/course-format"; +import { connect, migrate } from "@stc/db"; +import indexHtml from "../../../apps/web/index.html"; +import { studioCourseDirs, studioFeature } from "../src"; + +const config = loadConfig(); +if (process.env.NODE_ENV === "production") throw new Error("dev/serve.ts is for development only"); +const role = process.env.STC_DEV_USER as AuthUser["roles"][number] | undefined; +const devUser: AuthUser | null = role ? { id: `dev-${role}`, name: `Dev ${role[0]!.toUpperCase()}${role.slice(1)}`, email: `${role}@dev.invalid`, roles: [role], groups: [] } : null; + +const db = await connect(config.databaseUrl); +await migrate(db); +const courses = await createCourseRepository(studioCourseDirs(config)); +const jobs = createJobQueue(db, {}); +const ctx: ServerContext = { config, db, courses, jobs, auth: { userFromRequest: async () => devUser } }; +const studio = studioFeature(ctx); +const app = createApp(ctx, [studio]); +jobs.start(); +const server = Bun.serve({ + port: config.port, + routes: { ...app.routes, "/api/*": (req) => app.fetch(req), "/*": indexHtml }, + fetch: (req) => app.fetch(req), + development: { hmr: false, console: true }, +}); +console.log(`STCourses studio dev server on ${server.url} as ${devUser ? devUser.name : "anonymous"}`); diff --git a/packages/feature-studio/src/ui/CourseStudio.tsx b/packages/feature-studio/src/ui/CourseStudio.tsx new file mode 100644 index 0000000..c5b1cbb --- /dev/null +++ b/packages/feature-studio/src/ui/CourseStudio.tsx @@ -0,0 +1,163 @@ +import { useState, type FormEvent } from "react"; +import { Link, useParams, useSearchParams } from "react-router-dom"; +import { CourseCover, Pill } from "@stc/ui"; +import { api, coursePath, uniqueId, type Diagnostic, type ManifestLesson } from "./api"; +import { AccessMessage } from "./StudioHome"; +import { clone, useWorkspace, type Workspace } from "./workspace"; +import { CourseSettings } from "./editors/CourseSettings"; +import { LessonEditor } from "./editors/LessonEditor"; +import { AttachmentsEditor, DecksEditor } from "./editors/Media"; +import { ConflictView, RawManifest, SyncPanel } from "./editors/GitPanels"; +import { Field } from "./editors/common"; +import "./studio.css"; + +type Notice = { kind: "ok" | "warn" | "bad"; text: string; files?: string[]; diagnostics?: Diagnostic[]; offerUpdate?: boolean }; + +export default function CourseStudio() { + const { courseId = "" } = useParams(); + const { ws, readOnly, error } = useWorkspace(courseId); + + if (error) return <div className="studio studio-page"><p className="chap"><Link to="/studio">Studio</Link></p>{error.status === 404 ? <div className="studio-note studio-note--bad"><strong>No such course.</strong></div> : <AccessMessage {...error} />}</div>; + if (readOnly) { + return ( + <div className="studio studio-page"> + <p className="chap"><Link to="/studio">Studio</Link></p> + <h1>{readOnly.title}</h1> + <div className="studio-note studio-note--warn" role="note"><strong>{readOnly.message}.</strong> It is read-only here; edit it in its own repository at <code>{readOnly.root}</code>.</div> + </div> + ); + } + if (!ws) return <div className="studio studio-page"><p className="faint">Loading…</p></div>; + return <Workbench ws={ws} />; +} + +function Workbench({ ws }: { ws: Workspace }) { + const [params, setParams] = useSearchParams(); + const view = ws.state.status.merging ? "conflicts" : params.get("view") ?? "settings"; + const lessonId = params.get("lesson"); + const go = (v: string, lesson?: string) => { + if (ws.docDirty && !window.confirm("You have unsaved changes to the course structure. Leave them unsaved for now?")) return; + setParams(lesson ? { view: v, lesson } : { view: v }); + }; + const [notice, setNotice] = useState<Notice | null>(null); + const [busy, setBusy] = useState(false); + const st = ws.state.status; + const hasErrors = ws.state.manifestErrors.length > 0 || ws.state.diagnostics.some((d) => d.level === "error"); + + const publish = async () => { + setBusy(true); + const { status, data } = await api("POST", coursePath(ws.courseId, "/publish")); + setBusy(false); + if (status === 200) setNotice(data.published ? { kind: "ok", text: "Published. Learners now see this version." } : { kind: "ok", text: "Nothing new to publish." }); + else if (data?.reason === "conflict") setNotice({ kind: "warn", text: "Not published: the published version was changed outside your draft, in the same places you edited. Nothing was overwritten. Update your draft, resolve the conflicts, then publish.", files: data.conflicts, offerUpdate: true }); + else if (data?.reason === "dirty") setNotice({ kind: "warn", text: "Not published: someone has unsaved edits in the published folder on the server. Ask them to commit or discard them first.", files: data.files }); + else if (data?.reason === "invalid") setNotice({ kind: "bad", text: "Not published: the draft has errors.", diagnostics: data.diagnostics }); + else setNotice({ kind: "bad", text: data?.error ?? `Publish failed (${status})` }); + await ws.reload(); + }; + const updateDraft = async () => { + setBusy(true); + const { status, data } = await api("POST", coursePath(ws.courseId, "/update-draft"), { from: "published" }); + setBusy(false); + setNotice(status === 200 ? { kind: "ok", text: "Your draft now includes the latest published changes." } : status === 409 && data?.reason === "conflict" ? null : { kind: "bad", text: data?.message ?? data?.error }); + await ws.reload(); + }; + + return ( + <div className="studio studio-ws"> + <aside className="studio-aside" aria-label="Course outline"> + <div className="studio-side-head" style={{ display: "flex", gap: 12 }}> + <CourseCover title={ws.doc.title} accent={ws.doc.accent} /> + <div style={{ minWidth: 0 }}> + <h2>{ws.doc.title}</h2> + <div className="src"><Link to="/studio">Studio</Link> · <code>{ws.courseId}</code></div> + </div> + </div> + <ul className="studio-nav"> + {[["settings", "Course settings"], ["decks", "Deck links"], ["attachments", "Attachments"], ["manifest", "stcourse.yaml"], ["sync", "Sync"]].map(([v, label]) => ( + <li key={v}><button type="button" aria-current={view === v ? "page" : undefined} onClick={() => go(v!)}>{label}{v === "sync" && st.remote ? <span className="k">remote</span> : null}</button></li> + ))} + {st.merging ? <li><button type="button" aria-current="page">Conflicts <span className="k">{st.conflicts.length}</span></button></li> : null} + </ul> + {ws.doc.modules.map((m) => ( + <ul className="studio-nav" key={m.id} aria-label={m.title}> + <li className="mod">{m.title}</li> + {m.lessons.map((l) => ( + <li key={l.id}> + <button type="button" aria-current={view === "lesson" && lessonId === l.id ? "page" : undefined} onClick={() => go("lesson", l.id)}> + <span>{l.title}</span><span className="k">{l.type === "text" ? "" : l.type}</span> + </button> + </li> + ))} + <li><AddLesson ws={ws} moduleId={m.id} onAdded={(id) => setParams({ view: "lesson", lesson: id })} /></li> + </ul> + ))} + </aside> + + <div className="studio-content"> + <div className="studio-bar" role="region" aria-label="Draft status"> + <div className="where"> + Editing your draft <b><code>{st.branch}</code></b> + {" · "}{st.ahead ? <b>{st.ahead} unpublished {st.ahead === 1 ? "change" : "changes"}</b> : "nothing unpublished"} + {st.behind ? <> · <b>{st.behind} new on published</b> since your draft</> : null} + {ws.docDirty ? <> · <b>unsaved structure changes</b></> : null} + </div> + {st.behind && !st.merging ? <Pill disabled={busy} onClick={updateDraft}>Update draft</Pill> : null} + <Pill variant="primary" disabled={busy || st.merging || !st.ahead || hasErrors} title={hasErrors ? "Fix the errors in the draft first" : undefined} onClick={publish}>Publish</Pill> + </div> + + {notice ? ( + <div className={`studio-note studio-note--${notice.kind}`} role="status"> + {notice.text} + {notice.files?.length ? <ul className="studio-errors" style={{ color: "var(--text)" }}>{notice.files.map((f) => <li key={f}>{f}</li>)}</ul> : null} + {notice.diagnostics?.length ? <ul className="studio-errors">{notice.diagnostics.map((d, i) => <li key={i}>{d.lessonId ? `${d.lessonId}: ` : ""}{d.message}</li>)}</ul> : null} + {notice.offerUpdate ? <div className="studio-actions" style={{ marginTop: 8 }}><Pill onClick={updateDraft}>Update draft and resolve</Pill></div> : null} + </div> + ) : null} + + {view === "conflicts" ? <ConflictView ws={ws} /> + : view === "lesson" && lessonId ? <LessonEditor key={lessonId} ws={ws} lessonId={lessonId} onGone={() => setParams({ view: "settings" })} /> + : view === "decks" ? <DecksEditor ws={ws} /> + : view === "attachments" ? <AttachmentsEditor ws={ws} /> + : view === "manifest" ? <RawManifest ws={ws} /> + : view === "sync" ? <SyncPanel ws={ws} /> + : <CourseSettings ws={ws} />} + </div> + </div> + ); +} + +/** Adds a lesson: manifest entry plus its Markdown file, saved in one commit. */ +function AddLesson({ ws, moduleId, onAdded }: { ws: Workspace; moduleId: string; onAdded: (id: string) => void }) { + const [open, setOpen] = useState(false); + const [title, setTitle] = useState(""); + const [type, setType] = useState<ManifestLesson["type"]>("text"); + const [err, setErr] = useState<string | null>(null); + if (!open) return <button type="button" className="add" onClick={() => setOpen(true)}>+ Add lesson</button>; + const submit = async (e: FormEvent) => { + e.preventDefault(); + const taken = new Set(ws.doc.modules.flatMap((m) => m.lessons.map((l) => l.id))); + const id = uniqueId(title, taken); + const doc = clone(ws.doc); + const lesson: ManifestLesson = type === "video" + ? { id, title: title.trim(), type, file: `media/${id}.mp4`, transcript: `lessons/${id}.md` } + : { id, title: title.trim(), type, file: `lessons/${id}.md` }; + doc.modules.find((m) => m.id === moduleId)!.lessons.push(lesson); + const r = await ws.save({ manifest: doc, files: [{ path: `lessons/${id}.md`, content: `# ${title.trim()}\n\n`, baseHash: null }], message: `Add lesson ${id}` }); + if (!r.ok) return setErr(r.error ?? "A file with that name already exists"); + setOpen(false); setTitle(""); + onAdded(id); + }; + return ( + <form onSubmit={submit} style={{ display: "grid", gap: 8, padding: "6px 16px 10px" }}> + <Field label="Lesson title"><input type="text" autoFocus required value={title} onChange={(e) => setTitle(e.target.value)} /></Field> + <Field label="Type"> + <select value={type} onChange={(e) => setType(e.target.value as ManifestLesson["type"])}> + <option value="text">Text</option><option value="video">Video</option><option value="exercise">Exercise</option> + </select> + </Field> + {err ? <span className="studio-errors" role="alert">{err}</span> : null} + <div className="studio-actions"><Pill type="submit" variant="primary" disabled={!title.trim()}>Add</Pill><Pill variant="ghost" onClick={() => setOpen(false)}>Cancel</Pill></div> + </form> + ); +} diff --git a/packages/feature-studio/src/ui/StudioHome.tsx b/packages/feature-studio/src/ui/StudioHome.tsx new file mode 100644 index 0000000..0a68b8c --- /dev/null +++ b/packages/feature-studio/src/ui/StudioHome.tsx @@ -0,0 +1,119 @@ +import { useEffect, useState, type FormEvent } from "react"; +import { Link, useNavigate } from "react-router-dom"; +import { CourseCover, Pill } from "@stc/ui"; +import { api, slug } from "./api"; +import "./studio.css"; + +interface CourseRow { + id: string; title: string; pitch: string | null; accent: string | null; managed: boolean; publishedStatus: "ok" | "error"; + ahead?: number; behind?: number; merging?: boolean; remote?: string | null; message?: string; +} + +export function AccessMessage({ status, error }: { status: number; error?: string }) { + if (status === 401) return <div className="studio-note studio-note--warn"><strong>Sign in to use the studio.</strong> The studio is for course authors and the owner.</div>; + if (status === 403) return <div className="studio-note studio-note--warn"><strong>The studio is for authors.</strong> Ask the owner for author access.</div>; + return <div className="studio-note studio-note--bad"><strong>Something went wrong.</strong> {error}</div>; +} + +export default function StudioHome() { + const [rows, setRows] = useState<CourseRow[] | null>(null); + const [canCreate, setCanCreate] = useState(false); + const [err, setErr] = useState<{ status: number; error?: string } | null>(null); + const [title, setTitle] = useState(""); + const [id, setId] = useState(""); + const [idTouched, setIdTouched] = useState(false); + const [createErr, setCreateErr] = useState<string | null>(null); + const [busy, setBusy] = useState(false); + const nav = useNavigate(); + + useEffect(() => { + api<{ courses: CourseRow[]; canCreate: boolean; error?: string }>("GET", "/api/studio/courses").then(({ status, data }) => { + if (status !== 200) return setErr({ status, error: data?.error }); + setRows(data.courses); + setCanCreate(data.canCreate); + }); + }, []); + + const create = async (e: FormEvent) => { + e.preventDefault(); + setBusy(true); + setCreateErr(null); + const { status, data } = await api("POST", "/api/studio/courses", { id, title }); + setBusy(false); + if (status === 201) nav(`/studio/${encodeURIComponent(data.id)}`); + else setCreateErr(data?.error ?? `Could not create the course (${status})`); + }; + + const managed = rows?.filter((r) => r.managed) ?? []; + const outside = rows?.filter((r) => !r.managed) ?? []; + + return ( + <div className="studio studio-page"> + <p className="chap">Studio</p> + <h1>Your courses</h1> + <p className="studio-lede">Edit on your own draft, save as you go, and publish when it is ready. Learners only see what you publish.</p> + {err ? <AccessMessage {...err} /> : null} + + {canCreate ? ( + <section className="studio-section" style={{ marginTop: 26 }} aria-labelledby="new-course"> + <h2 id="new-course">New course</h2> + <form className="studio-form studio-form--inline" onSubmit={create}> + <label className="studio-field" style={{ flex: "2 1 260px" }}> + <span>Title</span> + <input type="text" required value={title} onChange={(e) => { setTitle(e.target.value); if (!idTouched) setId(slug(e.target.value)); }} /> + </label> + <label className="studio-field" style={{ flex: "1 1 180px" }}> + <span>Course id</span> + <input type="text" className="code" required pattern="[a-z0-9][a-z0-9-]*" value={id} onChange={(e) => { setId(e.target.value); setIdTouched(true); }} /> + </label> + <Pill type="submit" variant="primary" disabled={busy || !title.trim() || !id}>Create course</Pill> + </form> + {createErr ? <p className="studio-errors" role="alert">{createErr}</p> : null} + </section> + ) : null} + + {rows ? ( + <section className="studio-section" aria-labelledby="studio-courses"> + <h2 id="studio-courses">Studio courses</h2> + {managed.length === 0 ? <p className="faint">No studio courses yet.</p> : ( + <ul className="studio-rows"> + {managed.map((c) => ( + <li className="studio-row" key={c.id}> + <CourseCover title={c.title} accent={c.accent ?? undefined} /> + <div className="t"> + <Link to={`/studio/${encodeURIComponent(c.id)}`}>{c.title}</Link> + <span className="s"><code>{c.id}</code>{c.pitch ? ` · ${c.pitch}` : ""}</span> + </div> + {c.merging ? <span className="studio-tag studio-tag--accent">Resolving conflicts</span> : null} + {c.ahead ? <span className="studio-tag studio-tag--accent">{c.ahead} unpublished</span> : null} + {c.behind ? <span className="studio-tag">{c.behind} new on published</span> : null} + {c.remote ? <span className="studio-tag">Remote</span> : null} + {c.publishedStatus === "error" ? <span className="studio-tag">Published version has errors</span> : null} + </li> + ))} + </ul> + )} + </section> + ) : null} + + {outside.length ? ( + <section className="studio-section" aria-labelledby="outside-courses"> + <h2 id="outside-courses">Managed outside STCourses</h2> + <p className="faint" style={{ margin: 0 }}>These courses come from repositories the platform does not own. Edit them in their own repository.</p> + <ul className="studio-rows"> + {outside.map((c) => ( + <li className="studio-row" key={c.id}> + <CourseCover title={c.title} accent={c.accent ?? undefined} /> + <div className="t"> + <Link to={`/studio/${encodeURIComponent(c.id)}`}>{c.title}</Link> + <span className="s"><code>{c.id}</code> · {c.message}</span> + </div> + <span className="studio-tag">Read-only</span> + </li> + ))} + </ul> + </section> + ) : null} + </div> + ); +} diff --git a/packages/feature-studio/src/ui/api.ts b/packages/feature-studio/src/ui/api.ts new file mode 100644 index 0000000..8efb89f --- /dev/null +++ b/packages/feature-studio/src/ui/api.ts @@ -0,0 +1,58 @@ +// Browser client for /api/studio. Types mirror the server responses (kept here so no server module enters the bundle). + +export interface ApiError { status: number; body: any } + +export async function api<T = any>(method: string, path: string, body?: unknown): Promise<{ status: number; data: T }> { + const init: RequestInit = { method, headers: {} }; + if (body instanceof FormData) init.body = body; + else if (body !== undefined) { + (init.headers as Record<string, string>)["content-type"] = "application/json"; + init.body = JSON.stringify(body); + } + const r = await fetch(path, init); + const text = await r.text(); + let data: any = null; + try { data = text ? JSON.parse(text) : null; } catch { data = { error: text }; } + return { status: r.status, data }; +} + +export const coursePath = (id: string, rest = "") => `/api/studio/courses/${encodeURIComponent(id)}${rest}`; +export const rawUrl = (id: string, rel: string) => coursePath(id, `/raw/${rel.split("/").map(encodeURIComponent).join("/")}`); + +export interface ManifestLesson { + id: string; title: string; type: "text" | "video" | "exercise"; file?: string; anchor?: string; duration?: number; transcript?: string; + hints?: { file: string; match: string }; + browser?: { language: "js" | "ts"; starter: string; tests: string; solution?: string }; + localCommand?: string | Record<string, string>; + [k: string]: unknown; +} +export interface ManifestModule { id: string; title: string; lessons: ManifestLesson[]; [k: string]: unknown } +export interface ManifestDeck { id: string; file: string; title?: string; attachTo: string } +export interface ManifestAttachment { file: string; title?: string; module?: string } +export interface ManifestDoc { + schema: 1; id: string; title: string; pitch?: string; authors?: string[]; cover?: string; accent?: string; collections?: string[]; + visibility?: "public" | "signed-in" | "restricted"; gating?: "none" | "sequential"; mentor?: string; + decks?: ManifestDeck[]; attachments?: ManifestAttachment[]; modules: ManifestModule[]; [k: string]: unknown; +} + +export interface DraftStatus { + branch: string; publishedBranch: string; draftHead: string; publishedHead: string; ahead: number; behind: number; + merging: boolean; conflicts: string[]; remote: string | null; publishedDirty: string[]; +} +export interface Diagnostic { level: "error" | "warning"; lessonId?: string; path?: string; message: string } + +export interface CourseState { + readOnly: false; course: { id: string; title: string }; status: DraftStatus; manifestText: string; manifestHash: string | null; + manifestRaw: ManifestDoc | null; manifestErrors: string[]; diagnostics: Diagnostic[]; files: string[]; +} +export interface ReadOnlyState { readOnly: true; message: string; course: { id: string; title: string; root: string }; diagnostics: Diagnostic[] } + +export interface FileWrite { path: string; content?: string; baseHash?: string | null; delete?: boolean } +export interface FileConflict { path: string; current: string | null; currentHash: string | null } + +export const slug = (s: string) => s.toLowerCase().normalize("NFKD").replace(/[̀-ͯ]/g, "").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "item"; +export const uniqueId = (base: string, taken: Set<string>) => { + let id = slug(base); + for (let n = 2; taken.has(id); n++) id = `${slug(base)}-${n}`; + return id; +}; diff --git a/packages/feature-studio/src/ui/editors/CourseSettings.tsx b/packages/feature-studio/src/ui/editors/CourseSettings.tsx new file mode 100644 index 0000000..6aae94b --- /dev/null +++ b/packages/feature-studio/src/ui/editors/CourseSettings.tsx @@ -0,0 +1,85 @@ +import { useState } from "react"; +import { Pill } from "@stc/ui"; +import { uniqueId } from "../api"; +import type { Workspace } from "../workspace"; +import { DiagnosticsList, Errors, Field, FilePicker, SaveBar, isImage, isMarkdown, useManifestValidation } from "./common"; + +const list = (s: string) => s.split(",").map((x) => x.trim()).filter(Boolean); + +/** Manifest fields editor (structured), with live validation from @stc/course-format. */ +export function CourseSettings({ ws }: { ws: Workspace }) { + const { doc, setDoc, state } = ws; + const errors = useManifestValidation(ws, { manifest: doc }); + const [newModule, setNewModule] = useState(""); + const set = <K extends keyof typeof doc>(k: K, v: (typeof doc)[K] | undefined) => setDoc((d) => { + if (v === undefined || v === "") delete d[k]; else d[k] = v; + }); + + return ( + <div className="studio-page" style={{ maxWidth: 760, margin: 0 }}> + <p className="chap">Course settings</p> + <h1>{doc.title || "Untitled course"}</h1> + <p className="studio-lede">These fields live in <code>stcourse.yaml</code>. Changes go to your draft when you save.</p> + + <div className="studio-form" style={{ marginTop: 22 }}> + <Field label="Title"><input type="text" value={doc.title} onChange={(e) => set("title", e.target.value)} /></Field> + <Field label="Pitch" hint="One or two sentences shown in the library."> + <textarea rows={2} value={doc.pitch ?? ""} onChange={(e) => set("pitch", e.target.value)} /> + </Field> + <div className="grid2"> + <Field label="Authors" hint="Comma-separated."><input type="text" value={(doc.authors ?? []).join(", ")} onChange={(e) => set("authors", list(e.target.value))} /></Field> + <Field label="Collections" hint="Comma-separated."><input type="text" value={(doc.collections ?? []).join(", ")} onChange={(e) => set("collections", list(e.target.value))} /></Field> + <Field label="Visibility"> + <select value={doc.visibility ?? "restricted"} onChange={(e) => set("visibility", e.target.value as never)}> + <option value="restricted">Restricted: named users or groups</option> + <option value="signed-in">Signed-in: any user</option> + <option value="public">Public: anyone</option> + </select> + </Field> + <Field label="Module gating"> + <select value={doc.gating ?? "none"} onChange={(e) => set("gating", e.target.value as never)}> + <option value="none">Off</option> + <option value="sequential">Sequential: a module unlocks when the previous one is done</option> + </select> + </Field> + <Field label="Accent colour" hint="Optional, as #rrggbb."><input type="text" className="code" value={doc.accent ?? ""} onChange={(e) => set("accent", e.target.value)} /></Field> + <Field label="Cover image"><FilePicker files={state.files} filter={isImage} allowEmpty value={doc.cover} onChange={(v) => set("cover", v)} /></Field> + <Field label="Mentor rules" hint="Instructions for the AI tutor."><FilePicker files={state.files} filter={isMarkdown} allowEmpty value={doc.mentor} onChange={(v) => set("mentor", v)} /></Field> + </div> + + <div className="studio-section"> + <h2>Modules</h2> + <ul className="studio-rows" style={{ marginTop: 0 }}> + {doc.modules.map((m, i) => ( + <li className="studio-row" key={m.id}> + <span className="faint" style={{ width: 22 }}>{i + 1}</span> + <div className="t"><input type="text" aria-label={`Title of module ${m.id}`} value={m.title} onChange={(e) => setDoc((d) => { d.modules[i]!.title = e.target.value; })} /></div> + <span className="faint">{m.lessons.length} lessons</span> + <Pill variant="ghost" disabled={i === 0} onClick={() => setDoc((d) => { const [x] = d.modules.splice(i, 1); d.modules.splice(i - 1, 0, x!); })}>Move up</Pill> + <Pill variant="ghost" disabled={m.lessons.length > 0} title={m.lessons.length ? "Remove its lessons first" : undefined} onClick={() => setDoc((d) => { d.modules.splice(i, 1); })}>Remove</Pill> + </li> + ))} + </ul> + <form className="studio-form studio-form--inline" style={{ marginTop: 12 }} onSubmit={(e) => { + e.preventDefault(); + if (!newModule.trim()) return; + setDoc((d) => { d.modules.push({ id: uniqueId(newModule, new Set(d.modules.map((x) => x.id))), title: newModule.trim(), lessons: [] }); }); + setNewModule(""); + }}> + <Field label="New module"><input type="text" value={newModule} onChange={(e) => setNewModule(e.target.value)} /></Field> + <Pill type="submit" disabled={!newModule.trim()}>Add module</Pill> + </form> + </div> + + <div className="studio-section"> + <h2>Validation</h2> + {errors.length === 0 && !state.diagnostics.length ? <p className="faint" style={{ margin: 0 }}>The manifest is valid.</p> : null} + <Errors errors={errors} /> + <DiagnosticsList diagnostics={state.diagnostics} /> + </div> + + <SaveBar ws={ws} dirty={ws.docDirty} collect={() => []} message={() => "Edit course settings"} /> + </div> + </div> + ); +} diff --git a/packages/feature-studio/src/ui/editors/GitPanels.tsx b/packages/feature-studio/src/ui/editors/GitPanels.tsx new file mode 100644 index 0000000..7663319 --- /dev/null +++ b/packages/feature-studio/src/ui/editors/GitPanels.tsx @@ -0,0 +1,123 @@ +import { useEffect, useState } from "react"; +import { Pill } from "@stc/ui"; +import { api, coursePath } from "../api"; +import type { Workspace } from "../workspace"; +import { Errors, Field, SaveBar, useManifestValidation } from "./common"; +import { useFile } from "../workspace"; + +interface ConflictFile { path: string; base: string | null; ours: string | null; theirs: string | null; working: string | null } + +/** Spec 9.1/12: conflicts are shown, and the author decides every file. Nothing is resolved automatically. */ +export function ConflictView({ ws }: { ws: Workspace }) { + const [files, setFiles] = useState<ConflictFile[] | null>(null); + const [text, setText] = useState<Record<string, string>>({}); + const [msg, setMsg] = useState<string | null>(null); + useEffect(() => { + api<{ files: ConflictFile[] }>("GET", coursePath(ws.courseId, "/conflicts")).then(({ data }) => { + setFiles(data.files); + setText(Object.fromEntries(data.files.map((f) => [f.path, f.working ?? ""]))); + }); + }, [ws.courseId, ws.state.status.draftHead, ws.state.status.conflicts.join("\0")]); + + const resolve = async (path: string, content: string | null) => { + const { status, data } = await api("POST", coursePath(ws.courseId, "/resolve"), { files: [{ path, content }] }); + setMsg(status === 200 ? (data.remaining.length ? null : "All conflicts resolved. Your draft now includes the published changes; publish when ready.") : data?.error); + await ws.reload(); + }; + + return ( + <div className="studio-page" style={{ margin: 0 }}> + <p className="chap">Conflicts</p> + <h1>Resolve conflicts</h1> + <p className="studio-lede">The published version and your draft changed the same lines. For each file, choose a version or edit the result, then mark it resolved. Your draft is not published until you publish it.</p> + {msg ? <div className="studio-note studio-note--ok" role="status">{msg}</div> : null} + {files?.map((f) => ( + <section className="studio-conflict" key={f.path} aria-label={`Conflict in ${f.path}`}> + <h2><code>{f.path}</code></h2> + <div className="studio-sides"> + <div><span className="faint">Your draft</span><pre>{f.ours ?? "(deleted in your draft)"}</pre></div> + <div><span className="faint">Published (changed outside your draft)</span><pre>{f.theirs ?? "(deleted on published)"}</pre></div> + </div> + <Field label="Result" hint="Starts with both versions between conflict markers. Remove the markers when you are done."> + <textarea className="code" rows={12} value={text[f.path] ?? ""} onChange={(e) => setText((t) => ({ ...t, [f.path]: e.target.value }))} /> + </Field> + <div className="studio-actions" style={{ marginTop: 10 }}> + <Pill onClick={() => setText((t) => ({ ...t, [f.path]: f.ours ?? "" }))}>Use my draft</Pill> + <Pill onClick={() => setText((t) => ({ ...t, [f.path]: f.theirs ?? "" }))}>Use published</Pill> + <Pill variant="primary" disabled={/^(<{7}|>{7})/m.test(text[f.path] ?? "")} onClick={() => resolve(f.path, text[f.path] ?? "")}>Mark resolved</Pill> + {f.ours === null || f.theirs === null ? <Pill variant="ghost" onClick={() => resolve(f.path, null)}>Delete the file</Pill> : null} + </div> + </section> + ))} + <div className="studio-section"> + <Pill variant="ghost" onClick={async () => { await api("POST", coursePath(ws.courseId, "/abort")); await ws.reload(); }}>Cancel the update</Pill> + <span className="faint" style={{ marginLeft: 10 }}>Returns your draft to how it was before the update.</span> + </div> + </div> + ); +} + +export function SyncPanel({ ws }: { ws: Workspace }) { + const [url, setUrl] = useState(ws.state.status.remote ?? ""); + const [msg, setMsg] = useState<{ kind: "ok" | "bad" | "warn"; text: string } | null>(null); + const [busy, setBusy] = useState(false); + const remote = ws.state.status.remote; + + const saveRemote = async () => { + const { status, data } = await api("PUT", coursePath(ws.courseId, "/remote"), { url: url.trim() || null }); + setMsg(status === 200 ? { kind: "ok", text: data.remote ? "Remote saved." : "Remote removed." } : { kind: "bad", text: data?.error }); + await ws.reload(); + }; + const sync = async () => { + setBusy(true); + const { status, data } = await api("POST", coursePath(ws.courseId, "/sync")); + setBusy(false); + if (status === 200) setMsg({ kind: "ok", text: `Synced. ${data.pulled ? "Pulled changes from the remote. " : ""}${data.pushed ? "Pushed the published version." : ""}${!data.pulled && !data.pushed ? "Already up to date." : ""}` }); + else if (data?.reason === "conflict") setMsg({ kind: "warn", text: `The remote changed the same lines as the published version (${data.conflicts.join(", ")}). Nothing was changed. Update your draft from the remote, resolve the conflicts, publish, then sync again.` }); + else setMsg({ kind: "bad", text: data?.message ?? data?.error ?? `Sync failed (${status})` }); + await ws.reload(); + }; + const fromRemote = async () => { + const { status, data } = await api("POST", coursePath(ws.courseId, "/update-draft"), { from: "remote" }); + setMsg(status === 200 ? { kind: "ok", text: "Your draft now includes the remote's changes." } : status === 409 ? { kind: "warn", text: "Conflicts to resolve: see Conflicts." } : { kind: "bad", text: data?.message ?? data?.error }); + await ws.reload(); + }; + + return ( + <div className="studio-page" style={{ maxWidth: 760, margin: 0 }}> + <p className="chap">Sync</p> + <h1>Remote repository</h1> + <p className="studio-lede">Nothing syncs on its own. Sync pulls the remote's published branch and pushes yours, only when you click it.</p> + <div className="studio-form" style={{ marginTop: 22 }}> + <form className="studio-form studio-form--inline" onSubmit={(e) => { e.preventDefault(); saveRemote(); }}> + <Field label="Remote URL" hint="Credentials come from the server (SSH keys or a token in the URL)."><input type="text" className="code" style={{ minWidth: 380 }} value={url} onChange={(e) => setUrl(e.target.value)} placeholder="git@example.org:me/course.git" /></Field> + <Pill type="submit">Save remote</Pill> + </form> + {msg ? <div className={`studio-note studio-note--${msg.kind}`} role="status">{msg.text}</div> : null} + <div className="studio-actions"> + <Pill variant="primary" disabled={!remote || busy} onClick={sync}>{busy ? "Syncing…" : "Sync now"}</Pill> + <Pill disabled={!remote} onClick={fromRemote}>Update my draft from the remote</Pill> + {!remote ? <span className="faint">Add a remote to enable Sync.</span> : null} + </div> + </div> + </div> + ); +} + +/** The whole manifest as YAML, for anything the structured editors do not cover. */ +export function RawManifest({ ws }: { ws: Workspace }) { + const f = useFile(ws.courseId, "stcourse.yaml", ws.version); + const errors = useManifestValidation(ws, { manifestText: f.content }); + return ( + <div className="studio-page" style={{ maxWidth: 900, margin: 0 }}> + <p className="chap">Manifest</p> + <h1><code style={{ fontSize: 24 }}>stcourse.yaml</code></h1> + <p className="studio-lede">The source of truth for the course structure. Unsaved changes in the other editors are not shown here.</p> + <div className="studio-form" style={{ maxWidth: "none", marginTop: 18 }}> + <textarea className="code" rows={28} spellCheck={false} value={f.content} onChange={(e) => f.setContent(e.target.value)} aria-label="stcourse.yaml" /> + {errors.length ? <Errors errors={errors} /> : <p className="faint" style={{ margin: 0 }}>Valid manifest.</p>} + <SaveBar ws={ws} dirty={f.dirty} withManifest={false} collect={() => [f.write()!]} message={() => "Edit stcourse.yaml"} /> + </div> + </div> + ); +} diff --git a/packages/feature-studio/src/ui/editors/LessonEditor.tsx b/packages/feature-studio/src/ui/editors/LessonEditor.tsx new file mode 100644 index 0000000..0c630a6 --- /dev/null +++ b/packages/feature-studio/src/ui/editors/LessonEditor.tsx @@ -0,0 +1,168 @@ +import { Pill } from "@stc/ui"; +import type { FileWrite, ManifestLesson } from "../api"; +import { LessonPreview } from "../preview"; +import { useFile, type OpenFile, type Workspace } from "../workspace"; +import { DiagnosticsList, Field, FilePicker, SaveBar, UploadButton, isMarkdown, isVideo, useManifestValidation } from "./common"; + +const STARTER = (lang: string) => `// Starter code for the learner.\nexport function solve(input${lang === "ts" ? ": unknown" : ""}) {\n // TODO\n}\n`; +const TESTS = `// export const tests: { name, run(mod, assert) }[]\nexport const tests = [\n {\n name: "solves the example",\n run(mod, assert) {\n assert.equal(mod.solve(1), 1);\n },\n },\n];\n`; +const HINT = (match: string, label: string) => `<details>\n<summary>${match} — ${label}</summary>\n\nWrite the hint here.\n\n</details>\n`; + +function CodeFile({ label, file, rows = 10 }: { label: string; file: OpenFile; rows?: number }) { + if (!file.path) return null; + return ( + <Field label={`${label} · ${file.path}`} hint={file.loaded && !file.exists ? "New file, created when you save." : undefined}> + <textarea className="code" rows={rows} spellCheck={false} value={file.content} onChange={(e) => file.setContent(e.target.value)} /> + </Field> + ); +} + +export function LessonEditor({ ws, lessonId, onGone }: { ws: Workspace; lessonId: string; onGone: () => void }) { + const { doc, setDoc, state } = ws; + let mi = -1, li = -1; + doc.modules.forEach((m, i) => m.lessons.forEach((l, j) => { if (l.id === lessonId) { mi = i; li = j; } })); + const lesson: ManifestLesson | undefined = mi >= 0 ? doc.modules[mi]!.lessons[li] : undefined; + + // the Markdown body: the lesson file for text/exercise lessons, the transcript for videos + const mdPath = lesson ? (lesson.type === "video" ? lesson.transcript : lesson.file && isMarkdown(lesson.file) ? lesson.file : undefined) : undefined; + const lang = lesson?.browser?.language ?? "js"; + const body = useFile(ws.courseId, mdPath, ws.version, lesson ? `# ${lesson.title}\n\n` : ""); + const starter = useFile(ws.courseId, lesson?.browser?.starter, ws.version, STARTER(lang)); + const tests = useFile(ws.courseId, lesson?.browser?.tests, ws.version, TESTS); + const solution = useFile(ws.courseId, lesson?.browser?.solution, ws.version, STARTER(lang).replace("// TODO", "return input;")); + const hints = useFile(ws.courseId, lesson?.hints?.file !== mdPath ? lesson?.hints?.file : undefined, ws.version); + const errors = useManifestValidation(ws, { manifest: doc }); + + if (!lesson) return <div className="studio-note">This lesson is no longer in the manifest. <Pill variant="ghost" onClick={onGone}>Back to course settings</Pill></div>; + + const upd = (f: (l: ManifestLesson) => void) => setDoc((d) => f(d.modules[mi]!.lessons[li]!)); + const setOpt = <K extends keyof ManifestLesson>(k: K, v: ManifestLesson[K] | undefined) => upd((l) => { if (v === undefined || v === "") delete l[k]; else l[k] = v; }); + const files = [body, starter, tests, solution, hints]; + const dirty = ws.docDirty || files.some((f) => f.dirty); + const collect = (): FileWrite[] => files.filter((f) => f.path && (f.dirty || (!f.exists && f.loaded))).map((f) => f.write()!).filter((w, i, a) => a.findIndex((x) => x.path === w.path) === i); + const lessonErrors = errors.filter((e) => e.startsWith(`modules.${mi}.lessons.${li}.`)); + const exDir = `exercises/${lesson.id}`; + + return ( + <div> + <p className="chap">{doc.modules[mi]!.title} · {lesson.type === "text" ? "Text lesson" : lesson.type === "video" ? "Video lesson" : "Exercise"}</p> + <div className="studio-form" style={{ maxWidth: "none" }}> + <div className="grid3"> + <Field label="Title"><input type="text" value={lesson.title} onChange={(e) => upd((l) => { l.title = e.target.value; })} /></Field> + <Field label="Type"> + <select value={lesson.type} onChange={(e) => upd((l) => { l.type = e.target.value as ManifestLesson["type"]; })}> + <option value="text">Text</option><option value="video">Video</option><option value="exercise">Exercise</option> + </select> + </Field> + <Field label="Duration (minutes)"> + <input type="number" min={0} step={1} value={lesson.duration ? Math.round(lesson.duration / 60) : ""} onChange={(e) => setOpt("duration", e.target.value ? Number(e.target.value) * 60 : undefined)} /> + </Field> + {lesson.type === "video" ? ( + <> + <Field label="Video file"><FilePicker files={state.files} filter={isVideo} value={lesson.file} onChange={(v) => setOpt("file", v)} /></Field> + <Field label="Transcript (Markdown)"><FilePicker files={state.files} filter={isMarkdown} allowEmpty value={lesson.transcript} onChange={(v) => setOpt("transcript", v)} /></Field> + <div className="studio-field"><span>Upload</span><div className="studio-actions"><UploadButton ws={ws} dir="media" accept="video/*" label="Upload video" onUploaded={(p) => { setOpt("file", p); }} /></div></div> + </> + ) : ( + <> + <Field label="Lesson file"><FilePicker files={state.files} filter={isMarkdown} allowEmpty={lesson.type === "exercise"} value={lesson.file} onChange={(v) => setOpt("file", v)} /></Field> + <Field label="Heading anchor" hint="Optional: show only this section of the file."><input type="text" className="code" value={lesson.anchor ?? ""} onChange={(e) => setOpt("anchor", e.target.value)} /></Field> + </> + )} + </div> + {lesson.type === "video" && !lesson.transcript ? ( + <div className="studio-actions"><Pill onClick={() => setOpt("transcript", `lessons/${lesson.id}.md`)}>Add a transcript file</Pill></div> + ) : null} + <Errorsish errors={lessonErrors} ws={ws} lessonId={lesson.id} /> + + {mdPath ? ( + <div className="studio-split"> + <Field label={`Markdown · ${mdPath}`} hint={body.loaded && !body.exists ? "New file, created when you save." : undefined}> + <textarea className="code" spellCheck value={body.content} onChange={(e) => body.setContent(e.target.value)} aria-describedby="preview-label" /> + </Field> + <div className="studio-preview-wrap" aria-live="off"> + <p className="faint" id="preview-label" style={{ margin: "0 0 10px" }}>Preview</p> + <LessonPreview courseId={ws.courseId} markdown={body.content} lesson={lesson} resolveUrl={(rel) => ws.resolveUrl(new URL(rel, `http://x/${mdPath}`).pathname.slice(1))} /> + </div> + </div> + ) : null} + + {lesson.type === "exercise" ? ( + <div className="studio-section"> + <h2>Exercise</h2> + <div className="grid2"> + <Field label="Local test command" hint="Shown to learners who run the exercise on their machine."> + <input type="text" className="code" value={typeof lesson.localCommand === "string" ? lesson.localCommand : ""} onChange={(e) => setOpt("localCommand", e.target.value)} /> + </Field> + <Field label="Browser runner"> + <select value={lesson.browser?.language ?? ""} onChange={(e) => { + const lang = e.target.value as "js" | "ts" | ""; + upd((l) => { + if (!lang) { delete l.browser; return; } + l.browser = l.browser ? { ...l.browser, language: lang } : { language: lang, starter: `${exDir}/starter.${lang}`, tests: `${exDir}/tests.${lang}`, solution: `${exDir}/solution.${lang}` }; + }); + }}> + <option value="">None (runs locally)</option><option value="js">JavaScript</option><option value="ts">TypeScript</option> + </select> + </Field> + </div> + {lesson.browser ? ( + <> + <CodeFile label="Starter" file={starter} /> + <CodeFile label="Tests" file={tests} /> + <CodeFile label="Solution (authors only)" file={solution} /> + </> + ) : null} + + <h3>Hints</h3> + {lesson.hints ? ( + <> + <div className="grid2"> + <Field label="Hints file"><FilePicker files={state.files} filter={isMarkdown} value={lesson.hints.file} onChange={(v) => upd((l) => { if (v) l.hints = { ...l.hints!, file: v }; })} /></Field> + <Field label="Summary label" hint="Hint blocks whose summary starts with this label belong to this exercise."> + <input type="text" value={lesson.hints.match} onChange={(e) => upd((l) => { l.hints = { ...l.hints!, match: e.target.value }; })} /> + </Field> + </div> + {hints.path ? <CodeFile label="Hint tiers" file={hints} rows={12} /> : <p className="faint">The hints live in the lesson file above.</p>} + <div className="studio-actions"> + {(["nudge", "stronger hint", "near-solution"] as const).map((tier) => ( + <Pill key={tier} variant="ghost" onClick={() => { + const target = hints.path ? hints : body; + target.setContent((target.content.trimEnd() ? target.content.trimEnd() + "\n\n" : "") + HINT(lesson.hints!.match, tier)); + }}>Add {tier}</Pill> + ))} + </div> + </> + ) : ( + <Pill onClick={() => { + const match = lesson.title; + upd((l) => { l.hints = { file: `${exDir}/hints.md`, match }; }); + }}>Add hint tiers</Pill> + )} + </div> + ) : null} + + <SaveBar ws={ws} dirty={dirty} collect={collect} message={() => `Edit lesson ${lesson.id}`} extra={ + <> + <Pill variant="ghost" disabled={li === 0} onClick={() => setDoc((d) => { const ls = d.modules[mi]!.lessons; const [x] = ls.splice(li, 1); ls.splice(li - 1, 0, x!); })}>Move up</Pill> + <Pill variant="ghost" onClick={() => { + if (!window.confirm(`Remove "${lesson.title}" from the course? Its files stay in the repository.`)) return; + setDoc((d) => { d.modules[mi]!.lessons.splice(li, 1); }); + onGone(); + }}>Remove lesson</Pill> + </> + } /> + <p className="faint" style={{ margin: 0 }}>Saved changes stay on your draft. Learners see them after you publish.</p> + </div> + </div> + ); +} + +function Errorsish({ errors, ws, lessonId }: { errors: string[]; ws: Workspace; lessonId: string }) { + return ( + <> + {errors.length ? <ul className="studio-errors">{errors.map((e) => <li key={e}>{e}</li>)}</ul> : null} + <DiagnosticsList diagnostics={ws.state.diagnostics} filter={(d) => d.lessonId === lessonId} /> + </> + ); +} diff --git a/packages/feature-studio/src/ui/editors/Media.tsx b/packages/feature-studio/src/ui/editors/Media.tsx new file mode 100644 index 0000000..c705295 --- /dev/null +++ b/packages/feature-studio/src/ui/editors/Media.tsx @@ -0,0 +1,87 @@ +import { Pill } from "@stc/ui"; +import { slug, uniqueId, type ManifestDoc } from "../api"; +import type { Workspace } from "../workspace"; +import { DiagnosticsList, Field, FilePicker, SaveBar, UploadButton, isPdf } from "./common"; + +const targets = (doc: ManifestDoc) => [ + ...doc.modules.map((m) => ({ id: m.id, label: `Module: ${m.title}` })), + ...doc.modules.flatMap((m) => m.lessons.map((l) => ({ id: l.id, label: `${m.title} · ${l.title}` }))), +]; + +/** Deck links: a PDF deck attached to a lesson (for example its video) or a module. */ +export function DecksEditor({ ws }: { ws: Workspace }) { + const { doc, setDoc, state } = ws; + const decks = doc.decks ?? []; + const opts = targets(doc); + const add = (file: string) => setDoc((d) => { + const ids = new Set((d.decks ?? []).map((x) => x.id)); + const name = file.split("/").pop()!.replace(/\.pdf$/i, ""); + (d.decks ??= []).push({ id: uniqueId(slug(name), ids), file, title: name, attachTo: opts[0]?.id ?? "" }); + }); + return ( + <div className="studio-page" style={{ maxWidth: 900, margin: 0 }}> + <p className="chap">Deck links</p> + <h1>Slide decks</h1> + <p className="studio-lede">A deck is a PDF. Link it to the lesson or module it belongs to, and learners see it next to that content.</p> + <ul className="studio-rows"> + {decks.length === 0 ? <li className="studio-row faint">No decks yet.</li> : null} + {decks.map((dk, i) => ( + <li className="studio-row" key={i} style={{ alignItems: "end" }}> + <div className="t grid3" style={{ gridTemplateColumns: "1.2fr 1fr 1.2fr" }}> + <Field label="Title"><input type="text" value={dk.title ?? ""} onChange={(e) => setDoc((d) => { d.decks![i]!.title = e.target.value || undefined; })} /></Field> + <Field label="PDF"><FilePicker files={state.files} filter={isPdf} value={dk.file} onChange={(v) => v && setDoc((d) => { d.decks![i]!.file = v; })} /></Field> + <Field label="Attached to"> + <select value={dk.attachTo} onChange={(e) => setDoc((d) => { d.decks![i]!.attachTo = e.target.value; })}> + {opts.map((o) => <option key={o.id} value={o.id}>{o.label}</option>)} + </select> + </Field> + </div> + <Pill variant="ghost" onClick={() => setDoc((d) => { d.decks!.splice(i, 1); if (!d.decks!.length) delete d.decks; })}>Remove</Pill> + </li> + ))} + </ul> + <div className="studio-actions" style={{ margin: "14px 0 20px" }}> + <UploadButton ws={ws} dir="media" accept="application/pdf,.pdf" label="Upload a PDF deck" onUploaded={add} /> + {state.files.some(isPdf) ? <Pill variant="ghost" onClick={() => add(state.files.find(isPdf)!)}>Link an existing PDF</Pill> : null} + </div> + <DiagnosticsList diagnostics={state.diagnostics} filter={(d) => /deck/.test(d.message)} /> + <SaveBar ws={ws} dirty={ws.docDirty} collect={() => []} message={() => "Edit deck links"} /> + </div> + ); +} + +/** Course attachments (PDFs, images, code archives), optionally scoped to a module. */ +export function AttachmentsEditor({ ws }: { ws: Workspace }) { + const { doc, setDoc, state } = ws; + const list = doc.attachments ?? []; + return ( + <div className="studio-page" style={{ maxWidth: 900, margin: 0 }}> + <p className="chap">Attachments</p> + <h1>Attachments</h1> + <p className="studio-lede">Files learners can download. Uploads are committed to your draft at once; the list below is saved with the manifest.</p> + <ul className="studio-rows"> + {list.length === 0 ? <li className="studio-row faint">No attachments yet.</li> : null} + {list.map((a, i) => ( + <li className="studio-row" key={i} style={{ alignItems: "end" }}> + <div className="t grid3" style={{ gridTemplateColumns: "1.3fr 1fr 1fr" }}> + <Field label="File"><a href={ws.resolveUrl(a.file)} className="code" style={{ fontFamily: "var(--mono)", fontSize: 12.5, lineHeight: "34px", overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap" }}>{a.file}</a></Field> + <Field label="Title"><input type="text" value={a.title ?? ""} onChange={(e) => setDoc((d) => { d.attachments![i]!.title = e.target.value || undefined; })} /></Field> + <Field label="Module"> + <select value={a.module ?? ""} onChange={(e) => setDoc((d) => { if (e.target.value) d.attachments![i]!.module = e.target.value; else delete d.attachments![i]!.module; })}> + <option value="">Whole course</option> + {doc.modules.map((m) => <option key={m.id} value={m.id}>{m.title}</option>)} + </select> + </Field> + </div> + <Pill variant="ghost" onClick={() => setDoc((d) => { d.attachments!.splice(i, 1); if (!d.attachments!.length) delete d.attachments; })}>Remove</Pill> + </li> + ))} + </ul> + <div className="studio-actions" style={{ margin: "14px 0 20px" }}> + <UploadButton ws={ws} dir="attachments" label="Upload a file" onUploaded={(p) => setDoc((d) => { (d.attachments ??= []).push({ file: p, title: p.split("/").pop() }); })} /> + </div> + <DiagnosticsList diagnostics={state.diagnostics} filter={(d) => /attachment/.test(d.message)} /> + <SaveBar ws={ws} dirty={ws.docDirty} collect={() => []} message={() => "Edit attachments"} /> + </div> + ); +} diff --git a/packages/feature-studio/src/ui/editors/common.tsx b/packages/feature-studio/src/ui/editors/common.tsx new file mode 100644 index 0000000..b660ee4 --- /dev/null +++ b/packages/feature-studio/src/ui/editors/common.tsx @@ -0,0 +1,131 @@ +import { useEffect, useRef, useState, type ReactNode } from "react"; +import { Pill } from "@stc/ui"; +import { api, coursePath, type Diagnostic, type FileConflict, type FileWrite } from "../api"; +import type { SaveOutcome, Workspace } from "../workspace"; + +export function Field({ label, hint, children, wide }: { label: string; hint?: ReactNode; children: ReactNode; wide?: boolean }) { + return ( + <label className="studio-field" style={wide ? { gridColumn: "1 / -1" } : undefined}> + <span>{label}</span> + {children} + {hint ? <small>{hint}</small> : null} + </label> + ); +} + +export function Errors({ errors }: { errors: string[] }) { + if (!errors.length) return null; + return <ul className="studio-errors" aria-live="polite">{errors.map((e, i) => <li key={i}>{e}</li>)}</ul>; +} + +export function DiagnosticsList({ diagnostics, filter }: { diagnostics: Diagnostic[]; filter?: (d: Diagnostic) => boolean }) { + const list = filter ? diagnostics.filter(filter) : diagnostics; + if (!list.length) return null; + return <Errors errors={list.map((d) => `${d.level === "warning" ? "warning: " : ""}${d.lessonId ? `${d.lessonId}: ` : ""}${d.message}`)} />; +} + +/** Live manifest validation through @stc/course-format on the server, debounced. */ +export function useManifestValidation(ws: Workspace, input: { manifest?: unknown; manifestText?: string }) { + const [errors, setErrors] = useState<string[]>(ws.state.manifestErrors); + const key = JSON.stringify(input); + useEffect(() => { + const t = setTimeout(async () => { + const { status, data } = await api("POST", coursePath(ws.courseId, "/validate"), input); + if (status === 200) setErrors(data.errors); + }, 350); + return () => clearTimeout(t); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [key, ws.courseId]); + return errors; +} + +/** + * Save button with the "changed outside the studio" flow: on a 409 it lists the files and lets the author + * either reload the outside version or explicitly keep theirs. Nothing is overwritten without that choice. + */ +export function SaveBar({ ws, dirty, collect, message, label = "Save draft", onSaved, extra, withManifest }: { + ws: Workspace; dirty: boolean; withManifest?: boolean; collect: () => FileWrite[]; message: () => string; label?: string; onSaved?: () => void; extra?: ReactNode; +}) { + const [busy, setBusy] = useState(false); + const [conflicts, setConflicts] = useState<FileConflict[] | null>(null); + const [error, setError] = useState<string | null>(null); + const [saved, setSaved] = useState(false); + const pending = useRef<FileWrite[]>([]); + + const run = async (files: FileWrite[]) => { + setBusy(true); setError(null); setSaved(false); + const r: SaveOutcome = await ws.save({ files, message: message(), withManifest }); + setBusy(false); + if (r.ok) { setConflicts(null); setSaved(true); onSaved?.(); return; } + if (r.conflicts) { pending.current = files; setConflicts(r.conflicts); } + else setError(r.error ?? "Save failed"); + }; + + return ( + <div> + <div className="studio-actions"> + <Pill variant="primary" disabled={busy || !dirty} onClick={() => run(collect())}>{busy ? "Saving…" : label}</Pill> + {extra} + {saved && !dirty ? <span className="faint" role="status">Saved to your draft</span> : null} + {dirty ? <span className="faint">Unsaved changes</span> : null} + </div> + {error ? <div className="studio-note studio-note--bad" role="alert">{error}</div> : null} + {conflicts ? ( + <div className="studio-note studio-note--warn" role="alert"> + <strong>Changed outside the studio since you opened it:</strong> {conflicts.map((c) => c.path).join(", ")}. + <div className="studio-sides"> + {conflicts.map((c) => ( + <div key={c.path}><span className="faint">{c.path}, current version on disk</span><pre>{c.current ?? "(deleted)"}</pre></div> + ))} + </div> + <div className="studio-actions"> + <Pill onClick={() => { setConflicts(null); ws.reload().then(() => onSaved?.()); }}>Discard mine, load theirs</Pill> + <Pill onClick={() => run(pending.current.map((f) => { + const c = conflicts.find((x) => x.path === f.path); + return c ? { ...f, baseHash: c.currentHash } : f; + }))}>Keep mine and save over theirs</Pill> + </div> + </div> + ) : null} + </div> + ); +} + +export function FilePicker({ files, value, onChange, filter, allowEmpty, id }: { + files: string[]; value: string | undefined; onChange: (v: string | undefined) => void; filter?: (f: string) => boolean; allowEmpty?: boolean; id?: string; +}) { + const list = filter ? files.filter(filter) : files; + const opts = value && !list.includes(value) ? [value, ...list] : list; + return ( + <select id={id} value={value ?? ""} onChange={(e) => onChange(e.target.value || undefined)}> + {allowEmpty || !value ? <option value="">None</option> : null} + {opts.map((f) => <option key={f} value={f}>{f}</option>)} + </select> + ); +} + +export function UploadButton({ ws, dir, accept, label, onUploaded }: { ws: Workspace; dir: string; accept?: string; label: string; onUploaded: (path: string) => void }) { + const ref = useRef<HTMLInputElement>(null); + const [busy, setBusy] = useState(false); + const [err, setErr] = useState<string | null>(null); + return ( + <> + <input ref={ref} type="file" accept={accept} className="sr" aria-label={label} tabIndex={-1} onChange={async (e) => { + const f = e.target.files?.[0]; + e.target.value = ""; + if (!f) return; + setBusy(true); setErr(null); + const r = await ws.upload(f, dir); + setBusy(false); + if (r.ok) onUploaded(r.path); else setErr(r.error); + }} /> + <Pill onClick={() => ref.current?.click()} disabled={busy}>{busy ? "Uploading…" : label}</Pill> + {err ? <span className="studio-errors" role="alert">{err}</span> : null} + </> + ); +} + +export const isMarkdown = (f: string) => /\.(md|markdown)$/i.test(f); +export const isPdf = (f: string) => /\.pdf$/i.test(f); +export const isVideo = (f: string) => /\.(mp4|webm|mov|m4v)$/i.test(f); +export const isImage = (f: string) => /\.(png|jpe?g|svg|webp|gif)$/i.test(f); diff --git a/packages/feature-studio/src/ui/markdown.ts b/packages/feature-studio/src/ui/markdown.ts new file mode 100644 index 0000000..799aa33 --- /dev/null +++ b/packages/feature-studio/src/ui/markdown.ts @@ -0,0 +1,129 @@ +/** + * A small, safe Markdown renderer for the studio's live preview (v1 adapter; Phase 2 swaps in the learner's LessonView). + * Everything is HTML-escaped; the only raw HTML let through is the bare `<details>`/`<summary>` used for hint tiers. + */ + +export interface RenderOptions { + /** Maps a relative image/link URL in the lesson to something the browser can load (the draft raw endpoint). */ + resolveUrl?: (url: string) => string; +} + +const esc = (s: string) => s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """); +const unesc = (s: string) => s.replace(/"/g, '"').replace(/>/g, ">").replace(/</g, "<").replace(/&/g, "&"); + +/** Same rule as @stc/course-format's slugify (GitHub style), duplicated to keep node-only modules out of the browser bundle. */ +export function slugify(heading: string): string { + return heading + .replace(/!?\[([^\]]*)\]\([^)]*\)/g, "$1") + .replace(/[`*~]/g, "") + .replace(/<[^>]+>/g, "") + .trim() + .toLowerCase() + .replace(/[^\p{L}\p{N}\p{M}\s_-]/gu, "") + .replace(/\s/g, "-"); +} + +const SAFE_SCHEME = /^(https?:|mailto:|#|\/|\.{0,2}\/|[^:]*$)/i; +const URL_PART = String.raw`((?:[^()]|\([^()]*\))*)`; + +function inline(text: string, o: RenderOptions): string { + const codes: string[] = []; + let s = esc(text).replace(/`([^`]+)`/g, (_, c: string) => `\u0000${codes.push(`<code>${c}</code>`) - 1}\u0000`); + const url = (raw: string) => { + const u = unesc(raw.trim()); + if (!SAFE_SCHEME.test(u)) return null; + const isRelative = !/^([a-z]+:|\/|#)/i.test(u); + return esc(isRelative && o.resolveUrl ? o.resolveUrl(u) : u); + }; + s = s.replace(new RegExp(String.raw`!\[([^\]]*)\]\(` + URL_PART + String.raw`\)`, "g"), (_, alt: string, u: string) => { + const href = url(u); + return href ? `<img src="${href}" alt="${alt}">` : alt; + }); + s = s.replace(new RegExp(String.raw`\[([^\]]+)\]\(` + URL_PART + String.raw`\)`, "g"), (_, label: string, u: string) => { + const href = url(u); + return href ? `<a href="${href}">${label}</a>` : label; + }); + s = s.replace(/\*\*(.+?)\*\*/g, "<strong>$1</strong>").replace(/__(.+?)__/g, "<strong>$1</strong>"); + s = s.replace(/(^|[^*\w])\*(?!\s)(.+?)\*(?!\w)/g, "$1<em>$2</em>").replace(/(^|[^_\w])_(?!\s)(.+?)_(?!\w)/g, "$1<em>$2</em>"); + return s.replace(/\u0000(\d+)\u0000/g, (_, i: string) => codes[Number(i)]!); +} + +export function renderMarkdown(md: string, o: RenderOptions = {}): string { + let lines = md.replace(/\r\n?/g, "\n").split("\n"); + if (lines[0] === "---") { + const end = lines.indexOf("---", 1); + if (end > 0) lines = lines.slice(end + 1); + } + const out: string[] = []; + const slugs = new Map<string, number>(); + let i = 0; + const isBlockStart = (l: string) => + /^ {0,3}(#{1,6}\s|```|~~~|>|[-*+]\s|\d+[.)]\s|(-{3,}|\*{3,}|_{3,})\s*$)/.test(l) || /^\s*<\/?(details|summary)>/.test(l); + + while (i < lines.length) { + const line = lines[i]!; + if (!line.trim()) { i++; continue; } + + const fence = /^ {0,3}(`{3,}|~{3,})\s*([\w+-]*)/.exec(line); + if (fence) { + const body: string[] = []; + i++; + while (i < lines.length && !lines[i]!.trimStart().startsWith(fence[1]!)) body.push(lines[i++]!); + i++; + const lang = fence[2] ? ` data-lang="${esc(fence[2])}"` : ""; + out.push(`<pre class="code"><code${lang}>${esc(body.join("\n"))}</code></pre>`); + continue; + } + + const h = /^ {0,3}(#{1,6})\s+(.*?)\s*#*\s*$/.exec(line); + if (h) { + const base = slugify(h[2]!); + const n = slugs.get(base) ?? 0; + slugs.set(base, n + 1); + const lvl = h[1]!.length; + out.push(`<h${lvl} id="${esc(n ? `${base}-${n}` : base)}">${inline(h[2]!, o)}</h${lvl}>`); + i++; + continue; + } + + if (/^ {0,3}(-{3,}|\*{3,}|_{3,})\s*$/.test(line)) { out.push("<hr>"); i++; continue; } + + // hint blocks: <details>, <summary>label</summary>, </details> + const det = /^\s*<(\/?)details>\s*$/.exec(line); + if (det) { out.push(det[1] ? "</details>" : "<details>"); i++; continue; } + const sum = /^\s*(<details>)?\s*<summary>(.*?)<\/summary>\s*$/.exec(line); + if (sum) { + out.push(`${sum[1] ? "<details>" : ""}<summary>${inline(sum[2]!, o)}</summary>`); + i++; + continue; + } + + if (/^ {0,3}>/.test(line)) { + const body: string[] = []; + while (i < lines.length && /^ {0,3}>/.test(lines[i]!)) body.push(lines[i++]!.replace(/^ {0,3}> ?/, "")); + out.push(`<blockquote>${renderMarkdown(body.join("\n"), o)}</blockquote>`); + continue; + } + + const list = /^ {0,3}([-*+]|\d+[.)])\s+/.exec(line); + if (list) { + const ordered = /\d/.test(list[1]!); + const re = ordered ? /^ {0,3}\d+[.)]\s+(.*)$/ : /^ {0,3}[-*+]\s+(.*)$/; + const items: string[] = []; + while (i < lines.length) { + const m = re.exec(lines[i]!); + if (m) { items.push(m[1]!); i++; continue; } + if (lines[i]!.trim() && /^\s{2,}/.test(lines[i]!) && items.length) { items[items.length - 1] += "\n" + lines[i]!.trim(); i++; continue; } + break; + } + const tag = ordered ? "ol" : "ul"; + out.push(`<${tag}>${items.map((it) => `<li>${inline(it, o)}</li>`).join("")}</${tag}>`); + continue; + } + + const para: string[] = []; + while (i < lines.length && lines[i]!.trim() && (para.length === 0 || !isBlockStart(lines[i]!))) para.push(lines[i++]!); + out.push(`<p>${inline(para.join("\n"), o)}</p>`); + } + return out.join("\n"); +} diff --git a/packages/feature-studio/src/ui/preview.tsx b/packages/feature-studio/src/ui/preview.tsx new file mode 100644 index 0000000..dfc4faf --- /dev/null +++ b/packages/feature-studio/src/ui/preview.tsx @@ -0,0 +1,28 @@ +import type { ComponentType } from "react"; +import { renderMarkdown } from "./markdown"; + +/** + * Live-preview adapter. v1 renders with a small built-in Markdown renderer; Phase 2 swaps in the learner + * package's `LessonView` via `setLessonPreview` (docs/contract-requests/studio.md). + */ +export interface LessonPreviewProps { + courseId: string; + markdown: string; + lesson?: { id: string; title: string; type: string; anchor?: string } | null; + /** Maps a course-relative path (images, attachments) to a URL that serves the draft file. */ + resolveUrl: (rel: string) => string; +} +export type LessonPreviewComponent = ComponentType<LessonPreviewProps>; + +export function SimpleMarkdownPreview({ markdown, resolveUrl }: LessonPreviewProps) { + return <div className="studio-preview prose reading" dangerouslySetInnerHTML={{ __html: renderMarkdown(markdown, { resolveUrl }) }} />; +} + +let current: LessonPreviewComponent = SimpleMarkdownPreview; +export function setLessonPreview(c: LessonPreviewComponent): void { + current = c; +} +export function LessonPreview(props: LessonPreviewProps) { + const C = current; + return <C {...props} />; +} diff --git a/packages/feature-studio/src/ui/studio.css b/packages/feature-studio/src/ui/studio.css new file mode 100644 index 0000000..c1ab02a --- /dev/null +++ b/packages/feature-studio/src/ui/studio.css @@ -0,0 +1,98 @@ +/* Studio: the denser "Work" look of the mockup. Tokens only; no new colours. */ +.studio { font-family: var(--sans); font-size: 14px; line-height: 1.55; color: var(--text); } +.studio-page { max-width: 1180px; margin: 0 auto; padding-bottom: 64px; } +.studio h1 { font: 600 28px/1.12 var(--sans); color: var(--head); margin: 0; letter-spacing: -.012em; } +.studio h2 { font: 600 16px/1.3 var(--sans); color: var(--head); margin: 0 0 10px; } +.studio h3 { font: 600 14px/1.3 var(--sans); color: var(--head); margin: 18px 0 8px; } +.studio .chap { font-size: 13px; font-weight: 500; color: var(--muted); margin: 0 0 10px; } +.studio .muted { color: var(--muted); } +.studio .faint { color: var(--faint); font-size: 12.5px; } +.studio code { font-size: 12.5px; } +.studio-lede { color: var(--muted); margin: 8px 0 0; max-width: 760px; } + +/* rows separated by thin rules */ +.studio-rows { list-style: none; margin: 18px 0 0; padding: 0; border-top: 1px solid var(--line); } +.studio-row { display: flex; align-items: center; gap: 14px; padding: 12px 2px; border-bottom: 1px solid var(--line); } +.studio-row .t { flex: 1; min-width: 0; } +.studio-row .t a, .studio-row .t strong { color: var(--head); font-weight: 500; text-decoration: none; font-size: 14.5px; } +.studio-row .t a:hover { text-decoration: underline; } +.studio-row .s { display: block; color: var(--faint); font-size: 12.5px; margin-top: 2px; } +.studio-tag { display: inline-flex; align-items: center; height: 22px; padding: 0 9px; border: 1px solid var(--line2); border-radius: 11px; font-size: 11.5px; color: var(--muted); white-space: nowrap; } +.studio-tag--accent { border-color: var(--accent-dim); color: var(--accent); } + +/* forms */ +.studio-form { display: grid; gap: 14px; max-width: 760px; } +.studio-form--inline { display: flex; flex-wrap: wrap; align-items: end; gap: 12px; max-width: none; } +.studio-field { display: grid; gap: 5px; min-width: 0; align-content: start; } +.studio-field > span { font-size: 12px; color: var(--faint); } +.studio-field small { font-size: 12px; color: var(--faint); } +.studio input[type="text"], .studio input[type="url"], .studio input[type="number"], .studio select, .studio textarea { + width: 100%; min-width: 0; height: 34px; padding: 0 12px; border: 1px solid var(--line2); border-radius: 8px; background: var(--side); color: var(--text); font: inherit; font-size: 13.5px; +} +.studio select { padding-right: 8px; } +.studio textarea { height: auto; padding: 10px 12px; line-height: 1.55; resize: vertical; } +.studio textarea.code, .studio input.code { font-family: var(--mono); font-size: 13px; tab-size: 2; } +.studio input:focus, .studio select:focus, .studio textarea:focus { outline: none; border-color: var(--faint); } +.studio .grid2 { display: grid; grid-template-columns: 1fr 1fr; gap: 14px; } +.studio .grid3 { display: grid; grid-template-columns: repeat(3, 1fr); gap: 14px; } +.studio-actions { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; } + +/* messages */ +.studio-note { border-left: 2px solid var(--line2); padding: 8px 0 8px 14px; color: var(--muted); margin: 14px 0; } +.studio-note strong { color: var(--text); font-weight: 500; } +.studio-note--warn { border-left-color: var(--accent); } +.studio-note--bad { border-left-color: var(--bad); } +.studio-note--ok { border-left-color: var(--ok); } +.studio-errors { list-style: none; margin: 6px 0 0; padding: 0; font-size: 12.5px; color: var(--bad); } +.studio-errors li { padding: 2px 0; font-family: var(--mono); } + +/* course workspace */ +.studio-side-head { padding: 18px 16px 14px; border-bottom: 1px solid var(--line); } +.studio-side-head h2 { font-size: 14.5px; margin: 0; } +.studio-side-head .src { font-size: 12px; color: var(--faint); margin-top: 2px; } +.studio-nav { list-style: none; margin: 0; padding: 6px 0; border-bottom: 1px solid var(--line); } +.studio-nav button, .studio-nav a { display: grid; grid-template-columns: 1fr auto; gap: 6px; width: 100%; align-items: center; padding: 6px 16px; border: 0; background: none; text-align: left; font-size: 13px; color: var(--muted); text-decoration: none; } +.studio-nav button:hover { background: color-mix(in srgb, var(--raised) 60%, transparent); color: var(--text); } +.studio-nav button[aria-current="page"] { background: var(--raised); color: var(--head); box-shadow: inset 2px 0 0 var(--accent); } +.studio-nav .mod { padding: 11px 16px 4px; font-size: 13.5px; color: var(--text); font-weight: 500; } +.studio-nav .k { font-size: 11.5px; color: var(--faint); } +.studio-nav .add { color: var(--faint); font-size: 12.5px; } + +.studio-ws { display: flex; align-items: flex-start; margin: -44px -40px 0; } +.studio-aside { position: sticky; top: var(--top); flex: none; width: var(--sidew); height: calc(100vh - var(--top)); overflow-y: auto; background: var(--side); border-right: 1px solid var(--line); overscroll-behavior: contain; } +.studio-aside .src a { color: var(--muted); } +.studio-content { flex: 1; min-width: 0; padding: 0 40px 64px; } +.studio-bar { position: sticky; top: var(--top); z-index: 5; display: flex; flex-wrap: wrap; align-items: center; gap: 10px 14px; margin: 0 -40px 26px; padding: 10px 40px; background: var(--ground); border-bottom: 1px solid var(--line); } +.studio-bar .where { flex: 1; min-width: 200px; font-size: 12.5px; color: var(--muted); } +.studio-bar .where b { color: var(--text); font-weight: 500; } + +.studio-split { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); gap: 24px; align-items: start; } +.studio-split textarea { min-height: 520px; } +.studio-preview-wrap { border-left: 1px solid var(--line); padding-left: 24px; min-width: 0; } +.studio-preview { --pf: var(--serif); font-size: 17px; } +.studio-preview :where(h1, h2, h3) { font-family: var(--serif); color: var(--head); } +.studio-preview h1 { font-size: 26px; line-height: 1.2; margin: 0 0 .6em; } +.studio-preview pre.code { font-size: 13px; line-height: 1.6; background: var(--side); border: 1px solid var(--line); border-radius: 10px; padding: 12px 14px; overflow-x: auto; color: var(--text); } +.studio-preview :not(pre) > code { font-size: .8em; padding: .08em .32em; background: color-mix(in srgb, var(--raised) 75%, transparent); border-radius: 4px; color: var(--text); } +.studio-preview blockquote { border-left: 2px solid var(--line2); margin: 0 0 1em; padding-left: 16px; color: var(--muted); } +.studio-preview details { border-top: 1px solid var(--line); border-bottom: 1px solid var(--line); margin: 0 0 1.2em; padding: 0 2px; } +.studio-preview summary { padding: 10px 0; font: 500 14px/1.4 var(--sans); color: var(--text); cursor: pointer; } +.studio-preview img { max-width: 100%; } +.studio-preview hr { border: 0; border-top: 1px solid var(--line); margin: 2em 0; } + +.studio-section { padding: 18px 0; border-top: 1px solid var(--line); } +.studio-section:first-of-type { border-top: 0; padding-top: 0; } + +/* conflicts */ +.studio-conflict { border-top: 1px solid var(--line); padding: 16px 0; } +.studio-sides { display: grid; grid-template-columns: 1fr 1fr; gap: 14px; margin: 10px 0; } +.studio-sides pre { margin: 0; max-height: 280px; overflow: auto; font-size: 12.5px; line-height: 1.55; background: var(--side); border: 1px solid var(--line); border-radius: 8px; padding: 10px 12px; white-space: pre-wrap; } + +@media (max-width: 900px) { + .studio-split, .studio-sides, .studio .grid2, .studio .grid3 { grid-template-columns: 1fr; } + .studio-preview-wrap { border-left: 0; padding-left: 0; border-top: 1px solid var(--line); padding-top: 18px; } + .studio-ws { flex-direction: column; margin: -44px -40px 0; } + .studio-aside { position: static; width: 100%; height: auto; max-height: 50vh; border-right: 0; border-bottom: 1px solid var(--line); } + .studio-content { padding: 0 16px 48px; width: 100%; } + .studio-bar { margin: 0 -16px 20px; padding: 10px 16px; } +} diff --git a/packages/feature-studio/src/ui/workspace.ts b/packages/feature-studio/src/ui/workspace.ts new file mode 100644 index 0000000..ec80f89 --- /dev/null +++ b/packages/feature-studio/src/ui/workspace.ts @@ -0,0 +1,110 @@ +import { useCallback, useEffect, useState } from "react"; +import { api, coursePath, rawUrl, type CourseState, type FileConflict, type FileWrite, type ManifestDoc } from "./api"; + +export type SaveOutcome = { ok: true } | { ok: false; conflicts?: FileConflict[]; error?: string }; + +/** What every editor gets: the loaded draft, a local manifest copy shared across editors, and save helpers. */ +export interface Workspace { + courseId: string; + state: CourseState; + doc: ManifestDoc; + docDirty: boolean; + setDoc(update: (d: ManifestDoc) => void): void; + /** One commit: optional files plus the local manifest when it has changes (or when `withManifest`). */ + save(opts: { files?: FileWrite[]; message: string; withManifest?: boolean; manifest?: ManifestDoc }): Promise<SaveOutcome>; + upload(file: File, dir: string): Promise<{ ok: true; path: string } | { ok: false; error: string }>; + reload(): Promise<void>; + /** Bumped after every save so open file editors refetch their base hash. */ + version: number; + resolveUrl(rel: string): string; +} + +export function clone<T>(x: T): T { + return JSON.parse(JSON.stringify(x)) as T; +} + +/** A course file opened for editing, with the hash it was loaded at (for conflict detection on save). */ +export function useFile(courseId: string, path: string | undefined, version: number, initialIfMissing = "") { + const [s, setS] = useState({ path: undefined as string | undefined, content: "", base: "", hash: null as string | null, loaded: false }); + useEffect(() => { + let live = true; + if (!path) { setS({ path, content: "", base: "", hash: null, loaded: true }); return; } + setS((p) => ({ ...p, loaded: false })); + api("GET", coursePath(courseId, `/file?path=${encodeURIComponent(path)}`)).then(({ status, data }) => { + if (!live) return; + if (status === 200) setS({ path, content: data.content, base: data.content, hash: data.hash, loaded: true }); + else setS({ path, content: initialIfMissing, base: "", hash: null, loaded: true }); + }); + return () => { live = false; }; + }, [courseId, path, version]); + return { + path, + content: s.content, + loaded: s.loaded && s.path === path, + exists: s.hash !== null, + dirty: s.content !== s.base, + setContent: (c: string) => setS((p) => ({ ...p, content: c })), + write: (): FileWrite | null => (path ? { path, content: s.content, baseHash: s.hash } : null), + }; +} +export type OpenFile = ReturnType<typeof useFile>; + +export function useWorkspace(courseId: string) { + const [state, setState] = useState<CourseState | null>(null); + const [readOnly, setReadOnly] = useState<{ message: string; title: string; root: string } | null>(null); + const [error, setError] = useState<{ status: number; error?: string } | null>(null); + const [doc, setDocState] = useState<ManifestDoc | null>(null); + const [docDirty, setDocDirty] = useState(false); + const [version, setVersion] = useState(0); + + const reload = useCallback(async () => { + const { status, data } = await api("GET", coursePath(courseId)); + if (status !== 200) { setError({ status, error: data?.error }); return; } + setError(null); + if (data.readOnly) { setReadOnly({ message: data.message, title: data.course.title, root: data.course.root }); return; } + setState(data); + setDocState(data.manifestRaw ? clone(data.manifestRaw) : null); + setDocDirty(false); + }, [courseId]); + + useEffect(() => { reload(); }, [reload]); + + const ws: Workspace | null = state && doc ? { + courseId, state, doc, docDirty, version, + setDoc(update) { + setDocState((d) => { + const n = clone(d!); + update(n); + return n; + }); + setDocDirty(true); + }, + async save({ files = [], message, withManifest, manifest }) { + const body: Record<string, unknown> = { message, files }; + if (manifest || withManifest === true || (withManifest !== false && docDirty)) { + body.manifest = manifest ?? doc; + body.manifestBaseHash = state.manifestHash; + } + const { status, data } = await api("PUT", coursePath(courseId, "/files"), body); + if (status === 200) { + setVersion((v) => v + 1); + await reload(); + return { ok: true }; + } + if (status === 409 && data?.conflicts) return { ok: false, conflicts: data.conflicts }; + return { ok: false, error: data?.error ?? `Save failed (${status})` }; + }, + async upload(file, dir) { + const fd = new FormData(); + fd.set("file", file); + fd.set("dir", dir); + const { status, data } = await api("POST", coursePath(courseId, "/upload"), fd); + if (status !== 200) return { ok: false, error: data?.error ?? `Upload failed (${status})` }; + return { ok: true, path: data.path }; + }, + reload, + resolveUrl: (rel) => rawUrl(courseId, rel), + } : null; + + return { ws, state, readOnly, error, reload, setVersion }; +} diff --git a/packages/feature-studio/src/web.ts b/packages/feature-studio/src/web.ts new file mode 100644 index 0000000..d60a5a4 --- /dev/null +++ b/packages/feature-studio/src/web.ts @@ -0,0 +1,12 @@ +import type { WebFeature } from "@stc/contracts"; + +/** Studio web routes. The pages call /api/studio, which allows only owners, authors and per-course authors. */ +export const studioWeb: WebFeature = { + name: "studio", + routes: [ + { path: "/studio", element: () => import("./ui/StudioHome"), nav: { label: "Studio", area: "header" } }, + { path: "/studio/:courseId", element: () => import("./ui/CourseStudio") }, + ], +}; + +export { setLessonPreview, type LessonPreviewProps, type LessonPreviewComponent } from "./ui/preview"; diff --git a/packages/feature-studio/test/app-helpers.ts b/packages/feature-studio/test/app-helpers.ts index 0c46da1..43b1d11 100644 --- a/packages/feature-studio/test/app-helpers.ts +++ b/packages/feature-studio/test/app-helpers.ts @@ -15,9 +15,20 @@ export const users: Record<string, AuthUser> = { }; /** A server context like main.ts builds, with header-selected users (`x-user: ada`). */ -export async function setupApp(extraCourseDirs: string[] = []) { +export async function setupApp(extraCourseDirs: string[] = [], dbUrl = "sqlite://:memory:") { const dataDir = tempDir(); - const db = await connect("sqlite://:memory:"); + let dropSchema = async () => {}; + if (dbUrl.startsWith("postgres")) { + // a private schema per test, like packages/db's forEachDb + const schema = `t_${Bun.randomUUIDv7().replaceAll("-", "")}`; + const admin = await connect(dbUrl); + await admin.sql.unsafe(`CREATE SCHEMA ${schema}`); + const u = new URL(dbUrl); + u.searchParams.set("options", `-c search_path=${schema}`); + dbUrl = u.toString(); + dropSchema = async () => { await admin.sql.unsafe(`DROP SCHEMA ${schema} CASCADE`); await admin.close(); }; + } + const db = await connect(dbUrl); await migrate(db); const config = { port: 0, databaseUrl: "sqlite://:memory:", dataDir, courseDirs: extraCourseDirs, secret: "t", publicUrl: "http://x" }; const courses = await createCourseRepository(studioCourseDirs(config)); @@ -39,5 +50,5 @@ export async function setupApp(extraCourseDirs: string[] = []) { try { json = JSON.parse(text); } catch { json = text; } return { status: r.status, json }; }; - return { ctx, db, dataDir, feature, app, call, stop: async () => { feature.stop(); await db.close(); } }; + return { ctx, db, dataDir, feature, app, call, stop: async () => { feature.stop(); await db.close(); await dropSchema(); } }; } diff --git a/packages/feature-studio/test/feature.test.ts b/packages/feature-studio/test/feature.test.ts index 2ed53a0..d788c73 100644 --- a/packages/feature-studio/test/feature.test.ts +++ b/packages/feature-studio/test/feature.test.ts @@ -11,8 +11,10 @@ test("anonymous gets 401, learners 403", async () => { expect((await env.call("lea", "POST", "/api/studio/courses", { id: "x", title: "X" })).status).toBe(403); }); -test("creating a course records the creator in course_authors; a per-course author without the role can edit it", async () => { - env = await setupApp(); +const DBS = [["sqlite", "sqlite://:memory:"], ...(process.env.STC_TEST_PG_URL ? [["postgres", process.env.STC_TEST_PG_URL]] : [])] as const; + +test.each(DBS)("creating a course records the creator in course_authors; a per-course author without the role can edit it [%s]", async (_name, url) => { + env = await setupApp([], url); expect((await env.call("ada", "POST", "/api/studio/courses", { id: "c1", title: "C" })).status).toBe(201); const rows = await env.db.sql`SELECT course_id, user_id FROM course_authors`; expect(rows.map((r: any) => [r.course_id, r.user_id])).toEqual([["c1", "u-ada"]]); diff --git a/packages/feature-studio/test/markdown.test.ts b/packages/feature-studio/test/markdown.test.ts new file mode 100644 index 0000000..9db40b9 --- /dev/null +++ b/packages/feature-studio/test/markdown.test.ts @@ -0,0 +1,38 @@ +import { expect, test } from "bun:test"; +import { renderMarkdown } from "../src/ui/markdown"; + +test("headings, paragraphs, emphasis, inline code", () => { + expect(renderMarkdown("# Title\n\nSome **bold** and *em* and `x < y`.\nSecond line.")).toBe( + '<h1 id="title">Title</h1>\n<p>Some <strong>bold</strong> and <em>em</em> and <code>x < y</code>.\nSecond line.</p>', + ); +}); + +test("raw HTML is escaped, except the details/summary hint blocks", () => { + expect(renderMarkdown("<script>alert(1)</script>")).toBe("<p><script>alert(1)</script></p>"); + const h = renderMarkdown("<details>\n<summary>Exercise 1 — nudge</summary>\n\nTry **this**.\n\n</details>"); + expect(h).toMatch(/<details>\s*<summary>Exercise 1 — nudge<\/summary>/); + expect(h).toContain("<p>Try <strong>this</strong>.</p>"); + expect(h).toContain("</details>"); + expect(renderMarkdown("<summary onclick=x>a</summary>")).not.toContain("<summary onclick"); +}); + +test("fenced code keeps content verbatim and escaped", () => { + expect(renderMarkdown("```js\nconst a = '<b>';\n# not a heading\n```")).toBe( + '<pre class="code"><code data-lang="js">const a = \'<b>\';\n# not a heading</code></pre>', + ); +}); + +test("lists, quotes, rules", () => { + expect(renderMarkdown("- a\n- b\n\n1. one\n2. two")).toBe("<ul><li>a</li><li>b</li></ul>\n<ol><li>one</li><li>two</li></ol>"); + expect(renderMarkdown("> quoted\n\n---")).toBe("<blockquote><p>quoted</p></blockquote>\n<hr>"); +}); + +test("links: safe schemes only; images resolved through the given resolver", () => { + expect(renderMarkdown("[ok](https://x.org/a?b=1&c=2)")).toBe('<p><a href="https://x.org/a?b=1&c=2">ok</a></p>'); + expect(renderMarkdown("[bad](javascript:alert(1))")).toBe("<p>bad</p>"); + expect(renderMarkdown("![d](img/a b.png)", { resolveUrl: (u) => `/raw/${encodeURIComponent(u)}` })).toBe('<p><img src="/raw/img%2Fa%20b.png" alt="d"></p>'); +}); + +test("front matter is not rendered", () => { + expect(renderMarkdown("---\ntitle: x\n---\n# H")).toBe('<h1 id="h">H</h1>'); +}); From 7adf9765b124bc23f1a0efbc1ab18733091ce790 Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 03:26:22 +0100 Subject: [PATCH 38/62] fix(import): allowed import roots, hostile zip symlink removal, tests Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- docs/contract-requests/import.md | 1 + packages/feature-import/src/index.ts | 2 + packages/feature-import/src/jobs/code-repo.ts | 2 + .../feature-import/src/jobs/packt-video.ts | 2 + packages/feature-import/src/jobs/pdf-deck.ts | 2 + .../feature-import/src/jobs/video-folder.ts | 2 + packages/feature-import/src/roots.ts | 30 ++++++++++++++ packages/feature-import/src/routes.ts | 15 ++++--- packages/feature-import/src/util/source.ts | 18 +++++++-- .../feature-import/src/web/ImportPage.tsx | 3 +- packages/feature-import/test/harness.ts | 3 +- .../feature-import/test/hostile-zip.test.ts | 39 +++++++++++++++++++ packages/feature-import/test/roots.test.ts | 26 +++++++++++++ packages/feature-import/test/routes.test.ts | 29 +++++++++++++- .../test/video-folder-pdf.test.ts | 2 +- 15 files changed, 164 insertions(+), 12 deletions(-) create mode 100644 packages/feature-import/src/roots.ts create mode 100644 packages/feature-import/test/hostile-zip.test.ts create mode 100644 packages/feature-import/test/roots.test.ts diff --git a/docs/contract-requests/import.md b/docs/contract-requests/import.md index 22513da..4831f36 100644 --- a/docs/contract-requests/import.md +++ b/docs/contract-requests/import.md @@ -9,3 +9,4 @@ - `<dataDir>/cache/transcripts/<courseId>/<lessonId>.json` (`{ lessonId, text, segments: [{ start, end, text }] }`). Suggest a `cacheDir`/`deckPageUrl` helper in core, and a media route for `cache/decks` so the deck viewer can load pages. 5. **Video lessons have no description field.** Packt clips come with a one-line description. It is dropped, because `transcript` is the only Markdown a video lesson can carry and the transcription job writes it. +6. **`STC_IMPORT_ROOTS`** (":"-separated, default `<dataDir>/import-inbox`, created when `importFeature` is built) is read inside the import package. Core config could own it later. diff --git a/packages/feature-import/src/index.ts b/packages/feature-import/src/index.ts index 1d7874a..367fe43 100644 --- a/packages/feature-import/src/index.ts +++ b/packages/feature-import/src/index.ts @@ -5,6 +5,7 @@ import { importPacktVideo } from "./jobs/packt-video"; import { importPdfDeck } from "./jobs/pdf-deck"; import { transcribeCourse } from "./jobs/transcribe"; import { importVideoFolder } from "./jobs/video-folder"; +import { ensureRoots } from "./roots"; import { importRoutes } from "./routes"; export type { ImportOptions } from "./context"; @@ -21,6 +22,7 @@ export function importFeature(ctx: ServerContext, opts: ImportOptions = {}): Fea return { ctx, env, tools: { ffmpeg: env.STC_FFMPEG || "ffmpeg", ffprobe: env.STC_FFPROBE || "ffprobe" }, python: env.STC_PYTHON || "python3" }; }; const job = (fn: (d: Deps, p: unknown, jc: Parameters<JobHandler>[1]) => Promise<unknown>): JobHandler => (p, jc) => fn(deps(), p, jc); + ensureRoots({ ctx, env: opts.env ?? process.env }); return { name: "import", routes: importRoutes(deps), diff --git a/packages/feature-import/src/jobs/code-repo.ts b/packages/feature-import/src/jobs/code-repo.ts index 7286ea1..29aff79 100644 --- a/packages/feature-import/src/jobs/code-repo.ts +++ b/packages/feature-import/src/jobs/code-repo.ts @@ -5,6 +5,7 @@ import { finish, managedCourse, saveManifest, stageDir, type Deps } from "../con import { attachDeck } from "../deck"; import { putFile } from "../media"; import { run } from "../util/exec"; +import { assertAllowedPath } from "../roots"; import { openSource } from "../util/source"; import { containment } from "../util/text"; import { optStr, parsePayload, withStage } from "./common"; @@ -50,6 +51,7 @@ export async function importCodeRepo(d: Deps, payload: unknown, jc: JobContext): const { path, options } = parsePayload(payload, "code-repo"); const warnings: string[] = []; const course = await managedCourse(d, optStr(options, "courseId")); + if (!GIT_URL.test(path)) await assertAllowedPath(d, path); const stage = await stageDir(d, "repo"); return withStage(stage, async () => { diff --git a/packages/feature-import/src/jobs/packt-video.ts b/packages/feature-import/src/jobs/packt-video.ts index 8c2dccf..0103736 100644 --- a/packages/feature-import/src/jobs/packt-video.ts +++ b/packages/feature-import/src/jobs/packt-video.ts @@ -4,12 +4,14 @@ import type { ImportResult, JobContext } from "@stc/contracts"; import { finish, frameCandidates, freshCourseId, publishCourse, saveManifest, sourceBlock, stageDir, type Deps } from "../context"; import { probeDuration } from "../media"; import { parsePacktIndex } from "../packt"; +import { assertAllowedPath } from "../roots"; import { openSource } from "../util/source"; import { adoptCover, maybeQueueTranscription, optBool, optStr, parsePayload, withStage } from "./common"; export async function importPacktVideo(d: Deps, payload: unknown, jc: JobContext): Promise<ImportResult> { const { path, options } = parsePayload(payload, "packt-video"); const warnings: string[] = []; + await assertAllowedPath(d, path); await jc.reportProgress(0.02, "Reading the package"); const src = await openSource(path); diff --git a/packages/feature-import/src/jobs/pdf-deck.ts b/packages/feature-import/src/jobs/pdf-deck.ts index 7cfe3e4..855815a 100644 --- a/packages/feature-import/src/jobs/pdf-deck.ts +++ b/packages/feature-import/src/jobs/pdf-deck.ts @@ -2,12 +2,14 @@ import { stat } from "node:fs/promises"; import { join } from "node:path"; import type { ImportResult, JobContext } from "@stc/contracts"; import { finish, managedCourse, saveManifest, type Deps } from "../context"; +import { assertAllowedPath } from "../roots"; import { attachDeck } from "../deck"; import { optStr, parsePayload } from "./common"; export async function importPdfDeck(d: Deps, payload: unknown, jc: JobContext): Promise<ImportResult> { const { path, options } = parsePayload(payload, "pdf-deck"); const warnings: string[] = []; + await assertAllowedPath(d, path); if (!(await stat(path).catch(() => null))?.isFile() || !/\.pdf$/i.test(path)) throw new Error(`not a PDF file: ${path}`); const course = await managedCourse(d, optStr(options, "courseId")); const attachTo = optStr(options, "attachTo"); diff --git a/packages/feature-import/src/jobs/video-folder.ts b/packages/feature-import/src/jobs/video-folder.ts index b4ff4f4..403f8af 100644 --- a/packages/feature-import/src/jobs/video-folder.ts +++ b/packages/feature-import/src/jobs/video-folder.ts @@ -3,6 +3,7 @@ import { basename, extname, join } from "node:path"; import type { ImportResult, JobContext } from "@stc/contracts"; import { finish, frameCandidates, freshCourseId, publishCourse, saveManifest, sourceBlock, stageDir, type Deps } from "../context"; import { probeDuration, putFile } from "../media"; +import { assertAllowedPath } from "../roots"; import { naturalSort } from "../util/text"; import { openSource } from "../util/source"; import { adoptCover, maybeQueueTranscription, optBool, optStr, parsePayload, withStage } from "./common"; @@ -19,6 +20,7 @@ export function lessonTitle(file: string): string { export async function importVideoFolder(d: Deps, payload: unknown, jc: JobContext): Promise<ImportResult> { const { path, options } = parsePayload(payload, "video-folder"); const warnings: string[] = []; + await assertAllowedPath(d, path); if (!(await stat(path).catch(() => null))?.isDirectory()) throw new Error(`not a folder: ${path}`); const src = await openSource(path); const files = naturalSort(src.entries.filter((e) => VIDEO.test(e) && !e.split("/").some((p) => p.startsWith(".")))); diff --git a/packages/feature-import/src/roots.ts b/packages/feature-import/src/roots.ts new file mode 100644 index 0000000..621c96f --- /dev/null +++ b/packages/feature-import/src/roots.ts @@ -0,0 +1,30 @@ +import { mkdirSync } from "node:fs"; +import { realpath } from "node:fs/promises"; +import { resolve, sep } from "node:path"; +import type { Deps } from "./context"; + +export const OUTSIDE = "path is outside the allowed import folders"; + +/** STC_IMPORT_ROOTS (":"-separated), default <dataDir>/import-inbox. */ +export function importRoots(d: Pick<Deps, "ctx" | "env">): string[] { + const raw = d.env.STC_IMPORT_ROOTS; + const list = raw ? raw.split(":").filter(Boolean) : [resolve(d.ctx.config.dataDir, "import-inbox")]; + return list.map((r) => resolve(r)); +} + +export function ensureRoots(d: Pick<Deps, "ctx" | "env">): void { + for (const r of importRoots(d)) { try { mkdirSync(r, { recursive: true }); } catch { /* reported when used */ } } +} + +/** Real path (symlinks followed) must lie inside the real path of one root. Throws OUTSIDE otherwise. */ +export async function assertAllowedPath(d: Pick<Deps, "ctx" | "env">, path: string): Promise<string> { + let real: string; + try { real = await realpath(resolve(path)); } catch { throw new Error(OUTSIDE); } + for (const root of importRoots(d)) { + const rr = await realpath(root).catch(() => null); + if (rr && (real === rr || real.startsWith(rr + sep))) return real; + } + throw new Error(OUTSIDE); +} + +export const isGitUrl = (p: string) => /^(https?:\/\/|ssh:\/\/|git@)\S+$/.test(p); diff --git a/packages/feature-import/src/routes.ts b/packages/feature-import/src/routes.ts index f1cd2b0..c44570f 100644 --- a/packages/feature-import/src/routes.ts +++ b/packages/feature-import/src/routes.ts @@ -1,9 +1,9 @@ import { copyFile, mkdir, readdir } from "node:fs/promises"; -import { existsSync } from "node:fs"; import { join, relative, resolve } from "node:path"; import type { Feature, RouteHandler } from "@stc/contracts"; import { serveMedia } from "@stc/core"; -import { coverCacheDir, coursesDir, exists, managedCourse, saveManifest, type Deps } from "./context"; +import { assertAllowedPath, importRoots, OUTSIDE } from "./roots"; +import { coverCacheDir, coursesDir, exists, isPlainId, managedCourse, saveManifest, type Deps } from "./context"; export const KINDS = ["packt-video", "code-repo", "pdf-deck", "video-folder"] as const; const OPTION_KEYS = ["title", "courseId", "attachTo", "media", "transcribe"] as const; @@ -26,7 +26,7 @@ export function importRoutes(deps: () => Deps): Feature["routes"] { const d = deps(); const dir = coursesDir(d); const served = d.ctx.config.courseDirs.some((c) => resolve(c) === dir); - return json({ transcribe: Boolean(d.env.STC_TRANSCRIBE_URL), coursesDir: dir, coursesDirServed: served }); + return json({ transcribe: Boolean(d.env.STC_TRANSCRIBE_URL), coursesDir: dir, coursesDirServed: served, importRoots: importRoots(d) }); }), }, "/api/import/jobs": { @@ -50,8 +50,10 @@ export function importRoutes(deps: () => Deps): Feature["routes"] { const path = typeof body?.path === "string" ? body.path.trim() : ""; if (!path) return json({ error: "path is required" }, 400); const isGit = kind === "code-repo" && GIT_URL.test(path); - if (!isGit && !existsSync(path)) return json({ error: `Path not found on the server: ${path}` }, 400); - if (!isGit && !path.startsWith("/")) return json({ error: "Use an absolute path" }, 400); + if (!isGit) { + if (!path.startsWith("/")) return json({ error: "Use an absolute path" }, 400); + try { await assertAllowedPath(d, path); } catch { return json({ error: OUTSIDE }, 400); } + } if ((kind === "code-repo" || kind === "pdf-deck") && typeof options.courseId !== "string") return json({ error: "courseId is required" }, 400); if (kind === "pdf-deck" && typeof options.attachTo !== "string") return json({ error: "attachTo is required" }, 400); return json({ id: await d.ctx.jobs.enqueue(`import.${kind}`, { kind, path, options }) }, 202); @@ -74,6 +76,7 @@ export function importRoutes(deps: () => Deps): Feature["routes"] { GET: owner(async (_req, rc) => { const d = deps(); const id = rc.params.courseId!; + if (!isPlainId(id)) return json({ error: "Invalid course id" }, 400); const c = d.ctx.courses.get(id); const names = (await readdir(coverCacheDir(d, id)).catch(() => [] as string[])).filter((n) => COVER_NAME.test(n)).sort(); return json({ current: c?.manifest?.cover ?? null, candidates: names }); @@ -84,6 +87,7 @@ export function importRoutes(deps: () => Deps): Feature["routes"] { try { body = await req.json(); } catch { return json({ error: "Body must be JSON" }, 400); } const name = String(body?.name ?? ""); if (!COVER_NAME.test(name)) return json({ error: "Invalid candidate name" }, 400); + if (!isPlainId(rc.params.courseId!)) return json({ error: "Invalid course id" }, 400); const src = join(coverCacheDir(d, rc.params.courseId!), name); if (!(await exists(src))) return json({ error: "No such candidate" }, 404); let course; @@ -100,6 +104,7 @@ export function importRoutes(deps: () => Deps): Feature["routes"] { GET: owner(async (req, rc) => { const name = rc.params.name!; if (!COVER_NAME.test(name)) return new Response("Not found", { status: 404 }); + if (!isPlainId(rc.params.courseId!)) return new Response("Not found", { status: 404 }); const file = join(coverCacheDir(deps(), rc.params.courseId!), name); if (!(await exists(file))) return new Response("Not found", { status: 404 }); return serveMedia(req, file); diff --git a/packages/feature-import/src/util/source.ts b/packages/feature-import/src/util/source.ts index 43bf664..40cb297 100644 --- a/packages/feature-import/src/util/source.ts +++ b/packages/feature-import/src/util/source.ts @@ -1,4 +1,4 @@ -import { cp, mkdir, readdir, readFile, stat } from "node:fs/promises"; +import { cp, mkdir, readdir, readFile, rm, stat } from "node:fs/promises"; import { dirname, join } from "node:path"; import { run } from "./exec"; @@ -22,6 +22,14 @@ async function walk(root: string, rel = "", out: string[] = []): Promise<string[ return out; } +async function removeSymlinks(dir: string): Promise<void> { + for (const e of await readdir(dir, { withFileTypes: true })) { + const p = join(dir, e.name); + if (e.isSymbolicLink()) await rm(p, { force: true }); + else if (e.isDirectory()) await removeSymlinks(p); + } +} + export async function openSource(path: string): Promise<Source> { const st = await stat(path).catch(() => null); if (!st) throw new Error(`source not found: ${path}`); @@ -47,8 +55,12 @@ export async function openSource(path: string): Promise<Source> { async extract(list, dest, signal) { await mkdir(dest, { recursive: true }); // unzip treats arguments as patterns, so glob characters in names are escaped; batches keep argv small. - for (let i = 0; i < list.length; i += 200) { - await run(["unzip", "-q", "-o", path, ...list.slice(i, i + 200).map(globEscape), "-d", dest], { signal }); + try { + for (let i = 0; i < list.length; i += 200) { + await run(["unzip", "-q", "-o", path, ...list.slice(i, i + 200).map(globEscape), "-d", dest], { signal }); + } + } finally { + await removeSymlinks(dest); // a zip can carry symlinks; none may survive extraction } }, }; diff --git a/packages/feature-import/src/web/ImportPage.tsx b/packages/feature-import/src/web/ImportPage.tsx index 44ebbdd..dd6a51d 100644 --- a/packages/feature-import/src/web/ImportPage.tsx +++ b/packages/feature-import/src/web/ImportPage.tsx @@ -5,7 +5,7 @@ import "./import.css"; type Kind = "packt-video" | "code-repo" | "pdf-deck" | "video-folder"; interface Job { id: string; kind: string; status: "queued" | "running" | "done" | "failed"; progress: number; message: string | null; payload: any; result: any; error: string | null; createdAt: string } interface CourseInfo { id: string; title: string; modules: { id: string; title: string; lessons: { id: string; title: string }[] }[] } -interface Info { transcribe: boolean; coursesDir: string; coursesDirServed: boolean } +interface Info { importRoots: string[]; transcribe: boolean; coursesDir: string; coursesDirServed: boolean } const KINDS: { value: Kind; label: string; hint: string; placeholder: string }[] = [ { value: "packt-video", label: "Packt video zip", hint: "A video package whose index.html lists the sections and clips.", placeholder: "/data/downloads/video_zip.zip" }, @@ -139,6 +139,7 @@ export default function ImportPage() { <label className="imp-field">Path on the server <input type="text" className="mono" value={path} onChange={(e) => setPath(e.target.value)} placeholder={spec.placeholder} required /> <span className="imp-hint">{spec.hint}</span> + {info ? <span className="imp-hint">Allowed folders: {info.importRoots.map((r) => <code key={r}>{r} </code>)}(set <code>STC_IMPORT_ROOTS</code>; git URLs are exempt)</span> : null} </label> {!needsCourse ? ( <label className="imp-field">Course title (optional) diff --git a/packages/feature-import/test/harness.ts b/packages/feature-import/test/harness.ts index d39d528..ff22cc2 100644 --- a/packages/feature-import/test/harness.ts +++ b/packages/feature-import/test/harness.ts @@ -1,3 +1,4 @@ +import { tmpdir } from "node:os"; import { join } from "node:path"; import type { JobContext, ServerContext } from "@stc/contracts"; import { createCourseRepository } from "@stc/course-format"; @@ -17,5 +18,5 @@ export async function makeEnv(env: Record<string, string | undefined> = {}): Pro jobs: { enqueue: async (kind: string, payload: unknown) => { queued.push({ kind, payload }); return "j1"; }, get: async () => null, list: async () => [] }, } as unknown as ServerContext; const jc: JobContext = { signal: new AbortController().signal, reportProgress: async (f, m) => { progress.push([f, m]); } }; - return { deps: { ctx, env, tools: { ffmpeg: "ffmpeg", ffprobe: "ffprobe" }, python: PYTHON }, dataDir, queued, jc, progress }; + return { deps: { ctx, env: { STC_IMPORT_ROOTS: tmpdir(), ...env }, tools: { ffmpeg: "ffmpeg", ffprobe: "ffprobe" }, python: PYTHON }, dataDir, queued, jc, progress }; } diff --git a/packages/feature-import/test/hostile-zip.test.ts b/packages/feature-import/test/hostile-zip.test.ts new file mode 100644 index 0000000..3fabdeb --- /dev/null +++ b/packages/feature-import/test/hostile-zip.test.ts @@ -0,0 +1,39 @@ +import { expect, test } from "bun:test"; +import { existsSync } from "node:fs"; +import { readdir } from "node:fs/promises"; +import { join } from "node:path"; +import { openSource } from "../src/util/source"; +import { PYTHON, tmp } from "./fixtures"; +import { run } from "../src/util/exec"; + +/** Builds a zip by hand with traversal, absolute and symlink entries. */ +async function hostileZip(path: string) { + await run([PYTHON, "-I", "-c", ` +import sys, zipfile, stat +z = zipfile.ZipFile(sys.argv[1], 'w') +z.writestr('../../evil-up.txt', 'x') +z.writestr('/tmp/stc-evil-abs.txt', 'x') +z.writestr('ok/fine.txt', 'fine') +i = zipfile.ZipInfo('ok/link'); i.create_system = 3; i.external_attr = (stat.S_IFLNK | 0o777) << 16 +z.writestr(i, '/etc') +i2 = zipfile.ZipInfo('ok/link/pwned.txt'); z.writestr(i2, 'x') +z.close()`, path]); +} + +test("hostile zip entries never write outside the extraction dir", async () => { + const base = await tmp("stc-zip-"); + await hostileZip(join(base, "h.zip")); + const src = await openSource(join(base, "h.zip")); + const dest = join(base, "a", "b", "out"); + await src.extract(src.entries, dest).catch(() => {}); // unzip may warn; what matters is where files land + expect(existsSync(join(base, "evil-up.txt"))).toBe(false); + expect(existsSync(join(base, "a", "evil-up.txt"))).toBe(false); + expect(existsSync(join(base, "..", "evil-up.txt"))).toBe(false); + expect(existsSync("/tmp/stc-evil-abs.txt")).toBe(false); + expect(existsSync("/etc/pwned.txt")).toBe(false); + expect(await Bun.file(join(dest, "ok/fine.txt")).text()).toBe("fine"); + const left: string[] = []; + const walk = async (d: string) => { for (const e of await readdir(d, { withFileTypes: true })) { if (e.isSymbolicLink()) left.push(e.name); else if (e.isDirectory()) await walk(join(d, e.name)); } }; + await walk(dest); + expect(left).toEqual([]); // the symlink entry is dropped +}); diff --git a/packages/feature-import/test/roots.test.ts b/packages/feature-import/test/roots.test.ts new file mode 100644 index 0000000..84ffbe5 --- /dev/null +++ b/packages/feature-import/test/roots.test.ts @@ -0,0 +1,26 @@ +import { expect, test } from "bun:test"; +import { mkdir, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { importCodeRepo } from "../src/jobs/code-repo"; +import { importPdfDeck } from "../src/jobs/pdf-deck"; +import { importVideoFolder } from "../src/jobs/video-folder"; +import { importPacktVideo } from "../src/jobs/packt-video"; +import { makeEnv } from "./harness"; +import { makePdf, tmp } from "./fixtures"; + +test("every job re-checks its path against the roots", async () => { + const inbox = await tmp("stc-inbox-"); + const outside = await tmp("stc-outside-"); + await makePdf(join(outside, "a.pdf")); + await symlink(outside, join(inbox, "link")); + const env = await makeEnv({ STC_IMPORT_ROOTS: inbox }); + const msg = /outside the allowed import folders/; + await expect(importVideoFolder(env.deps, { path: outside }, env.jc)).rejects.toThrow(msg); + await expect(importVideoFolder(env.deps, { path: join(inbox, "link") }, env.jc)).rejects.toThrow(msg); + await expect(importPacktVideo(env.deps, { path: join(outside, "x.zip") }, env.jc)).rejects.toThrow(msg); + await expect(importPdfDeck(env.deps, { path: join(outside, "a.pdf"), options: {} }, env.jc)).rejects.toThrow(msg); + await expect(importCodeRepo(env.deps, { path: outside, options: { courseId: "x" } }, env.jc)).rejects.toThrow(/created|outside/); + await mkdir(join(inbox, "empty")); + await writeFile(join(inbox, "empty", "n.txt"), ""); + await expect(importVideoFolder(env.deps, { path: join(inbox, "empty") }, env.jc)).rejects.toThrow(/no video files/); // allowed, then fails on content +}); diff --git a/packages/feature-import/test/routes.test.ts b/packages/feature-import/test/routes.test.ts index a80b91e..3d2f359 100644 --- a/packages/feature-import/test/routes.test.ts +++ b/packages/feature-import/test/routes.test.ts @@ -1,10 +1,12 @@ import { expect } from "bun:test"; +import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AuthUser, ServerContext } from "@stc/contracts"; import { createApp, createJobQueue } from "@stc/core"; import { createCourseRepository } from "@stc/course-format"; import { forEachDb } from "../../db/test/helpers"; import { importFeature } from "../src"; +import { mkdir, stat, symlink } from "node:fs/promises"; import { makeVideo, tmp } from "./fixtures"; const owner: AuthUser = { id: "o", email: "o@x", name: "O", roles: ["owner"], groups: [] }; @@ -18,7 +20,7 @@ async function setup(db: any, env: Record<string, string | undefined> = {}) { db, courses, auth: { userFromRequest: async (req: Request) => ({ owner, learner }[req.headers.get("x-user") ?? ""] ?? null) }, } as unknown as ServerContext; - const feature = importFeature(ctx, { env: { STC_PYTHON: process.env.STC_PYTHON, ...env } }); + const feature = importFeature(ctx, { env: { STC_PYTHON: process.env.STC_PYTHON, STC_IMPORT_ROOTS: tmpdir(), ...env } }); const queue = createJobQueue(db, feature.jobs!, { pollMs: 10 }); ctx.jobs = queue; const app = createApp(ctx, [feature]); @@ -46,6 +48,7 @@ forEachDb("job creation validates input", async (db) => { expect((await post({ kind: "nope", path: "/tmp" })).status).toBe(400); expect((await post({ kind: "video-folder" })).status).toBe(400); expect((await post({ kind: "video-folder", path: "/definitely/not/here" })).status).toBe(400); + expect((await post({ kind: "video-folder", path: "relative" })).status).toBe(400); expect((await post({ kind: "pdf-deck", path: "/tmp", options: { courseId: "x" } })).status).toBe(400); expect((await post({ kind: "code-repo", path: "/tmp" })).status).toBe(400); expect((await post({ kind: "transcribe", options: { courseId: "x" } })).status).toBe(409); @@ -86,3 +89,27 @@ forEachDb("a folder import runs through the real queue, then cover candidates ca expect(await Bun.file(join(dataDir, "courses/queued/media/cover.jpg")).exists()).toBe(true); } finally { await queue.stop(); } }); + +forEachDb("paths must stay inside the import roots", async (db) => { + const inbox = await tmp("stc-inbox-"); + const outside = await tmp("stc-outside-"); + await mkdir(join(inbox, "ok")); + await symlink(outside, join(inbox, "escape")); + const { call, ctx } = await setup(db, { STC_IMPORT_ROOTS: inbox }); + const post = (path: string) => call("/api/import/jobs", { as: "owner", method: "POST", json: { kind: "video-folder", path } }); + const bad = await post(outside); + expect(bad.status).toBe(400); + expect(await bad.json()).toEqual({ error: "path is outside the allowed import folders" }); + expect((await post(join(inbox, "..", "x"))).status).toBe(400); + expect((await post(join(inbox, "escape"))).status).toBe(400); // symlink out of the root + expect((await post("/etc")).status).toBe(400); + expect((await post(join(inbox, "ok"))).status).toBe(202); + const info = (await (await call("/api/import/info", { as: "owner" })).json()) as any; + expect(info.importRoots).toEqual([inbox]); + void ctx; +}); + +forEachDb("the default root is created under dataDir", async (db) => { + const { dataDir } = await setup(db, { STC_IMPORT_ROOTS: undefined }); + expect((await stat(join(dataDir, "import-inbox"))).isDirectory()).toBe(true); +}); diff --git a/packages/feature-import/test/video-folder-pdf.test.ts b/packages/feature-import/test/video-folder-pdf.test.ts index 1df5d4b..a319c82 100644 --- a/packages/feature-import/test/video-folder-pdf.test.ts +++ b/packages/feature-import/test/video-folder-pdf.test.ts @@ -61,7 +61,7 @@ test("deck attach validates the target and refuses courses the importer did not await expect(importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: r.courseId } }, env.jc)).rejects.toThrow(/attachTo is required/); await expect(importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: "sample", attachTo: "x" } }, env.jc)).rejects.toThrow(/only edits courses it created/); await expect(importPdfDeck(env.deps, { path: join(d, "a.pdf"), options: { courseId: "../x", attachTo: "x" } }, env.jc)).rejects.toThrow(/only edits courses it created/); - await expect(importPdfDeck(env.deps, { path: join(d, "missing.pdf"), options: {} }, env.jc)).rejects.toThrow(/not a PDF/); + await expect(importPdfDeck(env.deps, { path: join(d, "missing.pdf"), options: {} }, env.jc)).rejects.toThrow(/not a PDF|outside the allowed/); }); test("without PyMuPDF the deck is still attached, with a warning", async () => { From 9ff58623828ae7f53a9b69e0ab4535a2489e9ee9 Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 03:26:32 +0100 Subject: [PATCH 39/62] feat(learner): JS/TS exercise runner in a Web Worker with timeout and network blocked Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .../feature-learner/src/server/feature.ts | 2 + .../feature-learner/src/server/transpile.ts | 27 +++++ .../feature-learner/src/web/runner/runner.ts | 73 ++++++++++++ .../src/web/runner/worker-source.ts | 104 ++++++++++++++++++ packages/feature-learner/test/runner.test.ts | 62 +++++++++++ 5 files changed, 268 insertions(+) create mode 100644 packages/feature-learner/src/server/transpile.ts create mode 100644 packages/feature-learner/src/web/runner/runner.ts create mode 100644 packages/feature-learner/src/web/runner/worker-source.ts create mode 100644 packages/feature-learner/test/runner.test.ts diff --git a/packages/feature-learner/src/server/feature.ts b/packages/feature-learner/src/server/feature.ts index a13c159..513bc79 100644 --- a/packages/feature-learner/src/server/feature.ts +++ b/packages/feature-learner/src/server/feature.ts @@ -7,6 +7,7 @@ import { libraryRoutes } from "./catalog"; import { lessonRoutes } from "./lesson"; import { userDataRoutes } from "./userdata"; import { codeRoutes } from "./code"; +import { transpileRoutes } from "./transpile"; /** * The learner feature: auth, library, reader, notes, progress and exercises. @@ -30,6 +31,7 @@ export function learnerFeature(ctx: ServerContext): Feature & { auth: AuthServic ...lessonRoutes(ctx), ...userDataRoutes(ctx), ...codeRoutes(ctx), + ...transpileRoutes(), }; // Every handler waits for the learner migrations (a no-op once Phase 2 runs them in migrate()). for (const methods of Object.values(routes)) { diff --git a/packages/feature-learner/src/server/transpile.ts b/packages/feature-learner/src/server/transpile.ts new file mode 100644 index 0000000..09e6aca --- /dev/null +++ b/packages/feature-learner/src/server/transpile.ts @@ -0,0 +1,27 @@ +import type { Feature } from "@stc/contracts"; +import { badRequest, body, json } from "./http"; + +const MAX = 200_000; +const ts = new Bun.Transpiler({ loader: "ts", target: "browser" }); + +/** Type-strips TypeScript for the in-browser runner (the worker itself runs plain JavaScript). */ +export function transpileTs(code: string): string { + return ts.transformSync(code); +} + +export function transpileRoutes(): Feature["routes"] { + return { + "/api/learner/transpile": { + POST: async (req) => { + const b = await body<{ code?: string }>(req); + if (!b || typeof b.code !== "string") return badRequest("code is required"); + if (b.code.length > MAX) return badRequest("code is too long"); + try { + return json({ code: transpileTs(b.code) }); + } catch (e) { + return json({ error: (e as Error).message }, 422); + } + }, + }, + }; +} diff --git a/packages/feature-learner/src/web/runner/runner.ts b/packages/feature-learner/src/web/runner/runner.ts new file mode 100644 index 0000000..793f596 --- /dev/null +++ b/packages/feature-learner/src/web/runner/runner.ts @@ -0,0 +1,73 @@ +import type { ExerciseLanguage, ExerciseRunner, TestResult } from "@stc/contracts"; +import { WORKER_SOURCE } from "./worker-source"; + +export interface RunnerOptions { + /** TypeScript → JavaScript. Defaults to the server's `POST /api/learner/transpile` (Bun.Transpiler). */ + transpile?: (code: string) => Promise<string>; + createWorker?: () => Worker; +} + +export async function serverTranspile(code: string): Promise<string> { + const r = await fetch("/api/learner/transpile", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ code }) }); + const b = (await r.json()) as { code?: string; error?: string }; + if (!r.ok || typeof b.code !== "string") throw new Error(b.error ?? "TypeScript could not be compiled"); + return b.code; +} + +const defaultWorker = () => new Worker(URL.createObjectURL(new Blob([WORKER_SOURCE], { type: "text/javascript" })), { type: "module" }); + +/** + * Runs learner code against a tests module in a fresh Web Worker (never in the page), then terminates it. + * When `timeoutMs` passes, the worker is killed and every test without a result is reported as timed out. + */ +export function createBrowserRunner(language: ExerciseLanguage, opts: RunnerOptions = {}): ExerciseRunner { + const transpile = opts.transpile ?? serverTranspile; + const make = opts.createWorker ?? defaultWorker; + return { + language, + async run(learnerCode, testsModule, timeoutMs) { + let code = learnerCode, tests = testsModule; + if (language === "ts") { + try { code = await transpile(learnerCode); } catch (e) { + return [{ name: "Your code did not compile", passed: false, message: (e as Error).message }]; + } + tests = await transpile(testsModule); + } + const worker = make(); + return new Promise<TestResult[]>((resolve) => { + let names: string[] = []; + const results: TestResult[] = []; + let settled = false; + const finish = (extra: TestResult[] = []) => { + if (settled) return; + settled = true; + clearTimeout(timer); + worker.terminate(); + resolve([...results, ...extra]); + }; + const pending = (message: string) => names.slice(results.length).map((name) => ({ name, passed: false, message })); + const timer = setTimeout(() => { + const rest = pending(`Timed out after ${timeoutMs} ms. Is there an endless loop?`); + finish(rest.length ? rest : [{ name: "Timed out", passed: false, message: `No result after ${timeoutMs} ms` }]); + }, timeoutMs); + worker.onmessage = (ev: MessageEvent) => { + const m = ev.data as { type: string; names?: string[]; result?: TestResult; stage?: string; message?: string }; + if (m.type === "tests") names = m.names ?? []; + else if (m.type === "result" && m.result) results.push(m.result); + else if (m.type === "done") finish(); + else if (m.type === "error") { + if (m.stage === "code") { + const rest = pending(`Your code did not load. ${m.message}`); + finish(rest.length ? rest : [{ name: "Your code did not load", passed: false, message: m.message }]); + } else finish([{ name: "The tests did not load", passed: false, message: m.message }]); + } + }; + worker.onerror = (ev: ErrorEvent) => { + ev.preventDefault?.(); + finish(pending(ev.message || "The exercise worker crashed")); + }; + worker.postMessage({ code, tests }); + }); + }, + }; +} diff --git a/packages/feature-learner/src/web/runner/worker-source.ts b/packages/feature-learner/src/web/runner/worker-source.ts new file mode 100644 index 0000000..c31fd75 --- /dev/null +++ b/packages/feature-learner/src/web/runner/worker-source.ts @@ -0,0 +1,104 @@ +/** + * Source of the exercise Web Worker, kept as a string so it can be started from a blob URL without a + * separate bundle. Protocol (worker → page): {type:"tests", names}, {type:"result", result}, {type:"done"}, + * {type:"error", stage:"tests"|"code", message}. Page → worker: {code, tests} (plain JavaScript ES modules). + */ +export const WORKER_SOURCE = String.raw` +"use strict"; +const blocked = () => { throw new Error("Network access is turned off while exercises run"); }; +for (const k of ["fetch", "XMLHttpRequest", "WebSocket", "EventSource", "importScripts"]) { + try { Object.defineProperty(self, k, { value: blocked, configurable: false, writable: false }); } catch (_) {} +} + +class AssertionError extends Error { + constructor(message, expected, actual) { super(message); this.name = "AssertionError"; this.expected = expected; this.actual = actual; this.hasValues = arguments.length > 1; } +} + +function fmt(v, seen = new Set()) { + if (v === undefined) return "undefined"; + if (v === null) return "null"; + if (typeof v === "string") return JSON.stringify(v); + if (typeof v === "number") return Object.is(v, -0) ? "-0" : String(v); + if (typeof v === "bigint") return v + "n"; + if (typeof v === "function") return "[Function " + (v.name || "anonymous") + "]"; + if (typeof v === "symbol") return v.toString(); + if (typeof v !== "object") return String(v); + if (seen.has(v)) return "[Circular]"; + seen.add(v); + let out; + if (Array.isArray(v)) out = "[" + v.map((x) => fmt(x, seen)).join(", ") + "]"; + else if (v instanceof Map) out = "Map {" + [...v].map(([k, x]) => fmt(k, seen) + " => " + fmt(x, seen)).join(", ") + "}"; + else if (v instanceof Set) out = "Set {" + [...v].map((x) => fmt(x, seen)).join(", ") + "}"; + else if (v instanceof Date) out = "Date(" + v.toISOString() + ")"; + else if (v instanceof Error) out = v.name + ": " + v.message; + else out = "{" + Object.keys(v).map((k) => (/^[A-Za-z_$][\w$]*$/.test(k) ? k : JSON.stringify(k)) + ": " + fmt(v[k], seen)).join(", ") + "}"; + seen.delete(v); + return out; +} + +function deepEq(a, b, seen = new Map()) { + if (Object.is(a, b)) return true; + if (typeof a !== "object" || typeof b !== "object" || a === null || b === null) return false; + if (Object.getPrototypeOf(a) !== Object.getPrototypeOf(b)) return false; + if (seen.get(a) === b) return true; + seen.set(a, b); + if (a instanceof Date) return a.getTime() === b.getTime(); + if (a instanceof Map || a instanceof Set) { + if (a.size !== b.size) return false; + if (a instanceof Set) { for (const x of a) if (!b.has(x)) return false; return true; } + for (const [k, x] of a) if (!b.has(k) || !deepEq(x, b.get(k), seen)) return false; + return true; + } + const ka = Object.keys(a), kb = Object.keys(b); + if (ka.length !== kb.length) return false; + for (const k of ka) if (!Object.prototype.hasOwnProperty.call(b, k) || !deepEq(a[k], b[k], seen)) return false; + return true; +} + +const assert = { + equal(actual, expected, message) { + if (!Object.is(actual, expected) && !(actual === expected)) throw new AssertionError(message || "Expected values to be strictly equal", expected, actual); + }, + deepEqual(actual, expected, message) { + if (!deepEq(actual, expected)) throw new AssertionError(message || "Expected values to be deeply equal", expected, actual); + }, + ok(value, message) { + if (!value) throw new AssertionError(message || "Expected a truthy value", true, value); + }, +}; + +const load = (src) => import(URL.createObjectURL(new Blob([src], { type: "text/javascript" }))); +const describe = (e) => (e && e.name && e.message ? e.name + ": " + e.message : String(e)); + +self.onmessage = async (ev) => { + const { code, tests } = ev.data; + let list; + try { + const t = await load(tests); + list = Array.isArray(t.tests) ? t.tests : Array.isArray(t.default) ? t.default : null; + if (!list) throw new Error("the tests module must export \"tests\" (an array)"); + } catch (e) { + self.postMessage({ type: "error", stage: "tests", message: describe(e) }); + return; + } + self.postMessage({ type: "tests", names: list.map((t, i) => String(t && t.name || "test " + (i + 1))) }); + let mod; + try { mod = await load(code); } catch (e) { + self.postMessage({ type: "error", stage: "code", message: describe(e) }); + return; + } + for (let i = 0; i < list.length; i++) { + const t = list[i]; + const name = String(t && t.name || "test " + (i + 1)); + try { + await t.run(mod, assert); + self.postMessage({ type: "result", result: { name, passed: true } }); + } catch (e) { + const r = { name, passed: false, message: describe(e) }; + if (e instanceof AssertionError && e.hasValues) { r.expected = fmt(e.expected); r.actual = fmt(e.actual); r.message = e.message; } + self.postMessage({ type: "result", result: r }); + } + } + self.postMessage({ type: "done" }); +}; +`; diff --git a/packages/feature-learner/test/runner.test.ts b/packages/feature-learner/test/runner.test.ts new file mode 100644 index 0000000..0fd1724 --- /dev/null +++ b/packages/feature-learner/test/runner.test.ts @@ -0,0 +1,62 @@ +import { expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { createBrowserRunner } from "../src/web/runner/runner"; +import { transpileTs } from "../src/server/transpile"; +import { FIXTURES } from "./app"; + +const ex = (f: string) => readFileSync(join(FIXTURES, "sample/exercises/add", f), "utf8"); +const js = createBrowserRunner("js"); + +test("the sample starter fails with expected and actual values", async () => { + const r = await js.run(ex("starter.js"), ex("tests.js"), 5000); + expect(r).toEqual([ + { name: "adds 2 and 3", passed: false, message: "Expected values to be strictly equal", expected: "5", actual: "-1" }, + { name: "adds -1 and 1", passed: false, message: "Expected values to be strictly equal", expected: "0", actual: "-2" }, + ]); +}); + +test("the sample exercise passes after editing", async () => { + const edited = ex("starter.js").replace("a - b", "a + b"); + expect((await js.run(edited, ex("tests.js"), 5000)).every((t) => t.passed)).toBe(true); +}); + +test("syntax errors and missing exports are reported per test", async () => { + const r = await js.run("export function add(a, b) { return a +", ex("tests.js"), 5000); + expect(r.length).toBe(2); + expect(r[0]!.passed).toBe(false); + expect(r[0]!.message).toContain("did not load"); + const r2 = await js.run("export const nope = 1", ex("tests.js"), 5000); + expect(r2[0]!.message).toContain("TypeError"); +}); + +test("an endless loop is killed by the timeout", async () => { + const t0 = Date.now(); + const r = await js.run("export function add() { while (true) {} }", ex("tests.js"), 400); + expect(Date.now() - t0).toBeLessThan(3000); + expect(r.map((x) => x.passed)).toEqual([false, false]); + expect(r[0]!.message).toContain("Timed out"); +}); + +test("network access is blocked inside the worker", async () => { + const tests = `export const tests = [{ name: "net", run: async (m, assert) => assert.equal(await m.probe(), "blocked") }];`; + const code = `export async function probe() { try { await fetch("http://127.0.0.1:1/"); return "open"; } catch (e) { return String(e.message).includes("turned off") ? "blocked" : "other " + e.message; } }`; + expect((await js.run(code, tests, 3000))[0]!.passed).toBe(true); +}); + +test("deepEqual compares structures", async () => { + const tests = `export const tests = [ + { name: "same", run: (m, a) => a.deepEqual(m.v(), { x: [1, 2], y: { z: null } }) }, + { name: "diff", run: (m, a) => a.deepEqual(m.v(), { x: [1, 3] }) }, + ];`; + const r = await js.run("export const v = () => ({ x: [1, 2], y: { z: null } })", tests, 3000); + expect(r[0]!.passed).toBe(true); + expect(r[1]).toMatchObject({ passed: false, expected: "{x: [1, 3]}", actual: "{x: [1, 2], y: {z: null}}" }); +}); + +test("TypeScript exercises are transpiled first", async () => { + const runner = createBrowserRunner("ts", { transpile: async (c) => transpileTs(c) }); + const tests = `import type { X } from "./x"; export const tests = [{ name: "typed", run: (m: any, a: any) => a.equal(m.add(1, 2), 3) }];`; + const r = await runner.run("export function add(a: number, b: number): number { return a + b; }", tests, 3000); + expect(r).toEqual([{ name: "typed", passed: true }]); +}); From 2849d3c911273af35a4a08d91389389ea60e3731 Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 03:26:58 +0100 Subject: [PATCH 40/62] docs(studio): contract requests (ctx feature factories, worktrees course dir, LessonView swap, web subpath, nav roles, DraftWriter) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- docs/contract-requests/studio.md | 82 ++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 docs/contract-requests/studio.md diff --git a/docs/contract-requests/studio.md b/docs/contract-requests/studio.md new file mode 100644 index 0000000..f5d1045 --- /dev/null +++ b/docs/contract-requests/studio.md @@ -0,0 +1,82 @@ +# Contract requests from WP-C (studio) + +Each item says what is missing, what the studio does locally for now, and what Phase 2 should change. + +## 1. Features that need the ServerContext (apps/server) + +`startServer(config, features: Feature[])` builds the `ServerContext` internally, so a feature factory such as `studioFeature(ctx)` cannot be put into the feature list. The studio therefore does **not** touch `apps/server/src/main.ts`. Locally it runs through `packages/feature-studio/dev/serve.ts`, which mirrors `startServer`. + +Proposal: +- `startServer(config, features: (Feature | ((ctx: ServerContext) => Feature))[])`, and main.ts adds `studioFeature`. +- An optional `stop?(): void | Promise<void>` on `Feature`, called from `stop()`. The studio uses it to stop its published-worktree watcher (an unref'd interval, so it never blocks exit). + +## 2. Learners must read studio courses from the published worktree (config / main.ts) + +Studio courses live at: +- `<dataDir>/courses/<id>`: a bare repo with no `stcourse.yaml`, so discovery never loads it; +- `<dataDir>/worktrees/<id>/published`: the published branch, which is what learners must see; +- `<dataDir>/worktrees/<id>/drafts/<author>`: drafts. They sit at depth 3, so the depth-2 discovery never loads them. + +`createCourseRepository(config.courseDirs)` does not scan `<dataDir>/worktrees`. The package exports `studioCourseDirs(config)`, which appends it. Tests and the dev entry use it. + +Proposal: main.ts calls `createCourseRepository(studioCourseDirs(config))`, or `loadConfig` always appends `<dataDir>/worktrees`. + +Related, for Phase 2: the importer writes plain folders to `<dataDir>/courses/<id>`. These coexist with the studio's bare repos. A course id that exists either way cannot be created again in the studio. Adopting an imported course into the studio (git init, move it to the bare/worktree layout) is not part of v1. + +## 3. Live preview: swap in the learner `LessonView` + +The studio preview sits behind an adapter in `@stc/feature-studio/web`: + +```ts +export interface LessonPreviewProps { + courseId: string; + markdown: string; + lesson?: { id: string; title: string; type: string; anchor?: string } | null; + resolveUrl: (rel: string) => string; // course-relative path -> URL of the draft file +} +export function setLessonPreview(c: React.ComponentType<LessonPreviewProps>): void; +``` + +v1 uses a small, safe built-in Markdown renderer. Proposal: `@stc/feature-learner` exports a `LessonView` that can render from a Markdown string plus a URL resolver, without fetching the published course. Phase 2 then calls `setLessonPreview(adapter)` in `packages/feature-studio/src/web.ts`, so drafts are previewed with the real renderer. + +## 4. Web entry points (tsconfig paths) + +Phase 1 packages have server code (`Bun.$`, `node:fs`) that must not enter the browser bundle, so the studio has a separate browser entry, `src/web.ts`. The `@stc/*` → `packages/*/src` path maps `@stc/feature-studio/web` to a directory that does not exist, and the workspace has no `node_modules/@stc` links. For now, `apps/web/src/features.ts` imports it with a relative path. + +Proposal: add `"@stc/*/web": ["packages/*/src/web.ts"]` to the root tsconfig `paths`, and switch the line to `import { studioWeb } from "@stc/feature-studio/web"`. + +## 5. Role-aware navigation (contracts/web.ts) + +`WebRoute.nav` cannot be limited by role, so the header "Studio" link shows for everyone. The page itself tells learners and anonymous users that the studio is for authors, and the API returns 401/403. + +Proposal: add `nav.roles?: Role[]`, and have the shell show the link only to matching users. This needs the learner package's session in the web shell. + +## 6. Draft API for WP-D (AI) + +Exported from `@stc/feature-studio`: + +```ts +export interface DraftWriter { + writeDraftFiles(courseId: string, authorId: string, files: { path: string; content: string }[], message: string): Promise<{ branch: string; commit: string }>; +} +export function createDraftWriter(ctx: { config: Pick<AppConfig, "dataDir"> }, opts?: { + identity?: (authorId: string) => Promise<{ name: string; email: string } | null> | { name: string; email: string } | null; +}): DraftWriter; +``` + +- It writes and commits on `draft/<authorId>` and never publishes. +- Without `identity`, commits are authored as `<authorId> <authorId@users.stcourses.invalid>`. WP-D should pass the signed-in user's name and email. +- It throws for courses not managed by the studio, and for paths that are absolute, contain `..` or touch `.git`. +- `studioFeature(ctx).draftWriter` is the same API, already bound to `ctx`. + +## 7. Authors and roles (decision to confirm) + +Studio access goes to: +- `owner` and `author` roles: all studio courses, and they can create courses; +- users with a `course_authors` row: that course only. + +Creating a course inserts the creator into `course_authors`. No new tables are added, so `migrationsDirs` is `[]`. + +## 8. External pushes straight into the bare repo (no action needed) + +The platform reloads when the published worktree's HEAD changes: an external commit in that worktree, a studio publish, or a Sync. A direct `git push` into `<dataDir>/courses/<id>` is refused by git itself ("branch is currently checked out", checked with git 2.47), so the published files cannot silently fall behind `main`. Outside tools should commit in the published worktree, or push to a configured remote that the author then Syncs. From 667f3e814bf0320209789b6700d684fdc0ff8cda Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 03:27:30 +0100 Subject: [PATCH 41/62] fix(studio): forbid ext:: remotes; type the per-dialect test table Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- packages/feature-studio/src/server/git.ts | 1 + packages/feature-studio/test/feature.test.ts | 2 +- packages/feature-studio/test/publish.test.ts | 9 +++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/feature-studio/src/server/git.ts b/packages/feature-studio/src/server/git.ts index 9ce4dc4..ee8d222 100644 --- a/packages/feature-studio/src/server/git.ts +++ b/packages/feature-studio/src/server/git.ts @@ -11,6 +11,7 @@ const BASE = [ "-c", "merge.conflictstyle=merge", "-c", "pull.rebase=false", "-c", "init.defaultBranch=main", + "-c", "protocol.ext.allow=never", // author-supplied remote URLs must never run commands ]; function env(who?: Identity): Record<string, string> { diff --git a/packages/feature-studio/test/feature.test.ts b/packages/feature-studio/test/feature.test.ts index d788c73..4636333 100644 --- a/packages/feature-studio/test/feature.test.ts +++ b/packages/feature-studio/test/feature.test.ts @@ -11,7 +11,7 @@ test("anonymous gets 401, learners 403", async () => { expect((await env.call("lea", "POST", "/api/studio/courses", { id: "x", title: "X" })).status).toBe(403); }); -const DBS = [["sqlite", "sqlite://:memory:"], ...(process.env.STC_TEST_PG_URL ? [["postgres", process.env.STC_TEST_PG_URL]] : [])] as const; +const DBS: [string, string][] = [["sqlite", "sqlite://:memory:"], ...(process.env.STC_TEST_PG_URL ? [["postgres", process.env.STC_TEST_PG_URL] as [string, string]] : [])]; test.each(DBS)("creating a course records the creator in course_authors; a per-course author without the role can edit it [%s]", async (_name, url) => { env = await setupApp([], url); diff --git a/packages/feature-studio/test/publish.test.ts b/packages/feature-studio/test/publish.test.ts index cc5c33e..a32d481 100644 --- a/packages/feature-studio/test/publish.test.ts +++ b/packages/feature-studio/test/publish.test.ts @@ -144,3 +144,12 @@ describe("sync", () => { void bob; }); }); + +test("ext:: remotes cannot run commands", async () => { + const { store } = await courseWithEdit(); + const marker = join(tempDir(), "pwned"); + await store.setRemote("c1", `ext::touch%s${marker}`); + const r = await store.sync("c1", ada); + expect(r.ok).toBe(false); + expect(await Bun.file(marker).exists()).toBe(false); +}); From 99c0e4929f21b0a25775480411ff8c278f454c96 Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 03:29:14 +0100 Subject: [PATCH 42/62] feat(learner): lesson markdown pipeline with track variants, folds, hint tiers and safe HTML Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- bun.lock | 5 - packages/feature-learner/package.json | 1 - packages/feature-learner/src/web/markdown.ts | 264 ++++++++++++++++++ .../feature-learner/test/markdown.test.ts | 126 +++++++++ 4 files changed, 390 insertions(+), 6 deletions(-) create mode 100644 packages/feature-learner/src/web/markdown.ts create mode 100644 packages/feature-learner/test/markdown.test.ts diff --git a/bun.lock b/bun.lock index fd7707b..a0be2cc 100644 --- a/bun.lock +++ b/bun.lock @@ -60,7 +60,6 @@ "dependencies": { "@better-auth/sso": "^1.7.14", "better-auth": "^1.7.14", - "dompurify": "^3.4.17", "marked": "^18.1.2", "pdfjs-dist": "^6.4.299", }, @@ -165,8 +164,6 @@ "@types/react-dom": ["@types/react-dom@19.3.0", "", { "peerDependencies": { "@types/react": "^19.3.0" } }, "sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q=="], - "@types/trusted-types": ["@types/trusted-types@2.0.7", "", {}, "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw=="], - "@xmldom/is-dom-node": ["@xmldom/is-dom-node@1.0.1", "", {}, "sha512-CJDxIgE5I0FH+ttq/Fxy6nRpxP70+e2O048EPe85J2use3XKdatVM7dDVvFNjQudd9B49NPoZ+8PG49zj4Er8Q=="], "@xmldom/xmldom": ["@xmldom/xmldom@0.9.12", "", {}, "sha512-5AXjrcMClTryPe9LgZrygpB1lj7s0S9E0+W+AHaVKAVyHanafK86iPSvG5xHVSp/jC+VH1UXu0TAEmY279xH7A=="], @@ -183,8 +180,6 @@ "defu": ["defu@6.1.7", "", {}, "sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ=="], - "dompurify": ["dompurify@3.4.17", "", { "optionalDependencies": { "@types/trusted-types": "^2.0.7" } }, "sha512-B/pSlYVHQ58kqISD/yDjuJ1yfzI34m1gF4lAod2bwSRhiN4K5Cl3H8Af7hqB2PmTL911tdEudnnXn5FnhRoz0w=="], - "escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="], "fast-xml-builder": ["fast-xml-builder@1.3.1", "", { "dependencies": { "path-expression-matcher": "^1.6.2", "xml-naming": "^0.3.0" } }, "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug=="], diff --git a/packages/feature-learner/package.json b/packages/feature-learner/package.json index 9238993..5da2328 100644 --- a/packages/feature-learner/package.json +++ b/packages/feature-learner/package.json @@ -7,7 +7,6 @@ "dependencies": { "@better-auth/sso": "^1.7.14", "better-auth": "^1.7.14", - "dompurify": "^3.4.17", "marked": "^18.1.2", "pdfjs-dist": "^6.4.299" } diff --git a/packages/feature-learner/src/web/markdown.ts b/packages/feature-learner/src/web/markdown.ts new file mode 100644 index 0000000..7f51718 --- /dev/null +++ b/packages/feature-learner/src/web/markdown.ts @@ -0,0 +1,264 @@ +import { Marked, type Token, type Tokens } from "marked"; +import { slugify } from "@stc/course-format"; +import { mediaUrl } from "../shared/types"; + +/** + * Lesson Markdown → HTML for the reader. + * + * - Raw HTML is reduced to an allowlist of attribute-free tags (`details`, `summary`, `kbd`, …); every other + * tag is shown as text, so course files cannot inject script or styles. Link and image URLs are limited to + * http(s), mailto, in-page anchors and course-relative paths. + * - Consecutive code blocks in two or more track languages (and their bold "**Scala**" labels) form a + * variant group; only the chosen track's variant is kept. A group without that track keeps every variant. + * - `<details>` become folds. Consecutive hint folds ("… — nudge", "Hint 2 — concept") become one tier group + * (`data-tier`, `data-tiers`) that the reader opens in order. + * - Relative links to another lesson file become reader links; other relative paths go to `/api/media`. + */ + +export interface RenderOptions { + courseId: string; + baseFile: string | null; + tracks?: { id: string; label: string }[]; + track?: string | null; + lessonFiles?: Record<string, string>; + /** Drop a leading heading (the page shows the lesson title itself). */ + stripFirstHeading?: boolean; +} + +const ALLOWED = new Set(["details", "summary", "br", "kbd", "sub", "sup", "em", "strong", "b", "i", "u", "mark", "small", "code", "s", "del", "ins", "abbr", "hr", "p", "span", "ul", "ol", "li", "blockquote", "pre", "dl", "dt", "dd", "table", "thead", "tbody", "tr", "th", "td"]); + +const escapeHtml = (s: string) => s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """); + +/** Keeps attribute-free allowlisted tags (and `<details open>`), escapes every other tag and comment. */ +export function sanitizeHtml(html: string): string { + return html.replace(/<!--[\s\S]*?-->|<\/?[A-Za-z][^>]*>|<(?![A-Za-z/!])/g, (tag) => { + const m = /^<(\/?)([A-Za-z][A-Za-z0-9]*)\s*(open)?\s*(\/?)>$/.exec(tag); + if (m && ALLOWED.has(m[2]!.toLowerCase()) && (!m[3] || m[2]!.toLowerCase() === "details")) return tag.toLowerCase().replace(/\s*\/>$/, ">"); + return escapeHtml(tag); + }); +} + +const LANG_ALIASES: Record<string, string[]> = { + js: ["js", "javascript", "mjs", "cjs", "jsx", "node"], + ts: ["ts", "typescript", "tsx"], + scala: ["scala", "scala3", "sc"], + fsharp: ["fsharp", "fs", "f#", "fsx"], + rust: ["rust", "rs"], + python: ["python", "py"], + go: ["go", "golang"], + java: ["java"], + kotlin: ["kotlin", "kt"], + csharp: ["csharp", "cs", "c#"], +}; + +/** Track id for a code fence language, when the course has that track. */ +export function trackForLang(lang: string | undefined, tracks: { id: string; label: string }[]): string | null { + if (!lang) return null; + const l = lang.trim().split(/\s+/)[0]!.toLowerCase(); + for (const t of tracks) { + const aliases = new Set([t.id.toLowerCase(), t.label.toLowerCase(), ...(LANG_ALIASES[t.id.toLowerCase()] ?? [])]); + for (const [k, v] of Object.entries(LANG_ALIASES)) if (v.includes(t.id.toLowerCase()) || v.includes(t.label.toLowerCase())) { aliases.add(k); for (const x of v) aliases.add(x); } + if (aliases.has(l)) return t.id; + } + return null; +} + +/** Track id for a paragraph that is only a bold track label, such as "**Scala**" or "**F#**". */ +function trackForLabel(tok: Token, tracks: { id: string; label: string }[]): string | null { + if (tok.type !== "paragraph") return null; + const p = tok as Tokens.Paragraph; + if (p.tokens?.length !== 1 || p.tokens[0]!.type !== "strong") return null; + const text = (p.tokens[0] as Tokens.Strong).text.trim().replace(/:$/, "").toLowerCase(); + for (const t of tracks) { + const label = t.label.toLowerCase(); + if (text === label || label.startsWith(text + " ") || text === t.id.toLowerCase() || trackForLang(text, [t]) === t.id) return t.id; + } + return null; +} + +function trackOf(tok: Token, tracks: { id: string; label: string }[]): string | null { + if (tok.type === "code") return trackForLang((tok as Tokens.Code).lang, tracks); + return trackForLabel(tok, tracks); +} + +/** Removes the other tracks' variants from groups of consecutive track-tagged blocks. */ +export function selectTrack(tokens: Token[], tracks: { id: string; label: string }[], track: string | null | undefined): Token[] { + if (tracks.length < 2 || !track) return tokens; + const out: Token[] = []; + let i = 0; + while (i < tokens.length) { + if (trackOf(tokens[i]!, tracks) === null) { out.push(tokens[i]!); i++; continue; } + let j = i; + const run: Token[] = []; + while (j < tokens.length && (tokens[j]!.type === "space" || trackOf(tokens[j]!, tracks) !== null)) run.push(tokens[j++]!); + while (run.length && run[run.length - 1]!.type === "space") { run.pop(); j--; } + const present = new Set(run.map((t) => trackOf(t, tracks)).filter(Boolean)); + if (present.size >= 2 && present.has(track)) out.push(...run.filter((t) => t.type === "space" || trackOf(t, tracks) === track)); + else out.push(...run); + i = j; + } + return out; +} + +const HINT = /\b(hint|nudge|stronger|near[- ]?solution|solution)\b/i; +const OPEN_DETAILS = /^\s*<details(\s+open)?\s*>\s*(?:<summary>([\s\S]*?)<\/summary>)?/i; + +interface FoldPlan { hint: boolean; group: string | null; tier: number; tiers: number } + +/** Splits html tokens so each `<details>` opening and `</details>` closing sits at the start of its own token. */ +function splitHtmlTokens(tokens: Token[]): Token[] { + const out: Token[] = []; + for (const tok of tokens) { + if (tok.type !== "html") { out.push(tok); continue; } + const parts = (tok as Tokens.HTML).text.split(/(?=<\/details>)|(?<=<\/details>)|(?=<details[\s>])/i).filter((p) => p.trim() !== ""); + if (parts.length <= 1) { out.push(tok); continue; } + for (const p of parts) out.push({ ...(tok as Tokens.HTML), raw: p, text: p } as Token); + } + return out; +} + +/** Plans folds for top-level `<details>` html tokens and groups consecutive hint folds into tiers. */ +function planFolds(tokens: Token[]): Map<Token, FoldPlan> { + const plans = new Map<Token, FoldPlan>(); + type Item = { tok: Token; hint: boolean; prefix: string | null; gapBefore: boolean }; + const items: Item[] = []; + let sinceClose = true; // non-space content since the last closing </details> + for (const tok of tokens) { + if (tok.type === "html" && OPEN_DETAILS.test((tok as Tokens.HTML).text)) { + const summary = (OPEN_DETAILS.exec((tok as Tokens.HTML).text)![2] ?? "").replace(/<[^>]+>/g, "").trim(); + const dash = summary.split(/\s+[—–-]\s+/); + const prefix = dash.length > 1 ? dash[0]!.replace(/\s*\d+$/, "").trim().toLowerCase() : null; + items.push({ tok, hint: HINT.test(summary), prefix, gapBefore: sinceClose }); + sinceClose = false; + if (/<\/details>\s*$/i.test((tok as Tokens.HTML).text)) sinceClose = false; + continue; + } + if (tok.type === "html" && /^\s*<\/details>/i.test((tok as Tokens.HTML).text)) { sinceClose = false; continue; } + if (tok.type !== "space") sinceClose = true; + } + let g = 0; + let current: Item[] = []; + const flush = () => { + const id = `h${g++}`; + current.forEach((it, k) => plans.set(it.tok, { hint: true, group: current.length > 1 ? id : null, tier: k + 1, tiers: current.length })); + current = []; + }; + for (const it of items) { + if (!it.hint) { if (current.length) flush(); plans.set(it.tok, { hint: false, group: null, tier: 0, tiers: 0 }); continue; } + const prev = current[current.length - 1]; + if (prev && (it.gapBefore || (it.prefix !== null && prev.prefix !== null && it.prefix !== prev.prefix))) flush(); + current.push(it); + } + if (current.length) flush(); + return plans; +} + +const CHEVRON = '<svg width="10" height="10" viewBox="0 0 10 10" fill="none" stroke="currentColor" stroke-width="1.6" aria-hidden="true"><path d="M3.5 2l3 3-3 3"/></svg>'; + +function dirOf(file: string | null): string { + if (!file) return ""; + const i = file.lastIndexOf("/"); + return i < 0 ? "" : file.slice(0, i + 1); +} + +/** Resolves `rel` against `base` (both course-relative), or null if it climbs above the course root. */ +export function joinPath(base: string, rel: string): string | null { + const parts = (base + rel).split("/"); + const out: string[] = []; + for (const p of parts) { + if (p === "" || p === ".") continue; + if (p === "..") { if (!out.length) return null; out.pop(); } else out.push(p); + } + return out.join("/"); +} + +function safeDecode(s: string): string { + try { return decodeURI(s); } catch { return s; } +} + +export function resolveHref(href: string, opts: RenderOptions, kind: "link" | "image"): { url: string; external: boolean } | null { + const h = href.trim(); + if (/^https?:\/\//i.test(h)) return { url: h, external: true }; + if (kind === "link" && /^mailto:/i.test(h)) return { url: h, external: true }; + if (kind === "image" && /^data:image\/(png|jpe?g|gif|webp);/i.test(h)) return { url: h, external: false }; + if (h.startsWith("#")) return kind === "link" ? { url: h, external: false } : null; + if (/^[a-z][a-z0-9+.-]*:/i.test(h) || h.startsWith("//")) return null; // javascript:, file:, protocol-relative … + const [pathPart, hash] = h.split("#", 2) as [string, string | undefined]; + const rel = h.startsWith("/") ? joinPath("", safeDecode(pathPart)) : joinPath(dirOf(opts.baseFile), safeDecode(pathPart)); + if (rel === null) return null; + const lessonId = kind === "link" ? opts.lessonFiles?.[rel] : undefined; + if (lessonId) return { url: `/c/${encodeURIComponent(opts.courseId)}/${encodeURIComponent(lessonId)}${hash ? `#${hash}` : ""}`, external: false }; + return { url: mediaUrl(opts.courseId, rel) + (hash ? `#${hash}` : ""), external: false }; +} + +export function renderLessonHtml(md: string, opts: RenderOptions): string { + const tracks = opts.tracks ?? []; + const marked = new Marked({ gfm: true }); + let tokens: Token[] = marked.lexer(md); + if (opts.stripFirstHeading) { + const first = tokens.findIndex((t) => t.type !== "space"); + if (first >= 0 && tokens[first]!.type === "heading") tokens = [...tokens.slice(0, first), ...tokens.slice(first + 1)]; + } + tokens = splitHtmlTokens(selectTrack(tokens, tracks, opts.track ?? tracks[0]?.id)); + const folds = planFolds(tokens); + const slugs = new Map<string, number>(); + + const renderer = { + heading(this: { parser: { parseInline(t: Token[]): string } }, { tokens: inl, depth, text }: Tokens.Heading) { + const base = slugify(text); + const n = slugs.get(base) ?? 0; + slugs.set(base, n + 1); + const id = n === 0 ? base : `${base}-${n}`; + return `<h${depth} id="${escapeHtml(id)}">${this.parser.parseInline(inl)}</h${depth}>\n`; + }, + code({ text, lang }: Tokens.Code) { + const l = (lang ?? "").trim().split(/\s+/)[0] ?? ""; + const t = trackForLang(l, tracks); + return `<pre class="code"${l ? ` data-lang="${escapeHtml(l)}"` : ""}${t ? ` data-t="${escapeHtml(t)}"` : ""}><code>${escapeHtml(text)}</code></pre>\n`; + }, + link(this: { parser: { parseInline(t: Token[]): string } }, { href, title, tokens: inl }: Tokens.Link) { + const inner = this.parser.parseInline(inl); + const r = resolveHref(href, opts, "link"); + if (!r) return inner; + const t = title ? ` title="${escapeHtml(title)}"` : ""; + return `<a href="${escapeHtml(r.url)}"${t}${r.external ? ' target="_blank" rel="noopener noreferrer"' : ""}>${inner}</a>`; + }, + image({ href, title, text }: Tokens.Image) { + const r = resolveHref(href, opts, "image"); + if (!r) return escapeHtml(text); + return `<img src="${escapeHtml(r.url)}" alt="${escapeHtml(text)}"${title ? ` title="${escapeHtml(title)}"` : ""} loading="lazy">`; + }, + html(token: Tokens.HTML | Tokens.Tag) { + const plan = folds.get(token as Token); + let text = token.text; + if (plan) { + const m = OPEN_DETAILS.exec(text)!; + const summary = sanitizeHtml(m[2] ?? (plan.hint ? "Hint" : "Details")); + const attrs = plan.hint + ? ` data-hint="1"${plan.group ? ` data-group="${plan.group}" data-tier="${plan.tier}" data-tiers="${plan.tiers}"` : ""}` + : ""; + const tier = plan.group ? `<span class="tier">${plan.tier} of ${plan.tiers}</span>` : ""; + const head = `<details class="fold"${attrs}><summary>${CHEVRON}<span class="sum">${summary}</span>${tier}</summary><div class="in">`; + text = head + sanitizeHtml(text.slice(m[0].length)).replace(/<\/details>/gi, "</div></details>"); + return text; + } + if (/^\s*<\/details>/i.test(text)) return sanitizeHtml(text).replace(/<\/details>/i, "</div></details>"); + return sanitizeHtml(text); + }, + }; + marked.use({ renderer: renderer as never }); + return marked.parser(tokens) as string; +} + +const cap = (s: string) => (s ? s[0]!.toUpperCase() + s.slice(1) : s); + +/** Exercise hints (already matched by the server) as one tier group of folds. */ +export function renderHintsHtml(hints: { label: string; body: string }[], opts: RenderOptions): string { + const n = hints.length; + return hints.map((h, i) => { + const label = h.label ? cap(h.label) : `Hint ${i + 1}`; + const tier = n > 1 ? `<span class="tier">${i + 1} of ${n}</span>` : ""; + const attrs = n > 1 ? ` data-hint="1" data-group="ex" data-tier="${i + 1}" data-tiers="${n}"` : ' data-hint="1"'; + return `<details class="fold"${attrs}><summary>${CHEVRON}<span class="sum">${escapeHtml(label)}</span>${tier}</summary><div class="in">${renderLessonHtml(h.body, opts)}</div></details>`; + }).join("\n"); +} diff --git a/packages/feature-learner/test/markdown.test.ts b/packages/feature-learner/test/markdown.test.ts new file mode 100644 index 0000000..604b444 --- /dev/null +++ b/packages/feature-learner/test/markdown.test.ts @@ -0,0 +1,126 @@ +import { expect, test } from "bun:test"; +import { renderHintsHtml, renderLessonHtml, resolveHref, sanitizeHtml } from "../src/web/markdown"; + +const tracks = [{ id: "js", label: "JavaScript" }, { id: "scala", label: "Scala 3" }, { id: "fsharp", label: "F#" }]; +const opts = { courseId: "pure-snake", baseFile: "course/01-purity.md", tracks, lessonFiles: { "course/02-data.md": "m02-text" } }; + +const variants = `Intro + +**JavaScript** +\`\`\`js +export const range = (n) => ... +\`\`\` +**Scala** +\`\`\`scala +def range(n: Int): List[Int] +\`\`\` +**F#** +\`\`\`fsharp +let range (n: int) : int list +\`\`\` + +Tests: after. +`; + +test("track switcher keeps only the chosen variant and its label", () => { + const scala = renderLessonHtml(variants, { ...opts, track: "scala" }); + expect(scala).toContain("def range"); + expect(scala).not.toContain("export const range"); + expect(scala).not.toContain("let range"); + expect(scala).toContain("<strong>Scala</strong>"); + expect(scala).not.toContain("<strong>JavaScript</strong>"); + expect(scala).toContain("Tests: after."); + const fs = renderLessonHtml(variants, { ...opts, track: "fsharp" }); + expect(fs).toContain("let range"); + expect(fs).not.toContain("def range"); +}); + +test("a lone code block in one track stays visible for other tracks", () => { + const md = "Text\n\n```js\nexport const pipe = 1\n```\n\nMore"; + expect(renderLessonHtml(md, { ...opts, track: "scala" })).toContain("export const pipe"); +}); + +test("a variant group without the chosen track keeps every variant", () => { + const md = "```js\nA\n```\n\n```scala\nB\n```\n"; + const html = renderLessonHtml(md, { ...opts, track: "fsharp" }); + expect(html).toContain("A"); + expect(html).toContain("B"); +}); + +test("headings get GitHub ids and the first heading can be dropped", () => { + const html = renderLessonHtml("# Module 1\n\n## Why this module\n\n### Exercise 1.1 — `range`\n\n## Why this module\n", { ...opts, stripFirstHeading: true }); + expect(html).not.toContain("Module 1"); + expect(html).toContain('<h2 id="why-this-module">'); + expect(html).toContain('<h3 id="exercise-11--range">'); + expect(html).toContain('<h2 id="why-this-module-1">'); +}); + +test("details become folds; consecutive hints become ordered tiers", () => { + const md = `<details><summary>Think about it first, then open the explanation</summary> + +All segments move **at once**. + +</details> + +## Hints + +<details><summary>Exercise 1.1 (range) — nudge</summary> + +N1 + +</details> + +<details><summary>Exercise 1.1 (range) — stronger hint</summary> + +S1 + +</details> + +<details><summary>Exercise 1.2 (dropLast) — nudge</summary> + +N2 + +</details> +`; + const html = renderLessonHtml(md, opts); + expect(html).toContain('<details class="fold"><summary>'); + expect(html).toContain("<strong>at once</strong>"); + expect((html.match(/data-tier="1" data-tiers="2"/g) ?? []).length).toBe(1); + expect(html).toContain('data-tier="2" data-tiers="2"'); + expect(html).toContain("1 of 2"); + // the 1.2 hint starts a new group of its own + expect(html).toMatch(/<details class="fold" data-hint="1"><summary>[\s\S]*Exercise 1\.2 \(dropLast\)/); + expect((html.match(/<\/div><\/details>/g) ?? []).length).toBe(4); +}); + +test("systemic-design hint numbering forms one group", () => { + const md = "<details><summary>Hint 1 — nudge</summary>\n\na\n\n</details>\n<details><summary>Hint 2 — concept</summary>\n\nb\n\n</details>\n<details><summary>Hint 3 — pseudo-code</summary>\n\nc\n\n</details>\n"; + const html = renderLessonHtml(md, opts); + expect(html).toContain('data-tier="3" data-tiers="3"'); +}); + +test("raw HTML is reduced to safe tags", () => { + expect(sanitizeHtml('<script>alert(1)</script><kbd>x</kbd><img src=x onerror=alert(1)><details open>')).toBe("<script>alert(1)</script><kbd>x</kbd><img src=x onerror=alert(1)><details open>"); + const html = renderLessonHtml('<div onclick="x()">hi</div>\n\n[x](javascript:alert(1)) <span style="color:red">s</span>', opts); + expect(html).not.toContain("<div"); + expect(html).not.toContain("javascript:"); + expect(html).not.toContain('<span style'); +}); + +test("links and images resolve inside the course", () => { + expect(resolveHref("02-data.md#state", opts, "link")).toEqual({ url: "/c/pure-snake/m02-text#state", external: false }); + expect(resolveHref("../fixtures/board 1.json", opts, "link")).toEqual({ url: "/api/media/pure-snake/fixtures/board%201.json", external: false }); + expect(resolveHref("../../etc/passwd", opts, "link")).toBeNull(); + expect(resolveHref("https://example.com", opts, "link")).toEqual({ url: "https://example.com", external: true }); + expect(resolveHref("img/a.png", opts, "image")).toEqual({ url: "/api/media/pure-snake/course/img/a.png", external: false }); + const html = renderLessonHtml("![board](img/board.png) and [ext](https://x.org)", opts); + expect(html).toContain('<img src="/api/media/pure-snake/course/img/board.png" alt="board" loading="lazy">'); + expect(html).toContain('target="_blank" rel="noopener noreferrer"'); +}); + +test("exercise hints render as one tier group", () => { + const html = renderHintsHtml([{ label: "nudge", body: "Which **operator**?" }, { label: "near-solution", body: "`a + b`" }], { courseId: "sample", baseFile: null }); + expect(html).toContain("Nudge"); + expect(html).toContain('data-tier="2" data-tiers="2"'); + expect(html).toContain("<strong>operator</strong>"); +}); From 08d3e3f642d42ce831690e7cd7a9f673f29d7171 Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 03:30:22 +0100 Subject: [PATCH 43/62] feat(ai): assistant panel, AI settings and studio generator UI Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- apps/web/src/features.ts | 1 + packages/feature-ai/package.json | 5 +- packages/feature-ai/scripts/dev.ts | 44 ++++ packages/feature-ai/src/feature.ts | 29 ++- packages/feature-ai/src/index.ts | 1 + packages/feature-ai/src/web/AccountAiPage.tsx | 84 ++++++++ packages/feature-ai/src/web/AdminAiPage.tsx | 164 ++++++++++++++ packages/feature-ai/src/web/AssistantPage.tsx | 204 ++++++++++++++++++ packages/feature-ai/src/web/GeneratePage.tsx | 195 +++++++++++++++++ packages/feature-ai/src/web/ai.css | 77 +++++++ packages/feature-ai/src/web/api.ts | 51 +++++ packages/feature-ai/src/web/components.tsx | 77 +++++++ packages/feature-ai/src/web/index.ts | 11 + packages/feature-ai/test/feature.test.ts | 1 + packages/feature-ai/test/web.test.tsx | 36 ++++ 15 files changed, 973 insertions(+), 7 deletions(-) create mode 100644 packages/feature-ai/scripts/dev.ts create mode 100644 packages/feature-ai/src/web/AccountAiPage.tsx create mode 100644 packages/feature-ai/src/web/AdminAiPage.tsx create mode 100644 packages/feature-ai/src/web/AssistantPage.tsx create mode 100644 packages/feature-ai/src/web/GeneratePage.tsx create mode 100644 packages/feature-ai/src/web/ai.css create mode 100644 packages/feature-ai/src/web/api.ts create mode 100644 packages/feature-ai/src/web/components.tsx create mode 100644 packages/feature-ai/src/web/index.ts create mode 100644 packages/feature-ai/test/web.test.tsx diff --git a/apps/web/src/features.ts b/apps/web/src/features.ts index 8b077c9..4a78195 100644 --- a/apps/web/src/features.ts +++ b/apps/web/src/features.ts @@ -2,3 +2,4 @@ import type { WebFeature } from "@stc/contracts"; // Phase 1 packages append one import line each and push into this list. export const webFeatures: WebFeature[] = []; +import { aiWeb } from "../../../packages/feature-ai/src/web"; webFeatures.push(aiWeb); diff --git a/packages/feature-ai/package.json b/packages/feature-ai/package.json index 0a7eb1e..639878d 100644 --- a/packages/feature-ai/package.json +++ b/packages/feature-ai/package.json @@ -4,5 +4,8 @@ "private": true, "type": "module", "license": "AGPL-3.0-only", - "main": "src/index.ts" + "exports": { + ".": "./src/index.ts", + "./web": "./src/web/index.ts" + } } diff --git a/packages/feature-ai/scripts/dev.ts b/packages/feature-ai/scripts/dev.ts new file mode 100644 index 0000000..4720fd5 --- /dev/null +++ b/packages/feature-ai/scripts/dev.ts @@ -0,0 +1,44 @@ +// Local dev server for the AI package until Phase 2 wires feature factories into apps/server. +// STC_PORT=3464 STC_DATA_DIR=/tmp/stc-ai-data STC_COURSE_DIRS=./fixtures/courses:./overlays bun packages/feature-ai/scripts/dev.ts +// STC_AI_DEV_USER=owner|learner|none picks the signed-in user (dev only). STC_AI_DEV_FAKE=1 uses a canned provider. +import type { AiProvider, AuthService, ServerContext } from "@stc/contracts"; +import { createApp, createJobQueue, loadConfig } from "@stc/core"; +import { createCourseRepository } from "@stc/course-format"; +import { connect, migrate } from "@stc/db"; +import indexHtml from "../../../apps/web/index.html"; +import { aiFeature } from "../src/feature"; + +const config = loadConfig(); +const who = process.env.STC_AI_DEV_USER ?? "owner"; +const auth: AuthService = { + userFromRequest: async () => + who === "none" ? null : { id: `dev-${who}`, email: `${who}@localhost`, name: who, roles: who === "owner" ? ["owner"] : ["learner"], groups: [] }, +}; + +const fake: AiProvider = { + id: "fake", + async *chat(messages) { + const sys = messages[0]?.content ?? ""; + const mentor = sys.includes("## Course mentor rules"); + const answer = mentor + ? "Before I explain: what does the failing test give as input, and what did you expect `add` to return?\n\nThe lesson's nudge asks **which operator combines two numbers**. Try it, run the tests, and tell me what happens. [lessons/04-exercise.md · Adding numbers]" + : "Here is the whole answer.\n\n```js\nexport const add = (a, b) => a + b\n```\n\nIt returns the sum of its arguments. [lessons/04-exercise.md · Adding numbers]"; + for (const part of answer.match(/.{1,12}/gs) ?? []) { await Bun.sleep(5); yield { text: part }; } + }, +}; + +const db = await connect(config.databaseUrl); +await migrate(db); +const courses = await createCourseRepository(config.courseDirs); +const ctx = { config, db, courses, auth } as ServerContext; +const feature = aiFeature(ctx, process.env.STC_AI_DEV_FAKE ? { providerFactory: () => fake } : {}); +const jobs = createJobQueue(db, feature.jobs ?? {}); +ctx.jobs = jobs; +const app = createApp(ctx, [feature]); +jobs.start(); +const server = Bun.serve({ + port: config.port, + routes: { ...app.routes, "/api/*": (req) => app.fetch(req), "/*": indexHtml }, + fetch: (req) => app.fetch(req), +}); +console.log(`STCourses (AI dev) on ${server.url} as ${who}`); diff --git a/packages/feature-ai/src/feature.ts b/packages/feature-ai/src/feature.ts index 02f59e2..f74aa04 100644 --- a/packages/feature-ai/src/feature.ts +++ b/packages/feature-ai/src/feature.ts @@ -171,13 +171,25 @@ export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feat const rows = await ctx.db.sql`SELECT id, user_id, course_id, title, created_at FROM ai_conversations WHERE id = ${id} AND user_id = ${userId}`; return rows[0] as { id: string; user_id: string; course_id: string; title: string | null; created_at: string } | undefined; }; - const addMessage = async (conversationId: string, role: "user" | "assistant", content: string) => { + // Roles: "user", "assistant", and "sources" (JSON citations for the assistant message before it). + const addMessage = async (conversationId: string, role: "user" | "assistant" | "sources", content: string) => { await ctx.db.sql`INSERT INTO ai_messages (id, conversation_id, role, content, created_at) VALUES (${Bun.randomUUIDv7()}, ${conversationId}, ${role}, ${content}, ${new Date().toISOString()})`; }; const messagesOf = async (conversationId: string) => (await ctx.db.sql`SELECT id, role, content, created_at FROM ai_messages WHERE conversation_id = ${conversationId}`) .map((r: { id: string; role: string; content: string; created_at: string }) => ({ id: r.id, role: r.role, content: r.content, createdAt: r.created_at })) - .sort((a: { id: string }, b: { id: string }) => (a.id < b.id ? -1 : 1)) as { id: string; role: "user" | "assistant"; content: string; createdAt: string }[]; + .sort((a: { id: string }, b: { id: string }) => (a.id < b.id ? -1 : 1)) as { id: string; role: "user" | "assistant" | "sources"; content: string; createdAt: string }[]; + /** Chat turns with each assistant message's stored citations attached. */ + const turnsOf = async (conversationId: string) => { + const out: { id: string; role: "user" | "assistant"; content: string; createdAt: string; citations?: unknown[] }[] = []; + for (const m of await messagesOf(conversationId)) { + if (m.role === "sources") { + const last = out[out.length - 1]; + if (last?.role === "assistant") { try { last.citations = JSON.parse(m.content); } catch {} } + } else out.push({ ...m, role: m.role }); + } + return out; + }; const signedIn = (rc: RequestContext) => (rc.user ? null : err(401, "Sign in to use the assistant.")); const ownerOnly = (rc: RequestContext) => (!rc.user ? err(401, "Sign in first.") : rc.user.roles.includes("owner") ? null : err(403, "Only the owner can change AI settings.")); @@ -205,7 +217,7 @@ export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feat conversationId = Bun.randomUUIDv7(); await ctx.db.sql`INSERT INTO ai_conversations (id, user_id, course_id, title, created_at) VALUES (${conversationId}, ${user.id}, ${course.id}, ${message.slice(0, 80)}, ${new Date().toISOString()})`; } - const history = (await messagesOf(conversationId)).slice(-HISTORY_TURNS); + const history = (await turnsOf(conversationId)).slice(-HISTORY_TURNS); await addMessage(conversationId, "user", message); const lessonId = str(b?.lessonId); @@ -235,13 +247,18 @@ export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feat { role: "user", content: message }, ]; let answer = ""; + let cited: unknown[] = []; for await (const ev of runAssistant({ provider, messages, tools, signal: req.signal })) { if (ev.type === "done") { answer = ev.text; - send({ type: "done", text: ev.text, citations: citationsFor([...sources, ...tools.sources]) }); + cited = citationsFor([...sources, ...tools.sources]); + send({ type: "done", text: ev.text, citations: cited }); } else send(ev); } - if (answer) await addMessage(convId, "assistant", answer); + if (answer) { + await addMessage(convId, "assistant", answer); + await addMessage(convId, "sources", JSON.stringify(cited)); + } } catch (e) { send({ type: "error", message: (e as Error).message }); } finally { @@ -360,7 +377,7 @@ export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feat if (deny) return deny; const conv = await conversationOf(rc.params.id ?? "", rc.user!.id); if (!conv) return err(404, "Conversation not found."); - return json({ id: conv.id, courseId: conv.course_id, title: conv.title, messages: await messagesOf(conv.id) }); + return json({ id: conv.id, courseId: conv.course_id, title: conv.title, messages: await turnsOf(conv.id) }); }, DELETE: async (_req, rc) => { const deny = signedIn(rc); diff --git a/packages/feature-ai/src/index.ts b/packages/feature-ai/src/index.ts index 62f5df7..405c90e 100644 --- a/packages/feature-ai/src/index.ts +++ b/packages/feature-ai/src/index.ts @@ -14,3 +14,4 @@ export * from "./generator/draft-sink"; export * from "./generator/generate"; export * from "./generator/templates"; export { aiFeature, type AiFeatureOptions } from "./feature"; +export { aiWeb } from "./web"; diff --git a/packages/feature-ai/src/web/AccountAiPage.tsx b/packages/feature-ai/src/web/AccountAiPage.tsx new file mode 100644 index 0000000..0b0d41f --- /dev/null +++ b/packages/feature-ai/src/web/AccountAiPage.tsx @@ -0,0 +1,84 @@ +import { useEffect, useState } from "react"; +import { Pill } from "@stc/ui"; +import { api, ApiError } from "./api"; +import { Field, PageHead } from "./components"; + +interface Account { + byok: { kind: "anthropic" | "openai"; model?: string; baseUrl?: string; hasApiKey: boolean } | null; + serverProvider: { kind: string; model: string | null } | null; +} + +export default function AccountAiPage() { + const [acct, setAcct] = useState<Account | null>(null); + const [denied, setDenied] = useState<string | null>(null); + const [kind, setKind] = useState<"anthropic" | "openai">("anthropic"); + const [apiKey, setApiKey] = useState(""); + const [model, setModel] = useState(""); + const [baseUrl, setBaseUrl] = useState(""); + const [msg, setMsg] = useState<{ ok: boolean; text: string } | null>(null); + + const load = () => api<Account>("/api/ai/account").then((a) => { + setAcct(a); + if (a.byok) { setKind(a.byok.kind); setModel(a.byok.model ?? ""); setBaseUrl(a.byok.baseUrl ?? ""); } + }, (e: ApiError) => setDenied(e.status === 401 ? "Sign in to manage your own AI key." : e.message)); + useEffect(() => { load(); }, []); + + const save = async () => { + setMsg(null); + try { + await api("/api/ai/account/key", { method: "PUT", body: { kind, apiKey, model, baseUrl } }); + setApiKey(""); + await load(); + setMsg({ ok: true, text: "Saved. Your questions now use your own key." }); + } catch (e) { setMsg({ ok: false, text: (e as Error).message }); } + }; + const remove = async () => { + await api("/api/ai/account/key", { method: "DELETE" }); + setApiKey(""); + await load(); + setMsg({ ok: true, text: "Your key is removed." }); + }; + + if (denied) return <div className="ai-page"><PageHead chap="Account" title="Your AI key" /><p className="ai-note">{denied}</p></div>; + if (!acct) return <div className="ai-page"><PageHead chap="Account" title="Your AI key" /></div>; + + return ( + <div className="ai-page"> + <PageHead chap="Account" title="Your AI key"> + <p> + Bring your own key to use the course assistant and the studio generator with your own account. It is used only for your requests, stored encrypted, and never shown again.{" "} + {acct.serverProvider ? "This server also has a shared provider; your key takes priority while it is saved." : "This server has no shared provider, so AI is off for you until you add a key."} + </p> + </PageHead> + + {acct.byok ? <p className="ai-banner" role="status">A {acct.byok.kind === "anthropic" ? "Anthropic" : "OpenAI-compatible"} key is saved{acct.byok.model ? ` (model ${acct.byok.model})` : ""}.</p> : null} + + <form onSubmit={(e) => { e.preventDefault(); save(); }}> + <div className="ai-grid"> + <Field label="Provider"> + <select className="ai-select" value={kind} onChange={(e) => setKind(e.target.value as "anthropic" | "openai")}> + <option value="anthropic">Anthropic</option> + <option value="openai">OpenAI-compatible</option> + </select> + </Field> + <Field label="Model" hint={kind === "anthropic" ? "Leave empty for claude-opus-5-5." : undefined}> + <input className="ai-input" value={model} onChange={(e) => setModel(e.target.value)} placeholder={kind === "anthropic" ? "claude-opus-5-5" : "model name"} /> + </Field> + </div> + {kind === "openai" ? ( + <Field label="Base URL" hint="The address that ends before /chat/completions."> + <input className="ai-input ai-mono" value={baseUrl} onChange={(e) => setBaseUrl(e.target.value)} placeholder="https://api.openai.com/v1" /> + </Field> + ) : null} + <Field label="API key" hint={acct.byok ? "Paste a new key to replace the saved one." : undefined}> + <input className="ai-input ai-mono" type="password" autoComplete="off" value={apiKey} onChange={(e) => setApiKey(e.target.value)} placeholder="paste key" /> + </Field> + <div className="ai-footer"> + <Pill variant="primary" type="submit" disabled={!apiKey.trim()}>Save key</Pill> + {acct.byok ? <Pill onClick={remove}>Remove my key</Pill> : null} + {msg ? <span className={msg.ok ? "ai-hint ai-ok" : "ai-error"} role="status" style={{ margin: 0 }}>{msg.text}</span> : null} + </div> + </form> + </div> + ); +} diff --git a/packages/feature-ai/src/web/AdminAiPage.tsx b/packages/feature-ai/src/web/AdminAiPage.tsx new file mode 100644 index 0000000..0c3d0ca --- /dev/null +++ b/packages/feature-ai/src/web/AdminAiPage.tsx @@ -0,0 +1,164 @@ +import { useEffect, useState } from "react"; +import { Pill } from "@stc/ui"; +import { api, ApiError } from "./api"; +import { Field, PageHead } from "./components"; + +type Kind = "" | "anthropic" | "openai"; +interface ProviderOut { kind: "anthropic" | "openai"; model?: string; baseUrl?: string; effort?: string; hasApiKey: boolean } +interface McpServer { id: string; name: string; transport: "stdio" | "http"; command?: string; args?: string[]; url?: string; courseIds?: string[] } +interface Config { provider: ProviderOut | null; mcpServers: McpServer[] } + +type ServerDraft = { id: string; name: string; transport: "stdio" | "http"; command: string; args: string; url: string; courses: string; test?: string }; +const toDraft = (s: McpServer): ServerDraft => ({ id: s.id, name: s.name, transport: s.transport, command: s.command ?? "", args: (s.args ?? []).join(" "), url: s.url ?? "", courses: (s.courseIds ?? []).join(", ") }); +const fromDraft = (d: ServerDraft): McpServer => ({ + id: d.id, name: d.name || d.id, transport: d.transport, + ...(d.transport === "stdio" ? { command: d.command, args: d.args.trim() ? d.args.trim().split(/\s+/) : [] } : { url: d.url }), + courseIds: d.courses.split(",").map((s) => s.trim()).filter(Boolean), +}); + +export default function AdminAiPage() { + const [loaded, setLoaded] = useState<Config | null>(null); + const [denied, setDenied] = useState<string | null>(null); + const [kind, setKind] = useState<Kind>(""); + const [model, setModel] = useState(""); + const [baseUrl, setBaseUrl] = useState(""); + const [effort, setEffort] = useState(""); + const [apiKey, setApiKey] = useState(""); + const [clearKey, setClearKey] = useState(false); + const [servers, setServers] = useState<ServerDraft[]>([]); + const [msg, setMsg] = useState<{ ok: boolean; text: string } | null>(null); + + const apply = (c: Config) => { + setLoaded(c); + setKind(c.provider?.kind ?? ""); + setModel(c.provider?.model ?? ""); + setBaseUrl(c.provider?.baseUrl ?? ""); + setEffort(c.provider?.effort ?? ""); + setApiKey(""); + setClearKey(false); + setServers(c.mcpServers.map(toDraft)); + }; + useEffect(() => { + api<Config>("/api/ai/admin/config").then(apply, (e: ApiError) => setDenied(e.status === 401 ? "Sign in as the owner to change AI settings." : e.message)); + }, []); + + const save = async () => { + setMsg(null); + try { + const provider = kind ? { kind, model, baseUrl, effort, apiKey: apiKey || undefined, clearApiKey: clearKey } : null; + apply(await api<Config>("/api/ai/admin/config", { method: "PUT", body: { provider, mcpServers: servers.map(fromDraft) } })); + setMsg({ ok: true, text: kind ? "Saved. The assistant is available to learners." : "Saved. AI is off for everyone without their own key." }); + } catch (e) { + setMsg({ ok: false, text: (e as Error).message }); + } + }; + + const setServer = (i: number, patch: Partial<ServerDraft>) => setServers((s) => s.map((x, j) => (j === i ? { ...x, ...patch } : x))); + const testServer = async (i: number) => { + setServer(i, { test: "Connecting…" }); + try { + const r = await api<{ ok: boolean; tools?: { name: string }[]; error?: string }>("/api/ai/admin/mcp/test", { method: "POST", body: fromDraft(servers[i]!) }); + setServer(i, { test: r.ok ? `Connected. Tools: ${r.tools!.map((t) => t.name).join(", ") || "none"}` : `Failed: ${r.error}` }); + } catch (e) { + setServer(i, { test: `Failed: ${(e as Error).message}` }); + } + }; + + if (denied) return <div className="ai-page"><PageHead title="AI settings" /><p className="ai-note">{denied}</p></div>; + if (!loaded) return <div className="ai-page"><PageHead title="AI settings" /></div>; + + return ( + <div className="ai-page"> + <PageHead title="AI settings"> + <p>AI is off until you choose a provider here. Learners and authors can also add their own key, which is used only for their own requests. Keys are encrypted on the server and never shown again.</p> + </PageHead> + + <form onSubmit={(e) => { e.preventDefault(); save(); }}> + <section className="ai-section"> + <h2>Server provider</h2> + <span className="ai-hint">Used for everyone who has not added their own key.</span> + <div className="ai-grid"> + <Field label="Provider"> + <select className="ai-select" value={kind} onChange={(e) => setKind(e.target.value as Kind)}> + <option value="">None (AI off)</option> + <option value="anthropic">Anthropic</option> + <option value="openai">OpenAI-compatible (OpenAI, Ollama, vLLM …)</option> + </select> + </Field> + {kind ? ( + <Field label="Model" hint={kind === "anthropic" ? "Leave empty for claude-opus-5-5." : "For example llama3.1 on Ollama."}> + <input className="ai-input" value={model} onChange={(e) => setModel(e.target.value)} placeholder={kind === "anthropic" ? "claude-opus-5-5" : "model name"} /> + </Field> + ) : null} + </div> + {kind ? ( + <> + <Field label={kind === "anthropic" ? "Base URL (optional)" : "Base URL"} hint={kind === "openai" ? "The address that ends before /chat/completions, for example http://ollama:11434/v1." : "Leave empty for api.anthropic.com."}> + <input className="ai-input ai-mono" value={baseUrl} onChange={(e) => setBaseUrl(e.target.value)} placeholder={kind === "openai" ? "http://localhost:11434/v1" : "https://api.anthropic.com"} /> + </Field> + <div className="ai-grid"> + <Field label="API key" hint={loaded.provider?.hasApiKey ? "A key is saved. Leave empty to keep it." : kind === "openai" ? "Optional for a local Ollama." : undefined}> + <input className="ai-input ai-mono" type="password" autoComplete="off" value={apiKey} onChange={(e) => setApiKey(e.target.value)} placeholder={loaded.provider?.hasApiKey ? "••••••••" : "paste key"} /> + </Field> + {kind === "anthropic" ? ( + <Field label="Effort" hint="How hard the model thinks. Empty uses the model's default."> + <select className="ai-select" value={effort} onChange={(e) => setEffort(e.target.value)}> + <option value="">Default</option> + <option value="low">Low</option> + <option value="medium">Medium</option> + <option value="high">High</option> + </select> + </Field> + ) : null} + </div> + {loaded.provider?.hasApiKey ? ( + <label className="ai-check"><input type="checkbox" checked={clearKey} onChange={(e) => setClearKey(e.target.checked)} /> Remove the saved key</label> + ) : null} + </> + ) : null} + </section> + + <section className="ai-section"> + <h2>MCP servers</h2> + <span className="ai-hint">Extra tools for the assistant. Leave "Courses" empty to offer a server in every course.</span> + {servers.map((s, i) => ( + <div className="ai-server" key={i}> + <div className="ai-grid"> + <Field label="Id"><input className="ai-input ai-mono" value={s.id} onChange={(e) => setServer(i, { id: e.target.value })} /></Field> + <Field label="Name"><input className="ai-input" value={s.name} onChange={(e) => setServer(i, { name: e.target.value })} /></Field> + <Field label="Transport"> + <select className="ai-select" value={s.transport} onChange={(e) => setServer(i, { transport: e.target.value as "stdio" | "http" })}> + <option value="http">Streamable HTTP</option> + <option value="stdio">Local command (stdio)</option> + </select> + </Field> + </div> + {s.transport === "http" ? ( + <Field label="URL"><input className="ai-input ai-mono" value={s.url} onChange={(e) => setServer(i, { url: e.target.value })} placeholder="https://mcp.example.com/mcp" /></Field> + ) : ( + <div className="ai-grid"> + <Field label="Command"><input className="ai-input ai-mono" value={s.command} onChange={(e) => setServer(i, { command: e.target.value })} placeholder="bunx" /></Field> + <Field label="Arguments"><input className="ai-input ai-mono" value={s.args} onChange={(e) => setServer(i, { args: e.target.value })} placeholder="some-mcp-server --flag" /></Field> + </div> + )} + <Field label="Courses" hint="Course ids, separated by commas."><input className="ai-input ai-mono" value={s.courses} onChange={(e) => setServer(i, { courses: e.target.value })} /></Field> + <div className="ai-footer" style={{ margin: "0 0 10px" }}> + <Pill onClick={() => testServer(i)}>Test connection</Pill> + <Pill variant="ghost" onClick={() => setServers((x) => x.filter((_, j) => j !== i))}>Remove</Pill> + {s.test ? <span className="ai-hint">{s.test}</span> : null} + </div> + </div> + ))} + <div className="ai-footer"> + <Pill onClick={() => setServers((x) => [...x, { id: `server-${x.length + 1}`, name: "", transport: "http", command: "", args: "", url: "", courses: "" }])}>Add MCP server</Pill> + </div> + </section> + + <div className="ai-footer"> + <Pill variant="primary" type="submit">Save AI settings</Pill> + {msg ? <span className={msg.ok ? "ai-hint ai-ok" : "ai-error"} role="status" style={{ margin: 0 }}>{msg.text}</span> : null} + </div> + </form> + </div> + ); +} diff --git a/packages/feature-ai/src/web/AssistantPage.tsx b/packages/feature-ai/src/web/AssistantPage.tsx new file mode 100644 index 0000000..3d180a8 --- /dev/null +++ b/packages/feature-ai/src/web/AssistantPage.tsx @@ -0,0 +1,204 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { Link, useParams } from "react-router-dom"; +import { Pill, SegmentedToggle } from "@stc/ui"; +import { api, readNdjson, type ChatEvent, type Citation, type Status } from "./api"; +import { Markdown, MentorBanner, PageHead } from "./components"; + +type Turn = { role: "user" | "assistant"; text: string; citations?: Citation[]; tools?: string[]; error?: string; notices?: string[] }; +type ConvSummary = { id: string; title: string | null; createdAt: string }; + +const TOOL_LABEL: Record<string, string> = { search_course: "searched the course", read_lesson: "read a lesson", read_file: "read a file", list_notes: "read your notes" }; +const toolLabel = (n: string) => TOOL_LABEL[n] ?? `used ${n.replace(/^mcp_[^_]+__/, "")}`; + +function Sources({ courseId, citations }: { courseId: string; citations: Citation[] }) { + // One row per file; a file shared by several lessons links to the first one. + const rows = [...new Map(citations.map((c) => [c.path, c] as const)).values()]; + if (!rows.length) return null; + return ( + <div className="ai-sources"> + <span>Sources</span> + <ul className="ai-rows"> + {rows.map((c) => ( + <li key={c.path}> + <span className="ai-path">{c.path}</span> + {c.lessonId ? <Link className="ai-meta" to={`/c/${encodeURIComponent(courseId)}/${encodeURIComponent(c.lessonId)}`}>{c.lessonTitle ?? c.lessonId}</Link> : <span className="ai-meta">{c.kind}</span>} + </li> + ))} + </ul> + </div> + ); +} + +export default function AssistantPage() { + const { courseId = "" } = useParams(); + const [status, setStatus] = useState<Status | null>(null); + const [loadError, setLoadError] = useState<string | null>(null); + const [turns, setTurns] = useState<Turn[]>([]); + const [conversationId, setConversationId] = useState<string | null>(null); + const [history, setHistory] = useState<ConvSummary[]>([]); + const [draft, setDraft] = useState(""); + const [busy, setBusy] = useState(false); + const abortRef = useRef<AbortController | null>(null); + const lessonId = new URLSearchParams(typeof location === "undefined" ? "" : location.search).get("lesson") ?? undefined; + + const loadHistory = useCallback(async () => { + try { setHistory(await api<ConvSummary[]>(`/api/ai/courses/${encodeURIComponent(courseId)}/conversations`)); } catch { setHistory([]); } + }, [courseId]); + + useEffect(() => { + api<Status>(`/api/ai/status?courseId=${encodeURIComponent(courseId)}`).then(setStatus, (e) => setLoadError(e.message)); + loadHistory(); + const linked = new URLSearchParams(location.search).get("c"); + if (linked) openConversation(linked).catch(() => {}); + return () => abortRef.current?.abort(); + }, [courseId, loadHistory]); + + const mentor = status?.mentor ?? { available: false, on: true }; + + const setMentor = async (on: boolean) => { + if (!status) return; + setStatus({ ...status, mentor: { ...mentor, on } }); + try { + const r = await api<{ mentor: Status["mentor"] }>(`/api/ai/courses/${encodeURIComponent(courseId)}/mentor`, { method: "PUT", body: { off: !on } }); + setStatus((s) => (s ? { ...s, mentor: r.mentor } : s)); + } catch (e) { + setStatus({ ...status, mentor }); + setLoadError((e as Error).message); + } + }; + + async function openConversation(id: string) { + const c = await api<{ id: string; messages: { role: "user" | "assistant"; content: string; citations?: Citation[] }[] }>(`/api/ai/conversations/${id}`); + setConversationId(c.id); + setTurns(c.messages.map((m) => ({ role: m.role, text: m.content, citations: m.citations }))); + } + const deleteConversation = async (id: string) => { + await api(`/api/ai/conversations/${id}`, { method: "DELETE" }); + if (id === conversationId) { setConversationId(null); setTurns([]); } + loadHistory(); + }; + + const send = async () => { + const message = draft.trim(); + if (!message || busy) return; + setDraft(""); + setBusy(true); + const update = (fn: (t: Turn) => Turn) => setTurns((ts) => [...ts.slice(0, -1), fn(ts[ts.length - 1]!)]); + setTurns((ts) => [...ts, { role: "user", text: message }, { role: "assistant", text: "", tools: [] }]); + const ctl = new AbortController(); + abortRef.current = ctl; + try { + const res = await fetch(`/api/ai/courses/${encodeURIComponent(courseId)}/chat`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ message, conversationId, lessonId }), + signal: ctl.signal, + }); + if (!res.ok || !res.body) { + let msg = res.statusText; + try { msg = (await res.json()).error ?? msg; } catch {} + update((t) => ({ ...t, error: msg })); + return; + } + for await (const ev of readNdjson<ChatEvent>(res.body)) { + if (ev.type === "meta") { setConversationId(ev.conversationId); update((t) => ({ ...t, citations: ev.citations })); } + else if (ev.type === "text") update((t) => ({ ...t, text: t.text + ev.text })); + else if (ev.type === "tool") update((t) => ({ ...t, tools: [...(t.tools ?? []), ev.name] })); + else if (ev.type === "notice") update((t) => ({ ...t, notices: [...(t.notices ?? []), ev.message] })); + else if (ev.type === "error") update((t) => ({ ...t, error: ev.message })); + else if (ev.type === "done") update((t) => ({ ...t, text: ev.text || t.text, citations: ev.citations })); + } + loadHistory(); + } catch (e) { + if ((e as Error).name !== "AbortError") update((t) => ({ ...t, error: (e as Error).message })); + } finally { + setBusy(false); + abortRef.current = null; + } + }; + + const newConversation = () => { abortRef.current?.abort(); setConversationId(null); setTurns([]); }; + + const courseTitle = status?.course?.title ?? courseId; + const chap = <Link to={`/c/${encodeURIComponent(courseId)}`}>{courseTitle}</Link>; + + if (loadError && !status) return <div className="ai-page"><PageHead chap={chap} title="Course assistant" /><p className="ai-error">{loadError}</p></div>; + if (!status) return <div className="ai-page"><PageHead chap={chap} title="Course assistant" /></div>; + if (!status.course) return <div className="ai-page"><PageHead title="Course assistant" /><p className="ai-note">This course was not found.</p></div>; + if (!status.signedIn) { + return ( + <div className="ai-page"> + <PageHead chap={chap} title="Course assistant" /> + <p className="ai-note">Sign in to ask the assistant about this course.</p> + </div> + ); + } + if (!status.enabled) { + return ( + <div className="ai-page"> + <PageHead chap={chap} title="Course assistant" /> + <p className="ai-note"> + The assistant is off. AI stays off until an admin sets up a server provider, or you add your own API key in <Link to="/account/ai">Account, AI</Link>. Your key is used only for your own questions. + </p> + </div> + ); + } + + return ( + <div className="ai-page"> + <PageHead chap={chap} title="Course assistant"> + <p>Answers come from this course's lessons, transcripts, slides, code and your notes, with the sources listed.{status.source === "byok" ? " Using your own key." : ""}</p> + </PageHead> + + {mentor.available ? ( + <div className="ai-bar"> + <SegmentedToggle label="Mentor mode" value={mentor.on ? "on" : "off"} onChange={(v) => setMentor(v === "on")} options={[{ value: "on", label: "Mentor on" }, { value: "off", label: "Mentor off" }]} /> + <span className="ai-hint">{mentor.on ? "Follows the course's tutoring rules: questions and hints first, no full solutions." : "The course's tutoring rules are off for you in this course."}</span> + </div> + ) : null} + <MentorBanner on={mentor.on} available={mentor.available} /> + {loadError ? <p className="ai-error">{loadError}</p> : null} + + {turns.length ? ( + <ol className="ai-thread" aria-live="polite"> + {turns.map((t, i) => ( + <li key={i} className={`ai-turn ai-turn--${t.role}`}> + <span className="ai-who">{t.role === "user" ? "You" : "Assistant"}</span> + {t.text ? <Markdown text={t.text} /> : t.role === "assistant" && busy && i === turns.length - 1 && !t.error ? <p className="ai-activity">Thinking…</p> : null} + {t.tools?.length ? <p className="ai-activity">{[...new Set(t.tools.map(toolLabel))].join(", ").replace(/^./, (c) => c.toUpperCase())}.</p> : null} + {t.notices?.map((n) => <p key={n} className="ai-activity">{n}</p>)} + {t.error ? <p className="ai-error">{t.error}</p> : null} + {t.role === "assistant" && t.citations ? <Sources courseId={courseId} citations={t.citations} /> : null} + </li> + ))} + </ol> + ) : ( + <p className="ai-note">Ask about anything in the course: an idea you are unsure of, a failing test, or "quiz me on this module".</p> + )} + + <form className="ai-composer" onSubmit={(e) => { e.preventDefault(); send(); }}> + <label className="sr" htmlFor="ai-message">Your question</label> + <textarea id="ai-message" className="ai-textarea" value={draft} onChange={(e) => setDraft(e.target.value)} placeholder="Ask a question about this course" rows={4} /> + <div className="ai-actions"> + {turns.length ? <Pill variant="ghost" onClick={newConversation}>New conversation</Pill> : null} + {busy ? <Pill onClick={() => abortRef.current?.abort()}>Stop</Pill> : null} + <Pill variant="primary" type="submit" disabled={busy || !draft.trim()}>Send</Pill> + </div> + </form> + + {history.length ? ( + <section className="ai-history"> + <h2>Earlier conversations</h2> + <ul className="ai-rows"> + {history.map((h) => ( + <li key={h.id}> + <button type="button" className="ai-linkbtn" aria-current={h.id === conversationId} onClick={() => openConversation(h.id)}>{h.title || "Untitled"}</button> + <Pill variant="ghost" onClick={() => deleteConversation(h.id)} aria-label={`Delete conversation ${h.title ?? ""}`}>Delete</Pill> + </li> + ))} + </ul> + </section> + ) : null} + </div> + ); +} diff --git a/packages/feature-ai/src/web/GeneratePage.tsx b/packages/feature-ai/src/web/GeneratePage.tsx new file mode 100644 index 0000000..2162102 --- /dev/null +++ b/packages/feature-ai/src/web/GeneratePage.tsx @@ -0,0 +1,195 @@ +import { useEffect, useMemo, useState } from "react"; +import { Pill, ProgressBar, SegmentedToggle } from "@stc/ui"; +import { api, ApiError, type Status } from "./api"; +import { Field, PageHead } from "./components"; + +interface CourseOpt { id: string; title: string } +interface Sources { lessons: { id: string; title: string; type: string; module: string; hasText: boolean }[]; files: string[]; cache: string[] } +interface Job { id: string; status: "queued" | "running" | "done" | "failed"; progress: number; message: string | null; error: string | null; result: { location: string; paths: string[]; warnings: string[] } | null } + +const KINDS = [ + { value: "document", label: "Document" }, + { value: "exercise", label: "Exercise" }, + { value: "questions", label: "Questions" }, +]; +const STYLES = [ + { value: "pure-snake", label: "Module shape" }, + { value: "systemic-design", label: "Observe to Expansion Drop" }, +]; +const STYLE_HINT: Record<string, string> = { + "pure-snake": "Pure Snake's shape: why, objectives, concepts, a puzzle before the explanation, exercises with nudge, stronger hint and near-solution, and check questions.", + "systemic-design": "Systemic Design's beats: Observe, Predict, Spec, Build, Name it, Expansion Drop, Further reading.", +}; + +export default function GeneratePage() { + const [courses, setCourses] = useState<CourseOpt[] | null>(null); + const [denied, setDenied] = useState<string | null>(null); + const [enabled, setEnabled] = useState(true); + const [courseId, setCourseId] = useState(""); + const [src, setSrc] = useState<Sources | null>(null); + const [kind, setKind] = useState("document"); + const [style, setStyle] = useState("pure-snake"); + const [topic, setTopic] = useState(""); + const [instructions, setInstructions] = useState(""); + const [lessons, setLessons] = useState<Set<string>>(new Set()); + const [files, setFiles] = useState<Set<string>>(new Set()); + const [cache, setCache] = useState<Set<string>>(new Set()); + const [pasted, setPasted] = useState(""); + const [filter, setFilter] = useState(""); + const [job, setJob] = useState<Job | null>(null); + const [error, setError] = useState<string | null>(null); + + useEffect(() => { + api<CourseOpt[]>("/api/ai/studio/courses").then((c) => { + setCourses(c); + if (c[0]) setCourseId(c[0].id); + }, (e: ApiError) => setDenied(e.status === 401 ? "Sign in as an author to generate drafts." : e.message)); + api<Status>("/api/ai/status").then((s) => setEnabled(s.enabled), () => {}); + }, []); + + useEffect(() => { + if (!courseId) return; + setSrc(null); + setLessons(new Set()); setFiles(new Set()); setCache(new Set()); + api<Sources>(`/api/ai/studio/sources?courseId=${encodeURIComponent(courseId)}`).then(setSrc, (e) => setError(e.message)); + }, [courseId]); + + useEffect(() => { + if (!job || job.status === "done" || job.status === "failed") return; + const t = setTimeout(() => api<Job>(`/api/ai/studio/jobs/${job.id}`).then(setJob, (e) => setError(e.message)), 1000); + return () => clearTimeout(t); + }, [job]); + + const toggle = (set: Set<string>, setter: (s: Set<string>) => void, v: string) => { + const n = new Set(set); + if (n.has(v)) n.delete(v); else n.add(v); + setter(n); + }; + const byModule = useMemo(() => { + const m = new Map<string, Sources["lessons"]>(); + for (const l of src?.lessons ?? []) m.set(l.module, [...(m.get(l.module) ?? []), l]); + return [...m]; + }, [src]); + const shownFiles = (src?.files ?? []).filter((f) => !filter || f.toLowerCase().includes(filter.toLowerCase())).slice(0, 200); + const sourceCount = lessons.size + files.size + cache.size + (pasted.trim() ? 1 : 0); + + const generate = async () => { + setError(null); + setJob(null); + const sources = [ + ...[...lessons].map((lessonId) => ({ kind: "lesson", lessonId })), + ...[...files].map((path) => ({ kind: "file", path })), + ...[...cache].map((path) => ({ kind: "cache", path })), + ...(pasted.trim() ? [{ kind: "text", label: "Pasted text", text: pasted }] : []), + ]; + try { + const { jobId } = await api<{ jobId: string }>("/api/ai/studio/generate", { method: "POST", body: { courseId, kind, style, topic, instructions, sources } }); + setJob({ id: jobId, status: "queued", progress: 0, message: "Queued", error: null, result: null }); + } catch (e) { setError((e as Error).message); } + }; + + if (denied) return <div className="ai-page"><PageHead chap="Studio" title="Generate a draft" /><p className="ai-note">{denied}</p></div>; + if (!courses) return <div className="ai-page"><PageHead chap="Studio" title="Generate a draft" /></div>; + if (!courses.length) return <div className="ai-page"><PageHead chap="Studio" title="Generate a draft" /><p className="ai-note">You are not an author of any course yet.</p></div>; + + const running = job && (job.status === "queued" || job.status === "running"); + return ( + <div className="ai-page"> + <PageHead chap="Studio" title="Generate a draft"> + <p>Drafts a document, an exercise or knowledge-check questions from material you choose. The result is written as draft files for you to review and edit. Nothing is published.</p> + </PageHead> + {!enabled ? <p className="ai-banner">AI is not set up. Ask the owner to configure a provider, or add your own key in Account, AI.</p> : null} + + <form onSubmit={(e) => { e.preventDefault(); generate(); }}> + <div className="ai-grid"> + <Field label="Course"> + <select className="ai-select" value={courseId} onChange={(e) => setCourseId(e.target.value)}> + {courses.map((c) => <option key={c.id} value={c.id}>{c.title}</option>)} + </select> + </Field> + <Field label="Topic"> + <input className="ai-input" value={topic} onChange={(e) => setTopic(e.target.value)} placeholder="For example: Folding time" required /> + </Field> + </div> + <div className="ai-bar"> + <SegmentedToggle label="What to draft" value={kind} onChange={setKind} options={KINDS} /> + <SegmentedToggle label="House style" value={style} onChange={setStyle} options={STYLES} /> + </div> + <p className="ai-hint" style={{ margin: "-6px 0 18px" }}>{STYLE_HINT[style]}</p> + + <section className="ai-section"> + <h2>Material</h2> + <span className="ai-hint">Lessons and transcripts, code and text files, slide text from imports, or text you paste. {sourceCount ? `${sourceCount} selected.` : ""}</span> + {!src ? <p className="ai-hint">Loading…</p> : ( + <> + <div className="ai-checklist" role="group" aria-label="Lessons"> + {byModule.map(([mod, ls]) => ( + <div key={mod}> + <div className="ai-group">{mod}</div> + {ls.map((l) => ( + <label className="ai-check" key={l.id}> + <input type="checkbox" disabled={!l.hasText} checked={lessons.has(l.id)} onChange={() => toggle(lessons, setLessons, l.id)} /> + <span>{l.title} <small>{l.type}{l.hasText ? "" : ", no transcript"}</small></span> + </label> + ))} + </div> + ))} + </div> + {src.files.length ? ( + <> + <Field label="Files"> + <input className="ai-input" value={filter} onChange={(e) => setFilter(e.target.value)} placeholder="Filter files" /> + </Field> + <div className="ai-checklist" role="group" aria-label="Files"> + {shownFiles.map((f) => ( + <label className="ai-check" key={f}><input type="checkbox" checked={files.has(f)} onChange={() => toggle(files, setFiles, f)} /><span className="ai-mono">{f}</span></label> + ))} + </div> + </> + ) : null} + {src.cache.length ? ( + <div className="ai-checklist" role="group" aria-label="Slide text and transcripts"> + <div className="ai-group">Slide text and transcripts</div> + {src.cache.map((f) => ( + <label className="ai-check" key={f}><input type="checkbox" checked={cache.has(f)} onChange={() => toggle(cache, setCache, f)} /><span className="ai-mono">{f}</span></label> + ))} + </div> + ) : null} + </> + )} + <Field label="Pasted text" hint="A transcript, notes or anything else to draw on."> + <textarea className="ai-textarea" value={pasted} onChange={(e) => setPasted(e.target.value)} /> + </Field> + <Field label="Instructions" hint="Optional. For example: aim at Module 5, two exercises, JavaScript only."> + <textarea className="ai-textarea" value={instructions} onChange={(e) => setInstructions(e.target.value)} rows={3} /> + </Field> + </section> + + <div className="ai-footer"> + <Pill variant="primary" type="submit" disabled={!enabled || !topic.trim() || !!running}>Generate draft</Pill> + {error ? <span className="ai-error" style={{ margin: 0 }}>{error}</span> : null} + </div> + </form> + + {job ? ( + <section className="ai-section" aria-live="polite"> + <h2>{job.status === "done" ? "Draft written" : job.status === "failed" ? "Generation failed" : "Generating"}</h2> + {running ? <div className="ai-progress"><ProgressBar value={job.progress} label="Generation progress" /><span>{job.message ?? "Queued"}</span></div> : null} + {job.status === "failed" ? <p className="ai-error">{job.error}</p> : null} + {job.result ? ( + <> + <span className="ai-hint">Saved as a draft at <span className="ai-mono">{job.result.location}</span>. Review it in the studio before publishing.</span> + <ul className="ai-rows">{job.result.paths.map((p) => <li key={p}><span className="ai-path">{p}</span><span className="ai-meta">draft</span></li>)}</ul> + {job.result.warnings.length ? ( + <div className="ai-section"> + <h2>Check these</h2> + <ul className="ai-rows">{job.result.warnings.map((w) => <li key={w}><span>{w}</span><span /></li>)}</ul> + </div> + ) : null} + </> + ) : null} + </section> + ) : null} + </div> + ); +} diff --git a/packages/feature-ai/src/web/ai.css b/packages/feature-ai/src/web/ai.css new file mode 100644 index 0000000..74e9885 --- /dev/null +++ b/packages/feature-ai/src/web/ai.css @@ -0,0 +1,77 @@ +/* AI feature: assistant panel, settings and generator. Tokens only (packages/ui/src/tokens.css). */ +.ai-page { max-width: 760px; margin: 0 auto 64px; font-size: 14px; } +.ai-head { margin: 0 0 22px; padding-bottom: 20px; border-bottom: 1px solid var(--line); } +.ai-chap { font-size: 13px; font-weight: 500; color: var(--muted); margin: 0 0 10px; } +.ai-chap a { text-decoration: none; } +.ai-chap a:hover { color: var(--head); } +.ai-title { font: 600 28px/1.15 var(--sans); color: var(--head); margin: 0; letter-spacing: -.01em; } +.ai-lede { margin: 10px 0 0; color: var(--muted); font-size: 14px; line-height: 1.55; } +.ai-lede p { margin: 0; } + +.ai-bar { display: flex; flex-wrap: wrap; align-items: center; gap: 10px 14px; margin: 0 0 18px; } +.ai-bar .ai-hint { flex: 1; min-width: 200px; } +.ai-banner { margin: 0 0 18px; padding: 10px 14px; border: 1px solid var(--line2); border-radius: 10px; background: var(--side); color: var(--head); font-size: 13.5px; } +.ai-note { margin: 0 0 18px; color: var(--muted); font-size: 13.5px; line-height: 1.55; } +.ai-note a { color: var(--text); } +.ai-error { margin: 0 0 14px; color: var(--bad); font-size: 13.5px; } +.ai-ok { color: var(--ok); } + +/* conversation */ +.ai-thread { list-style: none; margin: 0 0 18px; padding: 0; border-top: 1px solid var(--line); } +.ai-turn { padding: 16px 2px; border-bottom: 1px solid var(--line); } +.ai-who { display: block; font-size: 12px; color: var(--faint); margin: 0 0 6px; } +.ai-turn--user .ai-md { color: var(--head); } +.ai-md { font-size: 15px; line-height: 1.6; color: var(--text); overflow-wrap: anywhere; } +.ai-md p, .ai-md ul, .ai-md ol { margin: 0 0 .8em; } +.ai-md > :last-child { margin-bottom: 0; } +.ai-md li { margin-bottom: .25em; } +.ai-md .ai-md-h { font-weight: 600; color: var(--head); } +.ai-md :not(pre) > code { font-size: .85em; padding: .08em .32em; background: color-mix(in srgb, var(--raised) 75%, transparent); border-radius: 4px; } +.ai-md pre.code, pre.code { font-size: 13px; line-height: 1.6; background: var(--side); border: 1px solid var(--line); border-radius: 10px; padding: 12px 14px; margin: 0 0 .8em; overflow-x: auto; white-space: pre; } +.ai-activity { margin: 8px 0 0; font-size: 12.5px; color: var(--faint); } +.ai-sources { margin: 12px 0 0; } +.ai-sources > span { display: block; font-size: 12px; color: var(--faint); margin: 0 0 4px; } + +/* rows (same rhythm as the mockup's .rows) */ +.ai-rows { list-style: none; margin: 0; padding: 0; border-top: 1px solid var(--line); } +.ai-rows li { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: 10px; align-items: center; padding: 9px 2px; border-bottom: 1px solid var(--line); font-size: 13.5px; } +.ai-rows .ai-path { font: 12.5px/1.4 var(--mono); color: var(--text); overflow-wrap: anywhere; } +.ai-rows .ai-meta { font-size: 12px; color: var(--faint); white-space: nowrap; } +.ai-rows a { text-decoration: none; } +.ai-rows a:hover { text-decoration: underline; text-underline-offset: 3px; } + +/* composer and forms */ +.ai-composer { display: grid; gap: 10px; } +.ai-composer .ai-actions { display: flex; gap: 8px; justify-content: flex-end; } +.ai-input, .ai-select, .ai-textarea { width: 100%; font-family: inherit; background: var(--side); border: 1px solid var(--line2); border-radius: 10px; padding: 9px 12px; color: var(--text); font-size: 14px; line-height: 1.5; } +.ai-textarea { min-height: 96px; resize: vertical; } +.ai-input:focus, .ai-select:focus, .ai-textarea:focus { outline: 0; border-color: var(--faint); } +.ai-input:focus-visible, .ai-select:focus-visible, .ai-textarea:focus-visible { outline: 2px solid var(--accent); outline-offset: 1px; } +.ai-mono { font-family: var(--mono); font-size: 13px; } +.ai-field { display: grid; gap: 5px; margin: 0 0 14px; align-content: start; } +.ai-label { font-size: 12.5px; font-weight: 500; color: var(--muted); } +.ai-hint { font-size: 12.5px; color: var(--faint); line-height: 1.45; } +.ai-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: 0 16px; align-items: start; } +.ai-section { margin: 30px 0 0; } +.ai-section > h2 { font: 600 15px/1.3 var(--sans); color: var(--head); margin: 0 0 4px; } +.ai-section > .ai-hint { display: block; margin: 0 0 12px; } +.ai-check { display: flex; gap: 10px; align-items: baseline; padding: 6px 2px; font-size: 13.5px; color: var(--text); cursor: pointer; } +.ai-check input { accent-color: var(--accent); } +.ai-check small { color: var(--faint); font-size: 12px; } +.ai-checklist { max-height: 300px; overflow-y: auto; border-top: 1px solid var(--line); border-bottom: 1px solid var(--line); padding: 4px 0; margin: 0 0 14px; } +.ai-group { font-size: 12px; color: var(--faint); padding: 8px 2px 2px; } +.ai-server { border-top: 1px solid var(--line); padding: 14px 0 4px; } +.ai-server:last-of-type { border-bottom: 1px solid var(--line); } +.ai-footer { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; margin: 22px 0 0; } +.ai-progress { display: flex; align-items: center; gap: 12px; margin: 18px 0 0; font-size: 13px; color: var(--muted); } +.ai-progress .stc-bar { max-width: 260px; } +.ai-history { margin: 34px 0 0; } +.ai-history h2 { font: 600 15px/1.3 var(--sans); color: var(--head); margin: 0 0 10px; } +.ai-linkbtn { background: none; border: 0; padding: 0; color: var(--text); text-align: left; font-size: 13.5px; } +.ai-linkbtn:hover { text-decoration: underline; text-underline-offset: 3px; } +.ai-linkbtn[aria-current="true"] { color: var(--head); font-weight: 500; } + +@media (max-width: 640px) { + .ai-title { font-size: 24px; } + .ai-rows li { grid-template-columns: minmax(0, 1fr); } +} diff --git a/packages/feature-ai/src/web/api.ts b/packages/feature-ai/src/web/api.ts new file mode 100644 index 0000000..f187ace --- /dev/null +++ b/packages/feature-ai/src/web/api.ts @@ -0,0 +1,51 @@ +export class ApiError extends Error { + constructor(public status: number, message: string) { super(message); } +} + +export async function api<T>(path: string, init: { method?: string; body?: unknown } = {}): Promise<T> { + const res = await fetch(path, { + method: init.method ?? "GET", + headers: init.body === undefined ? undefined : { "content-type": "application/json" }, + body: init.body === undefined ? undefined : JSON.stringify(init.body), + credentials: "same-origin", + }); + if (!res.ok) { + let msg = res.statusText; + try { msg = (await res.json()).error ?? msg; } catch {} + throw new ApiError(res.status, msg); + } + return res.json() as Promise<T>; +} + +/** Parses a newline-delimited JSON stream into objects. */ +export async function* readNdjson<T = unknown>(body: ReadableStream<Uint8Array>): AsyncIterable<T> { + const reader = body.getReader(); + const decoder = new TextDecoder(); + let buf = ""; + for (;;) { + const { value, done } = await reader.read(); + if (value) buf += decoder.decode(value, { stream: true }); + let nl: number; + while ((nl = buf.indexOf("\n")) !== -1) { + const line = buf.slice(0, nl).trim(); + buf = buf.slice(nl + 1); + if (line) yield JSON.parse(line) as T; + } + if (done) break; + } + buf += decoder.decode(); + if (buf.trim()) yield JSON.parse(buf) as T; +} + +export interface Citation { path: string; kind: string; lessonId?: string; lessonTitle?: string } +export interface Status { + enabled: boolean; source: "byok" | "server" | null; signedIn: boolean; + course?: { id: string; title: string }; mentor?: { available: boolean; on: boolean }; +} +export type ChatEvent = + | { type: "meta"; conversationId: string; mentorOn: boolean; citations: Citation[] } + | { type: "text"; text: string } + | { type: "tool"; name: string } + | { type: "notice"; message: string } + | { type: "error"; message: string } + | { type: "done"; text: string; citations: Citation[] }; diff --git a/packages/feature-ai/src/web/components.tsx b/packages/feature-ai/src/web/components.tsx new file mode 100644 index 0000000..3120645 --- /dev/null +++ b/packages/feature-ai/src/web/components.tsx @@ -0,0 +1,77 @@ +import { Fragment, type ReactNode } from "react"; +import "./ai.css"; + +/** Shown whenever the learner has switched the course's mentor rules off. */ +export function MentorBanner({ on, available }: { on: boolean; available: boolean }) { + if (on || !available) return null; + return <p className="ai-banner" role="status">Mentor mode off: I'll give full answers</p>; +} + +export function Field({ label, hint, children }: { label: string; hint?: string; children: ReactNode }) { + return ( + <label className="ai-field"> + <span className="ai-label">{label}</span> + {children} + {hint ? <span className="ai-hint">{hint}</span> : null} + </label> + ); +} + +export function PageHead({ chap, title, children }: { chap?: ReactNode; title: string; children?: ReactNode }) { + return ( + <header className="ai-head"> + {chap ? <p className="ai-chap">{chap}</p> : null} + <h1 className="ai-title">{title}</h1> + {children ? <div className="ai-lede">{children}</div> : null} + </header> + ); +} + +function inline(text: string, key: string): ReactNode[] { + const out: ReactNode[] = []; + const re = /(`[^`]+`|\*\*[^*]+\*\*)/g; + let last = 0; + let m: RegExpExecArray | null; + let i = 0; + while ((m = re.exec(text))) { + if (m.index > last) out.push(text.slice(last, m.index)); + const tok = m[0]; + out.push(tok.startsWith("`") ? <code key={`${key}-${i++}`}>{tok.slice(1, -1)}</code> : <strong key={`${key}-${i++}`}>{tok.slice(2, -2)}</strong>); + last = m.index + tok.length; + } + if (last < text.length) out.push(text.slice(last)); + return out; +} + +/** A small, safe Markdown subset for assistant answers: paragraphs, lists, headings, fenced code, `code`, **bold**. */ +export function Markdown({ text }: { text: string }) { + const blocks: ReactNode[] = []; + const lines = text.replace(/\r\n/g, "\n").split("\n"); + let i = 0; + let k = 0; + while (i < lines.length) { + const line = lines[i]!; + if (/^\s*```/.test(line)) { + const code: string[] = []; + i++; + while (i < lines.length && !/^\s*```/.test(lines[i]!)) code.push(lines[i++]!); + i++; + blocks.push(<pre className="code" key={k++}>{code.join("\n")}</pre>); + } else if (/^\s*([-*]|\d+\.)\s+/.test(line)) { + const ordered = /^\s*\d+\./.test(line); + const items: string[] = []; + while (i < lines.length && /^\s*([-*]|\d+\.)\s+/.test(lines[i]!)) items.push(lines[i++]!.replace(/^\s*([-*]|\d+\.)\s+/, "")); + const kids = items.map((it, j) => <li key={j}>{inline(it, `${k}-${j}`)}</li>); + blocks.push(ordered ? <ol key={k++}>{kids}</ol> : <ul key={k++}>{kids}</ul>); + } else if (/^#{1,6}\s/.test(line)) { + blocks.push(<p className="ai-md-h" key={k++}>{inline(line.replace(/^#+\s*/, ""), String(k))}</p>); + i++; + } else if (!line.trim()) i++; + else { + const para: string[] = []; + while (i < lines.length && lines[i]!.trim() && !/^\s*(```|[-*]\s|\d+\.\s|#{1,6}\s)/.test(lines[i]!)) para.push(lines[i++]!); + blocks.push(<p key={k++}>{para.map((p, j) => <Fragment key={j}>{j ? <br /> : null}{inline(p, `${k}-${j}`)}</Fragment>)}</p>); + } + } + return <div className="ai-md">{blocks}</div>; +} diff --git a/packages/feature-ai/src/web/index.ts b/packages/feature-ai/src/web/index.ts new file mode 100644 index 0000000..b3c7387 --- /dev/null +++ b/packages/feature-ai/src/web/index.ts @@ -0,0 +1,11 @@ +import type { WebFeature } from "@stc/contracts"; + +export const aiWeb: WebFeature = { + name: "ai", + routes: [ + { path: "/c/:courseId/assistant", element: () => import("./AssistantPage") }, + { path: "/admin/ai", element: () => import("./AdminAiPage"), nav: { label: "AI", area: "admin" } }, + { path: "/account/ai", element: () => import("./AccountAiPage") }, + { path: "/studio/generate", element: () => import("./GeneratePage") }, + ], +}; diff --git a/packages/feature-ai/test/feature.test.ts b/packages/feature-ai/test/feature.test.ts index 3912dfc..6a8c700 100644 --- a/packages/feature-ai/test/feature.test.ts +++ b/packages/feature-ai/test/feature.test.ts @@ -101,6 +101,7 @@ forEachDb("chat cites course files, applies mentor rules, and stores the convers expect(convs).toHaveLength(1); const conv = await (await call("GET", `/api/ai/conversations/${convs[0].id}`, "ann")).json(); expect(conv.messages.map((m: { role: string }) => m.role)).toEqual(["user", "assistant"]); + expect(conv.messages[1].citations.some((c: { path: string }) => c.path === "lessons/04-exercise.md")).toBe(true); expect((await call("GET", `/api/ai/conversations/${convs[0].id}`, "bob")).status).toBe(404); // Learner switches mentor mode off for this course: rules leave the prompt. diff --git a/packages/feature-ai/test/web.test.tsx b/packages/feature-ai/test/web.test.tsx new file mode 100644 index 0000000..3f0d540 --- /dev/null +++ b/packages/feature-ai/test/web.test.tsx @@ -0,0 +1,36 @@ +import { expect, test } from "bun:test"; +import { renderToString } from "react-dom/server"; +import { readNdjson } from "../src/web/api"; +import { MentorBanner, Markdown } from "../src/web/components"; +import { aiWeb } from "../src/web"; + +test("readNdjson splits events across chunk boundaries", async () => { + const text = '{"type":"meta","n":1}\n{"type":"text","text":"héllo"}\n{"type":"done"}\n'; + const bytes = new TextEncoder().encode(text); + const stream = new ReadableStream<Uint8Array>({ + start(c) { for (let i = 0; i < bytes.length; i += 5) c.enqueue(bytes.slice(i, i + 5)); c.close(); }, + }); + const out: unknown[] = []; + for await (const ev of readNdjson(stream)) out.push(ev); + expect(out).toEqual([{ type: "meta", n: 1 }, { type: "text", text: "héllo" }, { type: "done" }]); +}); + +test("the mentor banner says so plainly when mentor mode is off", () => { + expect(renderToString(<MentorBanner on={false} available />)).toContain("Mentor mode off: I'll give full answers"); + expect(renderToString(<MentorBanner on available />)).toBe(""); + expect(renderToString(<MentorBanner on={false} available={false} />)).toBe(""); +}); + +test("markdown renders code and text without raw HTML", () => { + const html = renderToString(<Markdown text={"Use **this**:\n\n```js\nconst a = 1 < 2\n```\n\n- one `x`\n- <b>two</b>"} />); + expect(html).toContain("<strong>this</strong>"); + expect(html).toContain('<pre class="code">'); + expect(html).toContain("const a = 1 < 2"); + expect(html).toContain("<code>x</code>"); + expect(html).toContain("<b>two</b>"); +}); + +test("web routes", () => { + expect(aiWeb.routes.map((r) => r.path)).toEqual(["/c/:courseId/assistant", "/admin/ai", "/account/ai", "/studio/generate"]); + expect(aiWeb.routes.find((r) => r.path === "/admin/ai")!.nav).toEqual({ label: "AI", area: "admin" }); +}); From 6fb0019cd92a22ae46bb9b6c77429339deb55b11 Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 04:36:48 +0100 Subject: [PATCH 44/62] feat(ai): read the importer's deck and transcript cache; studio DraftWriter fits DraftSink as is Retrieval now reads <dataDir>/cache/decks/<courseId>/<deckId>/text.json and <dataDir>/cache/transcripts/<courseId>/<lessonId>.json (skipped when the lesson has a transcript file). DraftSink returns { location?, branch?, commit? }, so the studio's createDraftWriter(ctx) needs no adapter; generateDraft computes paths itself and reports the draft branch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .../feature-ai/src/generator/draft-sink.ts | 22 ++++++-- packages/feature-ai/src/generator/generate.ts | 13 ++++- .../feature-ai/src/retrieval/course-index.ts | 56 ++++++++++++++++++- packages/feature-ai/src/web/GeneratePage.tsx | 4 +- packages/feature-ai/test/generator.test.ts | 24 +++++++- packages/feature-ai/test/retrieval.test.ts | 49 +++++++++++++++- 6 files changed, 156 insertions(+), 12 deletions(-) diff --git a/packages/feature-ai/src/generator/draft-sink.ts b/packages/feature-ai/src/generator/draft-sink.ts index 720c811..794c75d 100644 --- a/packages/feature-ai/src/generator/draft-sink.ts +++ b/packages/feature-ai/src/generator/draft-sink.ts @@ -4,11 +4,25 @@ import { dirname, join, resolve, sep } from "node:path"; export interface DraftFile { path: string; content: string } /** - * Where generated files go. Same signature as the studio's `createDraftWriter(ctx).writeDraftFiles`, which - * commits onto the author's draft branch; Phase 2 passes that in. Drafts are never published by the AI package. + * What a sink reports back. The studio's writer returns `{ branch, commit }` (a commit on `draft/<authorId>`); + * the filesystem fallback returns `{ location }` (a folder under `<dataDir>/drafts`). + */ +export interface DraftWriteResult { location?: string; branch?: string; commit?: string } + +/** + * Where generated files go. Same method as the studio's `DraftWriter` (`createDraftWriter(ctx, { identity? })` in + * @stc/feature-studio), so that writer is a `DraftSink` as is, with no adapter: Phase 2 passes it in as + * `aiFeature(ctx, { draftSink: createDraftWriter(ctx) })`. Drafts are never published by the AI package. */ export interface DraftSink { - writeDraftFiles(courseId: string, authorId: string, files: DraftFile[], message: string): Promise<{ location: string; paths: string[] }>; + writeDraftFiles(courseId: string, authorId: string, files: DraftFile[], message: string): Promise<DraftWriteResult>; +} + +/** Human-readable place of a draft: the folder, or `draft/<author> @ <short commit>`. */ +export function draftLocation(r: DraftWriteResult): string { + if (r.location) return r.location; + if (r.branch) return r.commit ? `${r.branch} @ ${r.commit.slice(0, 7)}` : r.branch; + return "draft"; } export function safeRelPath(p: string): string | null { @@ -40,7 +54,7 @@ export function createFsDraftSink(dataDir: string): DraftSink { join(location, "DRAFT.json"), JSON.stringify({ courseId, authorId, message, createdAt: new Date().toISOString(), published: false, files: paths }, null, 2), ); - return { location, paths }; + return { location }; }, }; } diff --git a/packages/feature-ai/src/generator/generate.ts b/packages/feature-ai/src/generator/generate.ts index c0da188..6d8b23d 100644 --- a/packages/feature-ai/src/generator/generate.ts +++ b/packages/feature-ai/src/generator/generate.ts @@ -2,7 +2,7 @@ import { stat } from "node:fs/promises"; import { join } from "node:path"; import type { AiProvider } from "@stc/contracts"; import { resolveInside, type CourseRepository, type LoadedCourse } from "@stc/course-format"; -import { safeRelPath, type DraftFile, type DraftSink } from "./draft-sink"; +import { draftLocation, safeRelPath, type DraftFile, type DraftSink } from "./draft-sink"; import { OUTPUT_FORMAT, REQUIRED, VOICE, styleTemplate, type Kind, type Style } from "./templates"; export type GenerateSource = @@ -20,7 +20,8 @@ export interface GenerateRequest { instructions?: string; } -export interface GenerateResult { location: string; paths: string[]; warnings: string[] } +/** `location` is a folder (filesystem sink) or `draft/<author> @ <commit>` (studio writer). */ +export interface GenerateResult { location: string; branch?: string; commit?: string; paths: string[]; warnings: string[] } const MAX_MATERIAL = 400_000; @@ -165,5 +166,11 @@ export async function generateDraft(o: { const files = parsed.files.map((f) => (/\.(md|markdown)$/i.test(f.path) ? { ...f, content: marker(o.req) + f.content } : f)); await o.onProgress?.(0.9, "Writing drafts"); const written = await o.sink.writeDraftFiles(course.id, o.authorId, files, `AI draft: ${o.req.kind} "${o.req.topic}" (${o.req.style} style)`); - return { ...written, warnings: [...material.warnings, ...parsed.warnings, ...checkHouseStyle(o.req, parsed.files)] }; + return { + location: draftLocation(written), + ...(written.branch ? { branch: written.branch } : {}), + ...(written.commit ? { commit: written.commit } : {}), + paths: files.map((f) => f.path), + warnings: [...material.warnings, ...parsed.warnings, ...checkHouseStyle(o.req, parsed.files)], + }; } diff --git a/packages/feature-ai/src/retrieval/course-index.ts b/packages/feature-ai/src/retrieval/course-index.ts index 57af7fb..0dadd53 100644 --- a/packages/feature-ai/src/retrieval/course-index.ts +++ b/packages/feature-ai/src/retrieval/course-index.ts @@ -128,6 +128,18 @@ function lessonTitleOf(course: LoadedCourse, lessonId: string | undefined): stri return lessonId ? course.lessonIndex.get(lessonId)?.lesson.title : undefined; } +async function readJson<T>(path: string): Promise<T | null> { + try { + const f = Bun.file(path); + if (!(await f.exists()) || f.size > MAX_FILE * 25) return null; + return (await f.json()) as T; + } catch { + return null; + } +} + +const clock = (s: number) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`; + export async function buildCourseIndex(course: LoadedCourse, dataDir: string): Promise<CourseIndex> { const bm = new Bm25<Chunk>(); const seen = new Set<string>(); @@ -179,8 +191,50 @@ export async function buildCourseIndex(course: LoadedCourse, dataDir: string): P }); } - // Derived text from the importer and transcription jobs: <dataDir>/cache/<courseId>/**. const decks = new Map((course.manifest?.decks ?? []).map((d) => [d.id, d])); + + // The importer's deck cache: <dataDir>/cache/decks/<courseId>/<deckId>/text.json = [{ page, text }]. + for (const deck of decks.values()) { + const pages = await readJson<{ page?: number; text?: string }[]>(join(dataDir, "cache", "decks", course.id, deck.id, "text.json")); + if (!Array.isArray(pages)) continue; + const lessonId = course.lessonIndex.has(deck.attachTo) ? deck.attachTo : undefined; + for (const p of pages) { + const text = typeof p?.text === "string" ? p.text.trim() : ""; + if (!text) continue; + for (const part of splitLong(text)) { + add({ path: posix(deck.file), kind: "deck", text: part, heading: p.page ? `Slide ${p.page}` : deck.title, lessonId, lessonTitle: lessonTitleOf(course, lessonId) }); + } + } + } + + // The importer's transcription cache: <dataDir>/cache/transcripts/<courseId>/<lessonId>.json = + // { lessonId, text, segments: [{ start, end, text }] }. Skipped when the lesson has a transcript file in the + // course, which the transcription job writes alongside and which is indexed above. + const trDir = join(dataDir, "cache", "transcripts", course.id); + for (const name of await readdir(trDir).catch(() => [] as string[])) { + if (extname(name) !== ".json") continue; + const lessonId = name.slice(0, -5); + const entry = course.lessonIndex.get(lessonId); + if (!entry || entry.lesson.transcript) continue; + const tr = await readJson<{ text?: string; segments?: { start?: number; text?: string }[] }>(join(trDir, name)); + if (!tr) continue; + const segs = Array.isArray(tr.segments) ? tr.segments.filter((s) => typeof s?.text === "string" && s.text.trim()) : []; + const blocks: { at: number; text: string }[] = []; + if (segs.length) { + for (const s of segs) { + const last = blocks[blocks.length - 1]; + if (last && last.text.length + s.text!.length < MAX_CHUNK) last.text += ` ${s.text!.trim()}`; + else blocks.push({ at: Number(s.start) || 0, text: s.text!.trim() }); + } + } else if (typeof tr.text === "string" && tr.text.trim()) { + blocks.push(...splitLong(tr.text.trim()).map((t) => ({ at: -1, text: t }))); + } + for (const b of blocks) { + add({ path: entry.lesson.file ? posix(entry.lesson.file) : `transcripts/${lessonId}`, kind: "transcript", text: b.text, heading: b.at >= 0 ? `at ${clock(b.at)}` : undefined, lessonId, lessonTitle: entry.lesson.title }); + } + } + + // Older layout: plain text under <dataDir>/cache/<courseId>/**. const cacheRoot = join(dataDir, "cache", course.id); await walk(cacheRoot, async (abs, rel) => { if (!CACHE_TEXT.has(extname(rel).toLowerCase())) return true; diff --git a/packages/feature-ai/src/web/GeneratePage.tsx b/packages/feature-ai/src/web/GeneratePage.tsx index 2162102..c96c0fe 100644 --- a/packages/feature-ai/src/web/GeneratePage.tsx +++ b/packages/feature-ai/src/web/GeneratePage.tsx @@ -5,7 +5,7 @@ import { Field, PageHead } from "./components"; interface CourseOpt { id: string; title: string } interface Sources { lessons: { id: string; title: string; type: string; module: string; hasText: boolean }[]; files: string[]; cache: string[] } -interface Job { id: string; status: "queued" | "running" | "done" | "failed"; progress: number; message: string | null; error: string | null; result: { location: string; paths: string[]; warnings: string[] } | null } +interface Job { id: string; status: "queued" | "running" | "done" | "failed"; progress: number; message: string | null; error: string | null; result: { location: string; branch?: string; commit?: string; paths: string[]; warnings: string[] } | null } const KINDS = [ { value: "document", label: "Document" }, @@ -178,7 +178,7 @@ export default function GeneratePage() { {job.status === "failed" ? <p className="ai-error">{job.error}</p> : null} {job.result ? ( <> - <span className="ai-hint">Saved as a draft at <span className="ai-mono">{job.result.location}</span>. Review it in the studio before publishing.</span> + <span className="ai-hint">{job.result.branch ? "Committed to your draft branch " : "Saved as a draft at "}<span className="ai-mono">{job.result.location}</span>. Review it in the studio before publishing.</span> <ul className="ai-rows">{job.result.paths.map((p) => <li key={p}><span className="ai-path">{p}</span><span className="ai-meta">draft</span></li>)}</ul> {job.result.warnings.length ? ( <div className="ai-section"> diff --git a/packages/feature-ai/test/generator.test.ts b/packages/feature-ai/test/generator.test.ts index 394a39e..8f91074 100644 --- a/packages/feature-ai/test/generator.test.ts +++ b/packages/feature-ai/test/generator.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { createCourseRepository, type CourseRepository } from "@stc/course-format"; import { buildGeneratorPrompt, gatherMaterial, generateDraft, parseFiles, checkHouseStyle, type GenerateRequest } from "../src/generator/generate"; -import { createFsDraftSink } from "../src/generator/draft-sink"; +import { createFsDraftSink, type DraftSink } from "../src/generator/draft-sink"; import { fakeProvider } from "./helpers"; const FIXTURES = join(import.meta.dir, "../../../fixtures/courses"); @@ -88,3 +88,25 @@ test("generation writes drafts through the sink and never touches the course", a expect(readdirSync(join(FIXTURES, "sample"), { recursive: true }).length).toBe(before); expect(existsSync(join(FIXTURES, "sample", "course"))).toBe(false); }); + +// The studio's DraftWriter (`createDraftWriter(ctx)` in @stc/feature-studio), reproduced structurally: it commits +// to `draft/<authorId>` and returns { branch, commit }. It must be usable as a DraftSink without an adapter. +interface StudioDraftWriterShape { + writeDraftFiles(courseId: string, authorId: string, files: { path: string; content: string }[], message: string): Promise<{ branch: string; commit: string }>; +} + +test("a studio DraftWriter is a DraftSink as is, and the result names the draft branch", async () => { + const calls: { courseId: string; authorId: string; paths: string[]; message: string }[] = []; + const writer: StudioDraftWriterShape = { + async writeDraftFiles(courseId, authorId, files, message) { + calls.push({ courseId, authorId, paths: files.map((f) => f.path), message }); + return { branch: `draft/${authorId}`, commit: "0123456789abcdef0123456789abcdef01234567" }; + }, + }; + const sink: DraftSink = writer; + const { provider } = fakeProvider([() => ({ text: "=== FILE: questions/adding.md ===\n## Check your understanding\n=== END FILE ===\n" })]); + const r = await generateDraft({ provider, repo, req: req({ kind: "questions" }), sink, authorId: "author-1", dataDir }); + expect(calls).toEqual([{ courseId: "sample", authorId: "author-1", paths: ["questions/adding.md"], message: expect.stringContaining("AI draft") }]); + expect(r).toMatchObject({ branch: "draft/author-1", commit: "0123456789abcdef0123456789abcdef01234567", paths: ["questions/adding.md"] }); + expect(r.location).toBe("draft/author-1 @ 0123456"); +}); diff --git a/packages/feature-ai/test/retrieval.test.ts b/packages/feature-ai/test/retrieval.test.ts index 96f51b9..f9d94c3 100644 --- a/packages/feature-ai/test/retrieval.test.ts +++ b/packages/feature-ai/test/retrieval.test.ts @@ -1,5 +1,5 @@ import { afterAll, beforeAll, expect, test } from "bun:test"; -import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { cpSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { createCourseRepository, type LoadedCourse } from "@stc/course-format"; @@ -81,3 +81,50 @@ test("notes are searched alongside the course", async () => { const hits = idx.search("zebras striped", 3, extra); expect(hits[0]).toMatchObject({ kind: "note", lessonId: "m1-intro", lessonTitle: "Introduction" }); }); + +// The importer's layout (feature-import): <dataDir>/cache/decks/<courseId>/<deckId>/text.json ([{ page, text }]) +// and <dataDir>/cache/transcripts/<courseId>/<lessonId>.json ({ lessonId, text, segments: [{ start, end, text }] }). +test("reads the importer's deck text.json and transcript cache", async () => { + const dir = mkdtempSync(join(tmpdir(), "stc-ai-imp-")); + try { + const courses = join(dir, "courses"); + cpSync(join(FIXTURES, "sample"), join(courses, "sample"), { recursive: true }); + const yaml = join(courses, "sample/stcourse.yaml"); + writeFileSync(yaml, readFileSync(yaml, "utf8").replace(", transcript: lessons/02-video.md", "")); + const c = (await createCourseRepository([courses])).get("sample")!; + expect(c.status).toBe("ok"); + const data = join(dir, "data"); + mkdirSync(join(data, "cache/decks/sample/d1"), { recursive: true }); + writeFileSync(join(data, "cache/decks/sample/d1/text.json"), JSON.stringify([ + { page: 1, text: "Title slide" }, + { page: 2, text: "Orchestration with Kubernetes pods and replica sets." }, + { page: 3, text: "" }, + ])); + mkdirSync(join(data, "cache/transcripts/sample"), { recursive: true }); + writeFileSync(join(data, "cache/transcripts/sample/m1-video.json"), JSON.stringify({ + lessonId: "m1-video", + text: "Welcome. Today the presenter shows a waveform monitor.", + segments: [{ start: 0, end: 4, text: "Welcome." }, { start: 65, end: 70, text: "Today the presenter shows a waveform monitor." }], + })); + const idx = await buildCourseIndex(c, data); + const deck = idx.search("Kubernetes replica sets", 3)[0]!; + expect(deck).toMatchObject({ kind: "deck", path: "media/deck.pdf", heading: "Slide 2", lessonId: "m1-video" }); + const tr = idx.search("waveform monitor", 3)[0]!; + expect(tr).toMatchObject({ kind: "transcript", lessonId: "m1-video", lessonTitle: "Watch the clip" }); + expect(tr.text).toContain("waveform monitor"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("a cached transcript is skipped when the lesson already has a transcript file", async () => { + const data = mkdtempSync(join(tmpdir(), "stc-ai-tr-")); + try { + mkdirSync(join(data, "cache/transcripts/sample"), { recursive: true }); + writeFileSync(join(data, "cache/transcripts/sample/m1-video.json"), JSON.stringify({ lessonId: "m1-video", text: "zebracorn unique words", segments: [] })); + const idx = await buildCourseIndex(course, data); + expect(idx.search("zebracorn", 3)).toEqual([]); + } finally { + rmSync(data, { recursive: true, force: true }); + } +}); From a47e6e09717b99b1f74f021fc75e84a8042d0802 Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 04:37:25 +0100 Subject: [PATCH 45/62] feat(ai): my_progress tool so the assistant sees the learner's progress Spec 9.2 has the assistant retrieve from the learner's notes and progress; progress was missing. Reads progressRepo(db).get for the signed-in user. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- packages/feature-ai/src/assistant/prompt.ts | 2 +- packages/feature-ai/src/assistant/tools.ts | 23 +++++++++++++++++++++ packages/feature-ai/src/feature.ts | 5 +++-- packages/feature-ai/test/assistant.test.ts | 13 ++++++++++++ packages/feature-ai/test/feature.test.ts | 1 + 5 files changed, 41 insertions(+), 3 deletions(-) diff --git a/packages/feature-ai/src/assistant/prompt.ts b/packages/feature-ai/src/assistant/prompt.ts index c70e0e7..c1b2496 100644 --- a/packages/feature-ai/src/assistant/prompt.ts +++ b/packages/feature-ai/src/assistant/prompt.ts @@ -29,7 +29,7 @@ export function buildSystemPrompt(o: { parts.push( `You are the course assistant for "${title}" on STCourses, a self-hosted course platform. You help one learner with this course: you answer questions, explain ideas, and quiz them when they ask.`, [ - "Ground your answers in the course material. Use the retrieved sources below and your tools (search_course, read_lesson, read_file, list_notes) to look things up rather than guessing.", + "Ground your answers in the course material. Use the retrieved sources below and your tools (search_course, read_lesson, read_file, list_notes, my_progress) to look things up rather than guessing.", "When you rely on a source, cite its file path and lesson in square brackets, for example [lessons/04-exercise.md · Adding numbers]. If the course does not cover something, say so plainly.", "Reference solutions are for authors only and are not available to you. Do not try to reconstruct them.", "Keep answers short and plain. Use Markdown for code.", diff --git a/packages/feature-ai/src/assistant/tools.ts b/packages/feature-ai/src/assistant/tools.ts index 142cc61..8ab1118 100644 --- a/packages/feature-ai/src/assistant/tools.ts +++ b/packages/feature-ai/src/assistant/tools.ts @@ -14,6 +14,7 @@ export interface ToolSet { close?(): Promise<void>; } +type ProgressLike = { state: "not-started" | "in-progress" | "done"; positionSeconds: number | null }; type NoteLike = { id: string; lessonId: string; body: string; anchorKind: string; anchorValue: string | null }; const MAX_READ = 60_000; const str = (v: unknown) => (typeof v === "string" ? v : ""); @@ -24,6 +25,8 @@ export function courseTools(o: { course: LoadedCourse; index: CourseIndex; listNotes: (lessonId?: string) => Promise<NoteLike[]>; + /** The learner's progress in this course, keyed by lesson id (shape of `progressRepo(db).get`). */ + getProgress?: () => Promise<Record<string, ProgressLike>>; }): ToolSet { const sources: Chunk[] = []; const { course } = o; @@ -54,6 +57,13 @@ export function courseTools(o: { description: "List the learner's own notes in this course, optionally for one lesson.", inputSchema: { type: "object", properties: { lessonId: { type: "string" } }, additionalProperties: false }, }, + ...(o.getProgress + ? [{ + name: "my_progress", + description: "The learner's progress in this course: which lessons are done, in progress (with the video position) or not started.", + inputSchema: { type: "object", properties: {}, additionalProperties: false }, + }] + : []), ]; const lessonList = () => (course.manifest?.modules ?? []).map((m) => `${m.title}: ${m.lessons.map((l) => `${l.id} (${l.title})`).join(", ")}`).join("\n"); @@ -100,6 +110,19 @@ export function courseTools(o: { if (!notes.length) return "The learner has no notes here yet."; return noteChunks(course, notes).map((c) => `- ${[c.lessonTitle ?? c.lessonId, c.heading].filter(Boolean).join(", ")}: ${c.text}`).join("\n"); } + case "my_progress": { + if (!o.getProgress) return `Unknown tool "${name}".`; + const p = await o.getProgress(); + const lessons = (course.manifest?.modules ?? []).flatMap((m) => m.lessons); + const done = lessons.filter((l) => p[l.id]?.state === "done").length; + const clock = (s: number) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`; + const line = (l: { id: string; title: string }) => { + const e = p[l.id]; + const state = e?.state === "done" ? "done" : e?.state === "in-progress" ? `in progress${e.positionSeconds ? `, at ${clock(e.positionSeconds)}` : ""}` : "not started"; + return `- ${l.title} (${l.id}): ${state}`; + }; + return [`${done} of ${lessons.length} lessons done.`, ...(course.manifest?.modules ?? []).flatMap((m) => [`${m.title}:`, ...m.lessons.map(line)])].join("\n"); + } default: return `Unknown tool "${name}".`; } diff --git a/packages/feature-ai/src/feature.ts b/packages/feature-ai/src/feature.ts index f74aa04..625fa23 100644 --- a/packages/feature-ai/src/feature.ts +++ b/packages/feature-ai/src/feature.ts @@ -3,7 +3,7 @@ import { extname, join, relative, sep } from "node:path"; import type { AiProvider, AuthUser, ChatMessage, Feature, JobContext, RequestContext, ServerContext } from "@stc/contracts"; import { canView } from "@stc/core"; import type { LoadedCourse } from "@stc/course-format"; -import { aclRepo, notesRepo } from "@stc/db"; +import { aclRepo, notesRepo, progressRepo } from "@stc/db"; import { buildSystemPrompt, citationsFor, readMentorText } from "./assistant/prompt"; import { combineTools, courseTools, mcpTools, type ToolSet } from "./assistant/tools"; import { runAssistant } from "./assistant/loop"; @@ -138,6 +138,7 @@ export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feat const settings = aiSettingsRepo(ctx.db, createSecretBox(ctx.config.secret)); const acl = aclRepo(ctx.db); const notes = notesRepo(ctx.db); + const progress = progressRepo(ctx.db); const sink = opts.draftSink ?? createFsDraftSink(ctx.config.dataDir); const makeProvider = opts.providerFactory ?? ((cfg: ProviderConfig, key: string | null) => createProvider(cfg, key, opts.fetch)); @@ -236,7 +237,7 @@ export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feat send({ type: "meta", conversationId: convId, mentorOn, mentorAvailable: !!mentorText, source: resolved.source, citations: citationsFor(sources) }); let tools: ToolSet | null = null; try { - const course_ = courseTools({ repo: ctx.courses, course, index, listNotes: (lid) => notes.list(user.id, course.id, lid) }); + const course_ = courseTools({ repo: ctx.courses, course, index, listNotes: (lid) => notes.list(user.id, course.id, lid), getProgress: () => progress.get(user.id, course.id) }); const servers = mcpServersFor(course.id, (await settings.getServer()).mcpServers); const mcp = servers.length ? await mcpTools(servers, { fetch: opts.fetch, timeoutMs: opts.mcpTimeoutMs }) : null; for (const w of mcp?.warnings ?? []) send({ type: "notice", message: w }); diff --git a/packages/feature-ai/test/assistant.test.ts b/packages/feature-ai/test/assistant.test.ts index 7b6b8f0..8b533d4 100644 --- a/packages/feature-ai/test/assistant.test.ts +++ b/packages/feature-ai/test/assistant.test.ts @@ -89,3 +89,16 @@ test("unknown tools and provider errors are reported, and the step limit holds", for await (const e of runAssistant({ provider: broken, messages: [{ role: "user", content: "x" }], tools: combineTools([t]) })) ev2.push(e); expect(ev2).toEqual([{ type: "error", message: "boom" }]); }); + +test("my_progress reports the learner's progress per lesson", async () => { + const t = courseTools({ + repo, course, index, listNotes: async () => [], + getProgress: async () => ({ "m1-intro": { state: "done", positionSeconds: null, updatedAt: "2026-10-30" }, "m1-video": { state: "in-progress", positionSeconds: 75, updatedAt: "2026-10-30" } }), + }); + expect(t.specs.map((s) => s.name)).toContain("my_progress"); + const out = await t.run("my_progress", {}); + expect(out).toContain("Introduction (m1-intro): done"); + expect(out).toContain("Watch the clip (m1-video): in progress, at 1:15"); + expect(out).toContain("Adding numbers (m2-text): not started"); + expect(out).toMatch(/1 of \d+ lessons done/); +}); diff --git a/packages/feature-ai/test/feature.test.ts b/packages/feature-ai/test/feature.test.ts index 6a8c700..8b657c6 100644 --- a/packages/feature-ai/test/feature.test.ts +++ b/packages/feature-ai/test/feature.test.ts @@ -96,6 +96,7 @@ forEachDb("chat cites course files, applies mentor rules, and stores the convers const system = fake.seen[0]!.messages[0]!; expect(system.role).toBe("system"); expect(system.content).toContain("Give hints, never full solutions."); + expect(fake.seen[0]!.tools!.map((t) => t.name)).toEqual(expect.arrayContaining(["search_course", "read_lesson", "list_notes", "my_progress"])); const convs = await (await call("GET", "/api/ai/courses/sample/conversations", "ann")).json(); expect(convs).toHaveLength(1); From 1f111b80db4412818bbdbe1ceab0252869f180ed Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 04:39:58 +0100 Subject: [PATCH 46/62] docs(ai): contract requests; dev fake provider returns a draft file Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- docs/contract-requests/ai.md | 61 ++++++++++++++++++++++++++++++ packages/feature-ai/scripts/dev.ts | 4 ++ 2 files changed, 65 insertions(+) create mode 100644 docs/contract-requests/ai.md diff --git a/docs/contract-requests/ai.md b/docs/contract-requests/ai.md new file mode 100644 index 0000000..222d977 --- /dev/null +++ b/docs/contract-requests/ai.md @@ -0,0 +1,61 @@ +# Contract requests from WP-D (AI) + +Phase 1 worked around each of these inside `packages/feature-ai`. Phase 2 decides. + +## 1. Feature factories in `startServer` (apps/server) + +`aiFeature(ctx, opts?)` needs the `ServerContext` (db, courses, jobs, auth, config), but `startServer(config, features)` builds the +context itself and takes ready-made `Feature` objects. So `apps/server/src/main.ts` is not wired yet (the web side is: +one line in `apps/web/src/features.ts`). Proposal: `startServer(config, factories: ((ctx: ServerContext) => Feature)[])`, +then append `(ctx) => aiFeature(ctx, { draftSink: createDraftWriter(ctx) })`. Jobs must be registered after the +factories run (the job handler `ai.generate` lives on the returned feature). +Local workaround: `packages/feature-ai/scripts/dev.ts` builds the context and composes the feature for the dev server on port 3464. + +## 2. Studio `DraftWriter` as the generator's sink (no adapter needed) + +The studio exports `createDraftWriter(ctx, { identity? })` whose `writeDraftFiles(courseId, authorId, files, message)` +returns `{ branch, commit }` and commits only to `draft/<authorId>`. The AI package's `DraftSink` has the same method +and returns `DraftWriteResult = { location?, branch?, commit? }`, so the studio writer is assignable as is: + +```ts +aiFeature(ctx, { draftSink: createDraftWriter(ctx, { identity: (id) => /* user name + email */ null }) }) +``` + +`generateDraft` then reports `location: "draft/<author> @ <short commit>"` plus `branch` and `commit`; the generate page +says "Committed to your draft branch". Test: `test/generator.test.ts` ("a studio DraftWriter is a DraftSink as is"). +Open point: the studio writer throws 404 for courses it does not manage (overlay courses such as Pure Snake). Phase 2 +can wrap it to fall back to `createFsDraftSink(dataDir)` (drafts under `<dataDir>/drafts/<courseId>/<run>/`), or make +generation for unmanaged courses an explicit error. Until wired, the filesystem sink is the default. + +## 3. Tool-call fields on `ChatMessage` + +`ChatMessage` has `toolCallId` and `name` but no structured slot for an assistant turn with several tool calls or with +thinking blocks. The package encodes one tool call per assistant message (`content` = JSON input) and the Anthropic +provider keeps each response's raw content blocks (thinking with signature, text, tool_use) in memory by tool_use id +and replays them unchanged. Proposal: `ChatMessage.toolCalls?: { id, name, input }[]` and an opaque +`providerData?: unknown` the provider can round-trip. + +## 4. Per-course mentor toggle in `ai_settings` + +The frozen key is `(scope, owner_id)` with one `mentor_off` column, so a per-course toggle cannot be a row. The flag +lives in the user row's `config_json.mentorOff[courseId]`; `mentor_off` stays 0 (reserved for "off everywhere"). +Proposal: add `course_id TEXT NOT NULL DEFAULT ''` to the key, or a small `ai_mentor_prefs(user_id, course_id, off)` table. + +## 5. Citations on `ai_messages` + +`ai_messages` has no column for an assistant message's sources. They are stored as an extra row with role `sources` +(JSON) right after the assistant row and folded back when reading. Proposal: `ai_messages.meta_json TEXT`. + +## 6. Derived text cache layout (shared with WP-B) + +Retrieval reads the importer's layout: +- `<dataDir>/cache/decks/<courseId>/<deckId>/text.json` = `[{ page, text }]` (page images alongside); +- `<dataDir>/cache/transcripts/<courseId>/<lessonId>.json` = `{ lessonId, text, segments: [{ start, end, text }] }` + (skipped when the lesson has a `transcript` file in the course, which the job writes too); +- plus plain `.txt/.md/.vtt/.srt` under `<dataDir>/cache/<courseId>/**` (older layout, kept for hand-made caches). +Proposal: export these paths from `@stc/core` (e.g. `cachePaths(dataDir)`) so both packages share one definition. + +## 7. Learner lesson URL + +Citations carry `lessonId`; the assistant links them to `/c/<courseId>/<lessonId>` (a guess at WP-A's route). Proposal: a `lessonUrl(courseId, lessonId)` +helper in `@stc/contracts/web` so WP-A owns the route shape. diff --git a/packages/feature-ai/scripts/dev.ts b/packages/feature-ai/scripts/dev.ts index 4720fd5..a31fe02 100644 --- a/packages/feature-ai/scripts/dev.ts +++ b/packages/feature-ai/scripts/dev.ts @@ -19,6 +19,10 @@ const fake: AiProvider = { id: "fake", async *chat(messages) { const sys = messages[0]?.content ?? ""; + if (sys.includes("=== FILE:")) { + yield { text: "=== FILE: questions/adding-many-numbers.md ===\n# Check your understanding: adding many numbers\n\n1. Predict: what does `sum([])` return?\n2. What would break if `sum` used `a - b`?\n=== END FILE ===\n" }; + return; + } const mentor = sys.includes("## Course mentor rules"); const answer = mentor ? "Before I explain: what does the failing test give as input, and what did you expect `add` to return?\n\nThe lesson's nudge asks **which operator combines two numbers**. Try it, run the tests, and tell me what happens. [lessons/04-exercise.md · Adding numbers]" From 0ff8e6a5de1e69ed0f004c6c1e943d2cb334e030 Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 04:40:02 +0100 Subject: [PATCH 47/62] docs(ai): mark plan tasks done Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- docs/superpowers/plans/2026-10-30-ai.md | 44 ++++++++++++------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/superpowers/plans/2026-10-30-ai.md b/docs/superpowers/plans/2026-10-30-ai.md index 6578ce5..14c457d 100644 --- a/docs/superpowers/plans/2026-10-30-ai.md +++ b/docs/superpowers/plans/2026-10-30-ai.md @@ -44,8 +44,8 @@ Inherited from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun only, plain - `createSecretBox(secret: string): { encrypt(plain: string): Promise<string>; decrypt(token: string): Promise<string> }`. HKDF-SHA-256 (salt `"stcourses/byok"`, info `"aes-gcm-256/v1"`) → AES-GCM-256; token `v1.<b64 iv>.<b64 ciphertext>`. - `migrationsDirs: string[]` (empty: the package adds no tables). -- [ ] Tests: round trip; two encryptions differ (random IV); a box with another secret rejects; a tampered token rejects. -- [ ] Implement; commit `feat(ai): package scaffold and AES-GCM secret box`. +- [x] Tests: round trip; two encryptions differ (random IV); a box with another secret rejects; a tampered token rejects. +- [x] Implement; commit `feat(ai): package scaffold and AES-GCM secret box`. ### Task 2: Settings repository @@ -59,8 +59,8 @@ Inherited from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun only, plain - `getUser(userId): Promise<{ byok: (ProviderConfig & { hasApiKey }) | null; mentorOff: Record<courseId, boolean> }>`, `saveUserKey(userId, cfg & { apiKey })`, `removeUserKey(userId)`, `userApiKey(userId, kind)`, `setMentorOff(userId, courseId, off)`, `isMentorOff(userId, courseId)`. - Storage: `ai_settings` row `('server','')` holds `{ provider, apiKeyCipher, mcpServers }`; row `('user', userId)` holds `{ byok, mentorOff }`. The per-course flag lives in `config_json` because the frozen primary key is `(scope, owner_id)`; the `mentor_off` column mirrors "off for every course" (always 0 for now). Contract request filed. Keys go in `user_api_keys` encrypted. Upserts are select-then-insert-or-update in a transaction. -- [ ] Tests (both DBs): empty server config is `null` provider; saving a key stores ciphertext only (raw row does not contain the key) and `serverApiKey()` returns it; omitting `apiKey` on save keeps the old key; BYOK save/remove; mentor toggle per course is independent. -- [ ] Implement; commit `feat(ai): settings repository for server config, BYOK keys and mentor toggles`. +- [x] Tests (both DBs): empty server config is `null` provider; saving a key stores ciphertext only (raw row does not contain the key) and `serverApiKey()` returns it; omitting `apiKey` on save keeps the old key; BYOK save/remove; mentor toggle per course is independent. +- [x] Implement; commit `feat(ai): settings repository for server config, BYOK keys and mentor toggles`. ### Task 3: SSE parsing and the Anthropic provider @@ -74,8 +74,8 @@ Inherited from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun only, plain - The provider remembers each response's full content blocks (thinking with signature, text, tool_use) by tool_use id and replays them unchanged for the matching assistant tool-call messages. - Streams `{ text }` per `text_delta`, `{ toolCall }` per finished `tool_use` block; HTTP errors and `error` events throw with the API message; `stop_reason: "refusal"` yields a plain sentence. -- [ ] Tests with a fake `fetch` that returns canned SSE: text streams in order; a tool_use with split `input_json_delta` yields one parsed toolCall; request body has model, system, tools as `input_schema`, no `thinking`/`temperature`; a second call after a tool round replays the thinking block with its signature unchanged before `tool_result`; non-2xx throws with the error message. -- [ ] Implement; commit `feat(ai): Anthropic provider over fetch with streaming tool use`. +- [x] Tests with a fake `fetch` that returns canned SSE: text streams in order; a tool_use with split `input_json_delta` yields one parsed toolCall; request body has model, system, tools as `input_schema`, no `thinking`/`temperature`; a second call after a tool round replays the thinking block with its signature unchanged before `tool_result`; non-2xx throws with the error message. +- [x] Implement; commit `feat(ai): Anthropic provider over fetch with streaming tool use`. ### Task 4: OpenAI-compatible provider @@ -85,8 +85,8 @@ Inherited from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun only, plain - `createOpenAiProvider({ baseUrl, apiKey?, model, fetch? }): AiProvider`. POST `<baseUrl>/chat/completions`, `stream: true`, tools as `{ type: "function", function: { name, description, parameters } }`. Assistant tool-call messages map to `tool_calls`; `tool` messages to `{ role: "tool", tool_call_id }`. Accumulates `delta.tool_calls` by index and emits on finish or `[DONE]`. - `createProvider(cfg: ProviderConfig, apiKey: string | null, fetchImpl?): AiProvider`. -- [ ] Tests: text deltas; fragmented tool-call arguments; no `Authorization` header without a key (Ollama); request mapping of a tool round. -- [ ] Implement; commit `feat(ai): OpenAI-compatible provider (covers Ollama)`. +- [x] Tests: text deltas; fragmented tool-call arguments; no `Authorization` header without a key (Ollama); request mapping of a tool round. +- [x] Implement; commit `feat(ai): OpenAI-compatible provider (covers Ollama)`. ### Task 5: Retrieval @@ -100,8 +100,8 @@ Inherited from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun only, plain - Sources: every `.md` under the course root (depth-limited, skipping `.git`, `node_modules`), manifest code dirs and common code extensions (size cap 200 KB per file, 2 000 files per course), `.md/.txt/.vtt/.srt/.json` text under `<dataDir>/cache/<courseId>/`. Markdown splits at headings; long chunks split at ~1 500 chars. Paths are mapped to lessons through `file`, `transcript` and `hints.file` (anchor heading when it matches). - Never indexed: `browser.solution` files and any path segment `solutions`, `solution`, `reference` under `expansion/`, or `reference/` in Systemic-Design style modules (see `isSolutionPath`). -- [ ] Tests on the `sample` fixture: "sum of its arguments" hits `lessons/04-exercise.md` with lessonId `m2-text` or `m2-ex-add`; `exercises/add/solution.js` never appears; a deck text file in a temp `cache/sample/decks/d1/page-1.txt` is found with kind `deck`; note chunks passed as `extra` rank. -- [ ] Implement; commit `feat(ai): BM25 course retrieval over lessons, transcripts, code, deck text and notes`. +- [x] Tests on the `sample` fixture: "sum of its arguments" hits `lessons/04-exercise.md` with lessonId `m2-text` or `m2-ex-add`; `exercises/add/solution.js` never appears; a deck text file in a temp `cache/sample/decks/d1/page-1.txt` is found with kind `deck`; note chunks passed as `extra` rank. +- [x] Implement; commit `feat(ai): BM25 course retrieval over lessons, transcripts, code, deck text and notes`. ### Task 6: MCP client @@ -112,8 +112,8 @@ Inherited from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun only, plain - JSON-RPC 2.0: `initialize` (protocolVersion `2025-06-18`), `notifications/initialized`, `tools/list`, `tools/call`. stdio = newline-delimited JSON over `Bun.spawn`; streamable HTTP = POST with `Accept: application/json, text/event-stream`, JSON or SSE replies, `Mcp-Session-Id` echoed. - `mcpServersFor(courseId, servers)` selects global servers plus ones listing the course. -- [ ] Tests: the fake server over stdio lists `echo` and `tools/call` returns its text; the same over HTTP (fake via `Bun.serve` on port 0, replying with SSE for `tools/call`) with the session id echoed; a server that exits gives a rejected promise, not a hang. -- [ ] Implement; commit `feat(ai): MCP client over stdio and streamable HTTP`. +- [x] Tests: the fake server over stdio lists `echo` and `tools/call` returns its text; the same over HTTP (fake via `Bun.serve` on port 0, replying with SSE for `tools/call`) with the session id echoed; a server that exits gives a rejected promise, not a hang. +- [x] Implement; commit `feat(ai): MCP client over stdio and streamable HTTP`. ### Task 7: Assistant core @@ -125,8 +125,8 @@ Inherited from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun only, plain - `runAssistant({ provider, messages, tools, signal, maxSteps = 6 }): AsyncIterable<AssistantEvent>` where `AssistantEvent = { type: "text", text } | { type: "tool", name } | { type: "error", message } | { type: "done", text }`. Tool calls become assistant messages with `toolCallId`/`name` and `tool` result messages; unknown tools and tool errors are fed back as error text. - Citations: `citationsFor(chunks): { path, lessonId?, lessonTitle?, kind }[]` deduplicated. -- [ ] Tests: mentor on puts the sample `MENTOR.md` text in the system prompt; mentor off omits it and states full answers; a fake provider that calls `search_course` then answers gets the tool result and the loop ends with text; `read_lesson` on the exercise returns markdown; solution paths are refused. -- [ ] Implement; commit `feat(ai): assistant prompt, course tools and tool loop`. +- [x] Tests: mentor on puts the sample `MENTOR.md` text in the system prompt; mentor off omits it and states full answers; a fake provider that calls `search_course` then answers gets the tool result and the loop ends with text; `read_lesson` on the exercise returns markdown; solution paths are refused. +- [x] Implement; commit `feat(ai): assistant prompt, course tools and tool loop`. ### Task 8: Generator and draft sink @@ -140,8 +140,8 @@ Inherited from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun only, plain - `checkHouseStyle(req, files): string[]` warnings for missing beats/tiers. - `generateDraft({ provider, req, course, ctx, sink, authorId }): Promise<{ location, paths, warnings }>`; every Markdown file starts with a draft marker comment. -- [ ] Tests: prompt for each style contains its beats; parse handles two files and rejects `../x`; fake provider output written by the fs sink lands under `<dataDir>/drafts/sample/…` and nothing changes in the course folder; missing tiers give warnings. -- [ ] Implement; commit `feat(ai): house-style generator writing drafts through a DraftSink`. +- [x] Tests: prompt for each style contains its beats; parse handles two files and rejects `../x`; fake provider output written by the fs sink lands under `<dataDir>/drafts/sample/…` and nothing changes in the course folder; missing tiers give warnings. +- [x] Implement; commit `feat(ai): house-style generator writing drafts through a DraftSink`. ### Task 9: Feature routes and jobs @@ -157,8 +157,8 @@ Inherited from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun only, plain - `GET /api/ai/studio/sources?courseId=`, `POST /api/ai/studio/generate` (enqueues job `ai.generate`), `GET /api/ai/studio/jobs/:id`. - Job `ai.generate` runs `generateDraft` with the requesting user's provider. -- [ ] Tests through `createApp` with a fake `AuthService`: status disabled by default; admin config hides the key; non-owner gets 403; BYOK enables the assistant for that user only; chat with a fake provider streams text and `meta.citations` include `lessons/04-exercise.md`; system prompt holds mentor rules, and not after `PUT mentor {off:true}`; anonymous on a restricted course gets 404; generate as a learner gets 403; the job handler writes drafts under `<dataDir>/drafts/sample`. -- [ ] Implement; commit `feat(ai): HTTP routes, chat streaming and generation job`. +- [x] Tests through `createApp` with a fake `AuthService`: status disabled by default; admin config hides the key; non-owner gets 403; BYOK enables the assistant for that user only; chat with a fake provider streams text and `meta.citations` include `lessons/04-exercise.md`; system prompt holds mentor rules, and not after `PUT mentor {off:true}`; anonymous on a restricted course gets 404; generate as a learner gets 403; the job handler writes drafts under `<dataDir>/drafts/sample`. +- [x] Implement; commit `feat(ai): HTTP routes, chat streaming and generation job`. ### Task 10: Web UI @@ -169,10 +169,10 @@ Inherited from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun only, plain - Admin: provider kind, model, base URL, write-only key, MCP server rows. Account: BYOK form and remove. Generate: course, kind, style, sources checklist, topic, instructions, job progress, list of draft files with location. - `readNdjson(stream)` client helper. -- [ ] Tests: `readNdjson` splits across chunk boundaries; `renderToString` of the banner component shows the exact sentence when off and nothing when on. -- [ ] Implement, check in headless Chrome via `packages/feature-ai/scripts/dev.ts` on port 3464; commit `feat(ai): assistant panel, AI settings and studio generator UI`. +- [x] Tests: `readNdjson` splits across chunk boundaries; `renderToString` of the banner component shows the exact sentence when off and nothing when on. +- [x] Implement, check in headless Chrome via `packages/feature-ai/scripts/dev.ts` on port 3464; commit `feat(ai): assistant panel, AI settings and studio generator UI`. ### Task 11: Contract requests and report -- [ ] `docs/contract-requests/ai.md`: feature factories in `startServer`; tool-call fields on `ChatMessage`; per-course `ai_settings` key; studio `createDraftWriter` wiring; cache layout for deck text and transcripts; learner lesson URL. -- [ ] Run `bun test`, `bun run typecheck`, the Postgres run; write `wp-ai-report.md`. +- [x] `docs/contract-requests/ai.md`: feature factories in `startServer`; tool-call fields on `ChatMessage`; per-course `ai_settings` key; studio `createDraftWriter` wiring; cache layout for deck text and transcripts; learner lesson URL. +- [x] Run `bun test`, `bun run typecheck`, the Postgres run; write `wp-ai-report.md`. From 2d162a428242c3508d2092f57be783f28b9ee7fa Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 04:43:53 +0100 Subject: [PATCH 48/62] fix(studio): review round 1 (symlink-safe writes, remote allowlist, upload cap/types/no-overwrite, owner rule at publish/sync/resolve, studio/repos layout and read-only imports, side-effect-free list) C1: writes lstat every path component and refuse symlinks (dangling included), check the parent's real path after mkdir, and open with O_NOFOLLOW. I1: only https://, ssh:// and git@host: remotes; protocol.file/ext/fd=never on every fetch and push; global/system git config ignored. I2: STC_UPLOAD_MAX_MB cap (default 100) by Content-Length and while streaming; extension allowlist (415); stcourse.yaml refused; 409 unless overwrite=true; sandboxing CSP on draft raw files. I3: owner-only visibility/source.kind rule checked on the merged result at publish and sync, and at resolve; publish also validates the merged tree. I4: bare repos in <dataDir>/studio/repos/<id>.git; imported courses read-only. I5: list status comes from the bare repo; drafts are created lazily. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- docs/contract-requests/studio.md | 19 +- packages/feature-studio/src/index.ts | 6 +- packages/feature-studio/src/server/feature.ts | 99 +++++-- packages/feature-studio/src/server/git.ts | 3 + packages/feature-studio/src/server/paths.ts | 12 +- packages/feature-studio/src/server/store.ts | 200 +++++++++++-- packages/feature-studio/test/app-helpers.ts | 3 +- packages/feature-studio/test/feature.test.ts | 5 +- packages/feature-studio/test/helpers.ts | 4 +- packages/feature-studio/test/publish.test.ts | 15 +- packages/feature-studio/test/security.test.ts | 268 ++++++++++++++++++ packages/feature-studio/test/store.test.ts | 5 +- 12 files changed, 563 insertions(+), 76 deletions(-) create mode 100644 packages/feature-studio/test/security.test.ts diff --git a/docs/contract-requests/studio.md b/docs/contract-requests/studio.md index f5d1045..6fb10c8 100644 --- a/docs/contract-requests/studio.md +++ b/docs/contract-requests/studio.md @@ -12,16 +12,17 @@ Proposal: ## 2. Learners must read studio courses from the published worktree (config / main.ts) -Studio courses live at: -- `<dataDir>/courses/<id>`: a bare repo with no `stcourse.yaml`, so discovery never loads it; -- `<dataDir>/worktrees/<id>/published`: the published branch, which is what learners must see; +Layout after the review (controller ruling on I4): +- `<dataDir>/studio/repos/<id>.git`: studio bare repos. They contain no `stcourse.yaml`, so discovery never loads them. +- `<dataDir>/worktrees/<id>/published`: the published branch, which is what learners read. - `<dataDir>/worktrees/<id>/drafts/<author>`: drafts. They sit at depth 3, so the depth-2 discovery never loads them. +- `<dataDir>/courses/<id>`: the importer's plain folders (WP-B), untouched by the studio. -`createCourseRepository(config.courseDirs)` does not scan `<dataDir>/worktrees`. The package exports `studioCourseDirs(config)`, which appends it. Tests and the dev entry use it. +`createCourseRepository(config.courseDirs)` scans neither `<dataDir>/worktrees` nor `<dataDir>/courses`. The package exports `studioCourseDirs(config)`, which returns the configured dirs plus `<dataDir>/courses` and `<dataDir>/worktrees`. Tests and the dev entry use it. -Proposal: main.ts calls `createCourseRepository(studioCourseDirs(config))`, or `loadConfig` always appends `<dataDir>/worktrees`. +Proposal: main.ts calls `createCourseRepository(studioCourseDirs(config))`, or `loadConfig` appends both dirs. -Related, for Phase 2: the importer writes plain folders to `<dataDir>/courses/<id>`. These coexist with the studio's bare repos. A course id that exists either way cannot be created again in the studio. Adopting an imported course into the studio (git init, move it to the bare/worktree layout) is not part of v1. +Imported courses (`source.kind` other than `authored`, with their root under `<dataDir>/courses`) are read-only in the studio for v1, with the message "Imported course: read-only in the studio for now". There is a `TODO(studio)` in `feature.ts` (`readOnlyMessage`) for converting one into a studio-managed repo. The studio refuses to create a course whose id already exists as `<dataDir>/courses/<id>`. WP-B should likewise refuse an import id that `<dataDir>/studio/repos/<id>.git` already uses. ## 3. Live preview: swap in the learner `LessonView` @@ -80,3 +81,9 @@ Creating a course inserts the creator into `course_authors`. No new tables are a ## 8. External pushes straight into the bare repo (no action needed) The platform reloads when the published worktree's HEAD changes: an external commit in that worktree, a studio publish, or a Sync. A direct `git push` into `<dataDir>/courses/<id>` is refused by git itself ("branch is currently checked out", checked with git 2.47), so the published files cannot silently fall behind `main`. Outside tools should commit in the published worktree, or push to a configured remote that the author then Syncs. + +## 9. Core media route serves SVG/HTML inline (core, Phase 2) + +The studio's draft `raw` route now sends `Content-Security-Policy: …; sandbox` and `X-Content-Type-Options: nosniff`. The learner-facing `/api/media/:courseId/*` in `packages/core` serves author-supplied SVG inline from the app's origin and sets neither header. + +Proposal: add the same two headers in `serveMedia`. diff --git a/packages/feature-studio/src/index.ts b/packages/feature-studio/src/index.ts index 1cf5e9b..6ced9bb 100644 --- a/packages/feature-studio/src/index.ts +++ b/packages/feature-studio/src/index.ts @@ -1,9 +1,9 @@ // Server entry of @stc/feature-studio. The browser imports "@stc/feature-studio/web" instead (no server code). -export { studioFeature, OUTSIDE_MESSAGE, type StudioFeature, type StudioFeatureOptions } from "./server/feature"; +export { studioFeature, OUTSIDE_MESSAGE, IMPORTED_MESSAGE, uploadMaxBytes, UPLOAD_EXTENSIONS, type StudioFeature, type StudioFeatureOptions } from "./server/feature"; export { createDraftWriter, type DraftWriter, type DraftWriterOptions } from "./server/draft-writer"; -export { createStudioStore, contentHash, type StudioStore, type Identity, type DraftStatus, type PublishResult, type SyncResult } from "./server/store"; +export { createStudioStore, contentHash, isAllowedRemote, ownerRequiredFor, type StudioStore, type Identity, type DraftStatus, type PublishResult, type SyncResult } from "./server/store"; export { createPublishedWatcher, type PublishedWatcher } from "./server/watch"; -export { studioCourseDirs, studioWorktreesDir } from "./server/paths"; +export { studioCourseDirs, studioWorktreesDir, studioReposDir } from "./server/paths"; export { studioWeb } from "./web"; /** The studio adds no tables: authors use Phase 0's `course_authors`. */ diff --git a/packages/feature-studio/src/server/feature.ts b/packages/feature-studio/src/server/feature.ts index 0460772..e691487 100644 --- a/packages/feature-studio/src/server/feature.ts +++ b/packages/feature-studio/src/server/feature.ts @@ -1,15 +1,48 @@ -import { basename, resolve, sep } from "node:path"; +import { existsSync, realpathSync } from "node:fs"; +import { basename, extname, resolve, sep } from "node:path"; import type { AuthUser, Feature, RequestContext, RouteHandler, ServerContext } from "@stc/contracts"; import { serveMedia } from "@stc/core"; import { ManifestSchema, parseManifest, resolveInside, type LoadedCourse } from "@stc/course-format"; import { aclRepo } from "@stc/db"; import { createDraftWriter, type DraftWriter } from "./draft-writer"; import { studioWorktreesDir } from "./paths"; -import { checkRelPath, createStudioStore, MANIFEST, StudioError, type FileWrite, type StudioStore } from "./store"; +import { checkRelPath, createStudioStore, MANIFEST, ownerRequiredFor, StudioError, type FileWrite, type StudioStore } from "./store"; import { createPublishedWatcher, type PublishedWatcher } from "./watch"; export const OUTSIDE_MESSAGE = "This course is managed outside STCourses"; -const MAX_UPLOAD = 2 * 1024 * 1024 * 1024; // 2 GB; large videos belong in the media dir, not git, but we do not forbid them +export const IMPORTED_MESSAGE = "Imported course: read-only in the studio for now"; + +/** Upload cap in bytes: STC_UPLOAD_MAX_MB, default 100 (review I2). */ +export function uploadMaxBytes(env: Record<string, string | undefined> = process.env): number { + const mb = Number(env.STC_UPLOAD_MAX_MB ?? "100"); + return (Number.isFinite(mb) && mb > 0 ? mb : 100) * 1024 * 1024; +} + +/** Server-side allowlist by extension for uploads: text, documents, images, media, archives and code. */ +export const UPLOAD_EXTENSIONS = new Set([ + "md", "markdown", "txt", "pdf", "png", "jpg", "jpeg", "gif", "svg", "webp", "mp4", "webm", "mp3", "vtt", "srt", "zip", + "js", "mjs", "cjs", "jsx", "ts", "tsx", "json", "yaml", "yml", "py", "sh", "css", "html", "csv", "toml", "sql", + "scala", "sc", "fs", "fsx", "fsproj", "rs", "go", "java", "kt", "c", "h", "cpp", "hpp", "cs", "rb", "php", "swift", "hs", "ex", "exs", "dockerfile", +]); + +/** Reads at most `max` bytes of the body; null when it is larger (checked while streaming, not after buffering). */ +async function readCapped(req: Request, max: number): Promise<Uint8Array<ArrayBuffer> | null> { + const reader = req.body?.getReader(); + if (!reader) return new Uint8Array(); + const chunks: Uint8Array[] = []; + let total = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > max) { await reader.cancel(); return null; } + chunks.push(value); + } + const out = new Uint8Array(total); + let off = 0; + for (const c of chunks) { out.set(c, off); off += c.byteLength; } + return out; +} export interface StudioFeature extends Feature { store: StudioStore; @@ -51,7 +84,19 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = { (await ctx.db.sql`SELECT course_id FROM course_authors WHERE user_id = ${u.id}`).map((r: { course_id: string }) => r.course_id); const canEdit = async (u: AuthUser, courseId: string) => hasRole(u) || (await acl.isAuthor(courseId, u.id)); const isStudioCourse = (c: LoadedCourse) => resolve(c.manifestPath).startsWith(wtDir + sep); - const who = (u: AuthUser) => ({ id: u.id, name: u.name || u.email || u.id, email: u.email || `${u.id}@users.stcourses.invalid` }); + const who = (u: AuthUser) => ({ id: u.id, name: u.name || u.email || u.id, email: u.email || `${u.id}@users.stcourses.invalid`, owner: u.roles.includes("owner") }); + const importedRoot = (() => { const d = resolve(ctx.config.dataDir, "courses"); return existsSync(d) ? realpathSync(d) : d; })(); + /** Why a course that is not studio-managed is read-only here. */ + const readOnlyMessage = (c: LoadedCourse): string => { + const kind = c.manifest?.source?.kind; + const underImports = c.root === importedRoot || c.root.startsWith(importedRoot + sep); + if (kind && kind !== "authored" && underImports) { + // TODO(studio): convert imported course to a studio-managed repo (git init into <dataDir>/studio/repos/<id>.git + // plus a published worktree), so authors can edit it and the owner-only visibility rule applies on publish. + return IMPORTED_MESSAGE; + } + return OUTSIDE_MESSAGE; + }; const afterPublishedChange = async (courseId: string) => { await ctx.courses.reload(courseId); @@ -80,7 +125,8 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = { const courseId = rc.params.courseId ?? ""; if (!(await canEdit(user, courseId))) return fail(403, "you are not an author of this course"); if (!store.isManaged(courseId)) { - return ctx.courses.get(courseId) ? fail(403, OUTSIDE_MESSAGE) : fail(404, "no such course"); + const c = ctx.courses.get(courseId); + return c ? fail(403, readOnlyMessage(c)) : fail(404, "no such course"); } return h(req, rc, user, courseId); }); @@ -120,7 +166,7 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = { for (const id of managed) { if (!all && !mine.has(id)) continue; const loaded = ctx.courses.get(id); - const st = await store.status(id, user.id).catch(() => null); + const st = await store.summary(id, user.id).catch(() => null); // read-only: no worktree is created here courses.push({ id, managed: true, title: loaded?.manifest?.title ?? id, @@ -133,7 +179,7 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = { if (all) { for (const c of ctx.courses.list()) { if (isStudioCourse(c) || managed.includes(c.id)) continue; - courses.push({ id: c.id, managed: false, title: c.manifest?.title ?? c.id, pitch: c.manifest?.pitch ?? null, accent: c.manifest?.accent ?? null, publishedStatus: c.status, message: OUTSIDE_MESSAGE }); + courses.push({ id: c.id, managed: false, title: c.manifest?.title ?? c.id, pitch: c.manifest?.pitch ?? null, accent: c.manifest?.accent ?? null, publishedStatus: c.status, message: readOnlyMessage(c) }); } } return json({ courses, canCreate: all }); @@ -157,7 +203,7 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = { if (!store.isManaged(courseId)) { const c = ctx.courses.get(courseId); if (!c) return fail(404, "no such course"); - return json({ readOnly: true, message: OUTSIDE_MESSAGE, course: { id: c.id, title: c.manifest?.title ?? c.id, managed: false, root: c.root }, diagnostics: c.diagnostics }); + return json({ readOnly: true, message: readOnlyMessage(c), course: { id: c.id, title: c.manifest?.title ?? c.id, managed: false, root: c.root }, diagnostics: c.diagnostics }); } return json(await courseState(courseId, user)); }), @@ -179,11 +225,10 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = { files.unshift({ path: MANIFEST, content: manifestToYaml(b.manifest), baseHash: b.manifestBaseHash }); } const mf = files.find((f) => checkRelPath(f.path) === MANIFEST); - if (mf && typeof mf.content === "string") { - const parsed = parseManifest(mf.content); - if (parsed.ok && parsed.manifest.source && parsed.manifest.source.kind !== "authored" && parsed.manifest.visibility !== "restricted" && !user.roles.includes("owner")) { - return fail(403, "only the owner can make an imported course public or signed-in visible"); - } + if (mf && typeof mf.content === "string" && !user.roles.includes("owner")) { + // Early feedback only; publish, sync and resolve enforce the same rule on the merged result. + const needOwner = ownerRequiredFor(await store.publishedManifest(courseId), mf.content); + if (needOwner) return fail(403, needOwner); } const r = await store.saveFiles(courseId, who(user), files, b.message ?? ""); if (!r.ok) return json({ ok: false, reason: "conflict", conflicts: r.conflicts }, 409); @@ -205,18 +250,27 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = { "/api/studio/courses/:courseId/upload": { POST: course(async (req, _rc, user, courseId) => { + const max = uploadMaxBytes(); + const declared = Number(req.headers.get("content-length") ?? "0"); + if (declared > max) return fail(413, `file too large (limit ${Math.round(max / 1048576)} MB)`); + const bytes = await readCapped(req, max); + if (!bytes) return fail(413, `file too large (limit ${Math.round(max / 1048576)} MB)`); let form: FormData; - try { form = await req.formData(); } catch { return fail(400, "expected multipart form data"); } + try { form = await new Response(bytes, { headers: { "content-type": req.headers.get("content-type") ?? "" } }).formData(); } catch { return fail(400, "expected multipart form data"); } const file = form.get("file"); if (!(file instanceof Blob)) return fail(400, "file is required"); - if (file.size > MAX_UPLOAD) return fail(413, "file too large"); const explicit = form.get("path"); const dir = String(form.get("dir") ?? "").replace(/^\/+|\/+$/g, ""); const name = basename((file as File).name || "upload.bin"); const path = checkRelPath(typeof explicit === "string" && explicit ? explicit : dir ? `${dir}/${name}` : name); + if (path === MANIFEST) return fail(400, "edit stcourse.yaml in the manifest editor, not by upload"); + const ext = (extname(path).slice(1) || basename(path)).toLowerCase(); + if (!UPLOAD_EXTENSIONS.has(ext)) return fail(415, `file type .${ext} is not accepted`); + const overwrite = String(form.get("overwrite") ?? "") === "true"; const message = String(form.get("message") ?? "") || `Upload ${path}`; - const r = await store.saveFiles(courseId, who(user), [{ path, content: new Uint8Array(await file.arrayBuffer()) }], message); - if (!r.ok) return json({ ok: false, reason: "conflict", conflicts: r.conflicts }, 409); + // baseHash null = the file must not exist yet; replacing needs an explicit overwrite=true + const r = await store.saveFiles(courseId, who(user), [{ path, content: new Uint8Array(await file.arrayBuffer()), ...(overwrite ? {} : { baseHash: null }) }], message); + if (!r.ok) return json({ ok: false, reason: "exists", error: `${path} already exists; upload again with overwrite to replace it`, conflicts: r.conflicts }, 409); return json({ ok: true, path, commit: r.commit }); }), }, @@ -229,7 +283,12 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = { const dir = await store.draftDir(courseId, user.id); try { checkRelPath(rel); } catch { return fail(404, "not found"); } const abs = resolveInside(dir, rel); - return abs ? serveMedia(req, abs, { download: rc.url.searchParams.has("download") }) : fail(404, "not found"); + if (!abs) return fail(404, "not found"); + const res = await serveMedia(req, abs, { download: rc.url.searchParams.has("download") }); + // Draft files are author content served from the app's origin: never let them run script (SVG, HTML). + res.headers.set("Content-Security-Policy", "default-src 'none'; img-src 'self' data:; style-src 'unsafe-inline'; media-src 'self'; sandbox"); + res.headers.set("X-Content-Type-Options", "nosniff"); + return res; }), }, @@ -240,7 +299,7 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = { await afterPublishedChange(courseId); return json(r); } - return json(r, r.reason === "invalid" ? 422 : 409); + return json(r, r.reason === "invalid" ? 422 : r.reason === "forbidden" ? 403 : 409); }), }, @@ -286,7 +345,7 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = { await afterPublishedChange(courseId); return json(r); } - return json(r, r.reason === "no-remote" ? 400 : r.reason === "network" ? 502 : 409); + return json(r, r.reason === "no-remote" ? 400 : r.reason === "network" ? 502 : r.reason === "forbidden" ? 403 : 409); }), }, }; diff --git a/packages/feature-studio/src/server/git.ts b/packages/feature-studio/src/server/git.ts index ee8d222..68a2895 100644 --- a/packages/feature-studio/src/server/git.ts +++ b/packages/feature-studio/src/server/git.ts @@ -20,6 +20,9 @@ function env(who?: Identity): Record<string, string> { if (v !== undefined && !k.startsWith("GIT_")) e[k] = v; // never inherit GIT_DIR, GIT_INDEX_FILE, ... } e.GIT_TERMINAL_PROMPT = "0"; + // Ignore the machine's global/system git config (url.*.insteadOf, credential helpers, ...). + e.GIT_CONFIG_GLOBAL = "/dev/null"; + e.GIT_CONFIG_NOSYSTEM = "1"; e.GIT_SSH_COMMAND = "ssh -o BatchMode=yes"; // A fallback identity so git never refuses a merge/commit; real saves always pass `who`. const id = who ?? { name: "STCourses", email: "studio@stcourses.invalid" }; diff --git a/packages/feature-studio/src/server/paths.ts b/packages/feature-studio/src/server/paths.ts index f95b9f0..4f954bc 100644 --- a/packages/feature-studio/src/server/paths.ts +++ b/packages/feature-studio/src/server/paths.ts @@ -6,12 +6,18 @@ export function studioWorktreesDir(dataDir: string): string { return join(dataDir, "worktrees"); } +/** Studio bare repos: `<dataDir>/studio/repos/<id>.git` (no manifest, never discovered). */ +export function studioReposDir(dataDir: string): string { + return join(dataDir, "studio", "repos"); +} + /** - * The course dirs the CourseRepository must scan so studio courses load from their published worktree. + * The course dirs the CourseRepository must scan: the configured ones, the importer's plain folders in + * `<dataDir>/courses`, and studio courses through their published worktree in `<dataDir>/worktrees`. * Phase 0's server builds the repository from `config.courseDirs` only; see docs/contract-requests/studio.md. */ export function studioCourseDirs(config: Pick<AppConfig, "dataDir" | "courseDirs">): string[] { - const wt = resolve(studioWorktreesDir(config.dataDir)); const dirs = config.courseDirs.map((d) => resolve(d)); - return dirs.includes(wt) ? dirs : [...dirs, wt]; + for (const d of [resolve(config.dataDir, "courses"), resolve(studioWorktreesDir(config.dataDir))]) if (!dirs.includes(d)) dirs.push(d); + return dirs; } diff --git a/packages/feature-studio/src/server/store.ts b/packages/feature-studio/src/server/store.ts index cb63def..6d1c3fe 100644 --- a/packages/feature-studio/src/server/store.ts +++ b/packages/feature-studio/src/server/store.ts @@ -1,5 +1,5 @@ -import { existsSync, mkdirSync, readdirSync, realpathSync, rmSync, statSync } from "node:fs"; -import { mkdir, rm, unlink, writeFile } from "node:fs/promises"; +import { constants, existsSync, lstatSync, mkdirSync, readdirSync, realpathSync, rmSync, statSync } from "node:fs"; +import { mkdir, open, rm, unlink } from "node:fs/promises"; import { dirname, isAbsolute, join, normalize, sep } from "node:path"; import { loadCourse, type Diagnostic } from "@stc/course-format"; import { git, gitOk, splitZ, withLock, type Identity } from "./git"; @@ -63,6 +63,7 @@ export type PublishResult = | { ok: false; reason: "conflict"; conflicts: string[] } | { ok: false; reason: "dirty"; files: string[] } | { ok: false; reason: "invalid"; diagnostics: Diagnostic[] } + | { ok: false; reason: "forbidden"; message: string } | { ok: false; reason: "merging" }; export type MergeResult = { ok: true; commit: string } | { ok: false; reason: "conflict"; conflicts: string[] } | { ok: false; reason: "merging" | "no-remote" | "dirty"; message: string }; @@ -72,25 +73,75 @@ export interface ConflictFile { path: string; base: string | null; ours: string export type SyncResult = | { ok: true; pulled: boolean; pushed: boolean; head: string } | { ok: false; reason: "no-remote" | "dirty" | "network"; message: string } - | { ok: false; reason: "conflict"; conflicts: string[] }; + | { ok: false; reason: "conflict"; conflicts: string[] } + | { ok: false; reason: "forbidden"; message: string }; export class StudioError extends Error { constructor(message: string, public status = 400) { super(message); } } -export interface StudioStoreOptions { dataDir: string; publishedBranch?: string } +const HOST = String.raw`[A-Za-z0-9](?:[A-Za-z0-9.-]*[A-Za-z0-9])?`; +const USERINFO = String.raw`(?:[^@/\s:]+(?::[^@/\s]*)?@)?`; +const REMOTE_RES = [ + new RegExp(String.raw`^https://${USERINFO}${HOST}(?::\d+)?/[^\s]*$`), + new RegExp(String.raw`^ssh://${USERINFO}${HOST}(?::\d+)?/[^\s]*$`), + new RegExp(String.raw`^git@${HOST}:[A-Za-z0-9_~.][^\s]*$`), +]; + +/** Only https://, ssh:// and git@host: remotes. Local paths, file://, ext::, fd:: and the rest are refused. */ +export function isAllowedRemote(url: string): boolean { + return REMOTE_RES.some((re) => re.test(url)); +} + +/** Transport hardening passed to every fetch/push, on top of the allowlist (defence in depth). */ +const REMOTE_GUARD = ["-c", "protocol.file.allow=never", "-c", "protocol.ext.allow=never", "-c", "protocol.fd.allow=never"]; + +type ManifestLike = { visibility?: string; source?: { kind?: string } } | null; +const kindOf = (m: ManifestLike) => m?.source?.kind ?? "authored"; +function parseLoose(text: string | null): ManifestLike { + if (text === null) return null; + try { return (Bun.YAML.parse(text) as ManifestLike) ?? null; } catch { return null; } +} + +/** + * Spec 12: only the owner may make an imported course public or signed-in, and (to close the obvious bypass) + * only the owner may change `source.kind`. Returns a reason when `resultText` needs the owner, else null. + */ +export function ownerRequiredFor(baseText: string | null, resultText: string | null): string | null { + const base = parseLoose(baseText); + const result = parseLoose(resultText); + if (!result) return null; + if (base && kindOf(base) !== kindOf(result)) return "only the owner can change source.kind"; + const vis = result.visibility ?? "restricted"; + if ((vis === "public" || vis === "signed-in") && (kindOf(result) !== "authored" || (base && kindOf(base) !== "authored"))) { + return "only the owner can make an imported course public or signed-in visible"; + } + return null; +} + +export type Actor = Identity & { id: string; owner?: boolean }; + +export interface StudioStoreOptions { + dataDir: string; + publishedBranch?: string; + /** Tests only: allow local-path remotes (the HTTP feature never sets this). */ + allowLocalRemotes?: boolean; +} export function createStudioStore(opts: StudioStoreOptions) { const dataDir = opts.dataDir; const main = opts.publishedBranch ?? "main"; - const coursesDir = join(dataDir, "courses"); + // Studio repos live apart from the importer's plain folders in <dataDir>/courses (controller ruling, review I4). + const reposDir = join(dataDir, "studio", "repos"); + const importedDir = join(dataDir, "courses"); const worktreesDir = join(dataDir, "worktrees"); + const remoteGuard = opts.allowLocalRemotes ? ["-c", "protocol.ext.allow=never", "-c", "protocol.fd.allow=never"] : REMOTE_GUARD; const paths = (courseId: string) => { if (!ID_RE.test(courseId)) throw new StudioError(`invalid course id: ${courseId}`, 404); const root = join(worktreesDir, courseId); return { - bare: join(coursesDir, courseId), + bare: join(reposDir, `${courseId}.git`), root, published: join(root, "published"), draftsDir: join(root, "drafts"), @@ -120,8 +171,46 @@ export function createStudioStore(opts: StudioStoreOptions) { const r = await git(bare, ["remote", "get-url", "origin"]); return r.code === 0 ? r.stdout.trim() : null; }; + const remoteOk = (url: string) => opts.allowLocalRemotes ? !/^\s*-/.test(url) && !/^(ext|fd)::/.test(url) : isAllowedRemote(url); + /** The configured remote, re-checked on every use (it could have been set outside the studio). */ + const usableRemote = async (bare: string): Promise<{ ok: true } | { ok: false; message: string }> => { + const url = await remoteUrl(bare); + if (!url) return { ok: false, message: "no remote is configured for this course" }; + if (!remoteOk(url)) return { ok: false, message: "the configured remote is not an https://, ssh:// or git@host: URL" }; + return { ok: true }; + }; + const showText = async (cwd: string, spec: string): Promise<string | null> => { + const r = await git(cwd, ["show", spec]); + return r.code === 0 ? r.stdout : null; + }; - /** Absolute path for `rel` inside `root`; also refuses when an existing parent is a symlink out of the root. */ + const lexists = (p: string) => { try { lstatSync(p); return true; } catch { return false; } }; + + /** + * For writes (review C1): refuses when any existing component of `rel`, the final one included, is a symlink + * (lstat, so dangling links count), creates missing parents, then re-checks the parent's real path is inside. + */ + const safeWritePath = async (root: string, rel: string, opts: { allowFinalSymlink?: boolean } = {}): Promise<string> => { + const n = checkRelPath(rel); + const realRoot = realpathSync(root); + const parts = n.split(sep); + let cur = realRoot; + for (let i = 0; i < parts.length; i++) { + cur = join(cur, parts[i]!); + if (!lexists(cur)) break; + const st = lstatSync(cur); + const last = i === parts.length - 1; + if (st.isSymbolicLink() && !(last && opts.allowFinalSymlink)) throw new Error(`invalid path (symlink): ${rel}`); + if (!last && !st.isDirectory()) throw new Error(`invalid path (not a directory): ${rel}`); + } + const abs = join(realRoot, n); + await mkdir(dirname(abs), { recursive: true }); + const realParent = realpathSync(dirname(abs)); + if (realParent !== realRoot && !realParent.startsWith(realRoot + sep)) throw new Error(`invalid path (escapes the course): ${rel}`); + return abs; + }; + + /** For reads: absolute path for `rel` whose real path (symlinks followed) stays inside `root`. */ const inside = (root: string, rel: string): string => { const n = checkRelPath(rel); const abs = join(root, n); @@ -167,12 +256,15 @@ export function createStudioStore(opts: StudioStoreOptions) { async function writeFiles(dir: string, files: FileWrite[]): Promise<string[]> { const touched: string[] = []; for (const f of files) { - const abs = inside(dir, f.path); if (f.delete) { - if (existsSync(abs)) await unlink(abs); + // unlink removes a symlink itself, never its target, so deleting a planted link is allowed + const abs = await safeWritePath(dir, f.path, { allowFinalSymlink: true }); + if (lexists(abs)) await unlink(abs); } else { - await mkdir(dirname(abs), { recursive: true }); - await writeFile(abs, f.content ?? ""); + const abs = await safeWritePath(dir, f.path); + // O_NOFOLLOW: a symlink swapped in after the check makes the open fail instead of writing through it + const fh = await open(abs, constants.O_WRONLY | constants.O_CREAT | constants.O_TRUNC | constants.O_NOFOLLOW, 0o644); + try { await fh.writeFile(f.content ?? ""); } finally { await fh.close(); } } touched.push(checkRelPath(f.path)); } @@ -187,8 +279,8 @@ export function createStudioStore(opts: StudioStoreOptions) { isManaged, async listManaged(): Promise<string[]> { - if (!existsSync(coursesDir)) return []; - return readdirSync(coursesDir).filter((id) => ID_RE.test(id) && isManaged(id)).sort(); + if (!existsSync(reposDir)) return []; + return readdirSync(reposDir).filter((n) => n.endsWith(".git")).map((n) => n.slice(0, -4)).filter((id) => ID_RE.test(id) && isManaged(id)).sort(); }, async createCourse(input: { id: string; title: string; pitch?: string }, who: Identity): Promise<void> { @@ -196,9 +288,9 @@ export function createStudioStore(opts: StudioStoreOptions) { if (!ID_RE.test(id)) throw new StudioError(`invalid course id "${id}": use lowercase letters, digits and dashes`); if (!title.trim()) throw new StudioError("title is required"); const p = paths(id); - if (existsSync(p.bare) || existsSync(p.root)) throw new StudioError(`course "${id}" already exists`, 409); await lock(id, async () => { - mkdirSync(coursesDir, { recursive: true }); + if (existsSync(p.bare) || existsSync(p.root) || existsSync(join(importedDir, id))) throw new StudioError(`course "${id}" already exists`, 409); + mkdirSync(reposDir, { recursive: true }); const tmp = join(dataDir, "tmp", `new-${id}-${Date.now()}`); mkdirSync(join(tmp, "lessons"), { recursive: true }); try { @@ -244,7 +336,7 @@ export function createStudioStore(opts: StudioStoreOptions) { draftDir: (courseId: string, authorId: string) => api.ensureDraft(courseId, authorId), - async saveFiles(courseId: string, who: Identity & { id: string }, files: FileWrite[], message: string): Promise<SaveResult> { + async saveFiles(courseId: string, who: Actor, files: FileWrite[], message: string): Promise<SaveResult> { for (const f of files) checkRelPath(f.path); if (files.length === 0) throw new StudioError("nothing to save"); return lock(courseId, async () => { @@ -281,6 +373,26 @@ export function createStudioStore(opts: StudioStoreOptions) { }; }, + /** List view status, read from the bare repo only: never creates a worktree or branch (review I5). */ + async summary(courseId: string, authorId: string): Promise<{ hasDraft: boolean; ahead: number; behind: number; merging: boolean; remote: string | null }> { + const p = requireManaged(courseId); + const branch = branchOf(authorId); + const hasDraft = (await git(p.bare, ["rev-parse", "-q", "--verify", `refs/heads/${branch}`])).code === 0; + const dir = p.draft(authorId); + return { + hasDraft, + ahead: hasDraft ? await count(p.bare, `${main}..${branch}`) : 0, + behind: hasDraft ? await count(p.bare, `${branch}..${main}`) : 0, + merging: existsSync(join(dir, ".git")) ? await isMerging(dir) : false, + remote: await remoteUrl(p.bare), + }; + }, + + /** The published manifest text (base for the owner-only rule). */ + async publishedManifest(courseId: string): Promise<string | null> { + return showText(requireManaged(courseId).bare, `${main}:${MANIFEST}`); + }, + async publishedHead(courseId: string): Promise<string> { return rev(requireManaged(courseId).published, "HEAD"); }, @@ -290,7 +402,7 @@ export function createStudioStore(opts: StudioStoreOptions) { return api.validateDraftUnlocked(courseId, await api.ensureDraft(courseId, authorId)); }, - publish(courseId: string, who: Identity & { id: string }): Promise<PublishResult> { + publish(courseId: string, who: Actor): Promise<PublishResult> { return lock(courseId, async () => { const p = requireManaged(courseId); const dir = await ensureDraftUnlocked(courseId, who.id); @@ -301,13 +413,27 @@ export function createStudioStore(opts: StudioStoreOptions) { const diags = (await api.validateDraftUnlocked(courseId, dir)).filter((d) => d.level === "error"); if (diags.length) return { ok: false, reason: "invalid", diagnostics: diags }; if ((await count(p.bare, `${main}..${branch}`)) === 0) return { ok: true, commit: await rev(p.published, "HEAD"), published: false }; - const r = await git(p.published, ["merge", "--no-ff", "-m", `Publish ${branch}`, branch], who); + const baseManifest = await showText(p.published, `HEAD:${MANIFEST}`); + // Merge without committing, so the merged result is checked before it becomes the published version. + const r = await git(p.published, ["merge", "--no-ff", "--no-commit", branch], who); if (r.code !== 0) { const conflicts = await unmerged(p.published); await git(p.published, ["merge", "--abort"]); if (!conflicts.length) throw new StudioError(`publish failed: ${r.stderr.trim() || r.stdout.trim()}`, 500); return { ok: false, reason: "conflict", conflicts }; } + const merged = (await readFileIn(p.published, MANIFEST))?.content ?? null; + const needOwner = ownerRequiredFor(baseManifest, merged); + if (needOwner && !who.owner) { + await git(p.published, ["merge", "--abort"]); + return { ok: false, reason: "forbidden", message: needOwner }; + } + const mergedErrors = (await api.validateDraftUnlocked(courseId, p.published)).filter((d) => d.level === "error"); + if (mergedErrors.length) { + await git(p.published, ["merge", "--abort"]); + return { ok: false, reason: "invalid", diagnostics: mergedErrors }; + } + await gitOk(p.published, ["commit", "-q", "-m", `Publish ${branch}`], who); // Bring the draft level with what was just published, so "behind" stays meaningful. await git(dir, ["merge", "-q", "--ff-only", main]); return { ok: true, commit: await rev(p.published, "HEAD"), published: true }; @@ -323,15 +449,16 @@ export function createStudioStore(opts: StudioStoreOptions) { }, /** Merges the published branch (or the remote's) into the author's draft. Conflicts stay in the draft for the author to resolve. */ - updateDraft(courseId: string, who: Identity & { id: string }, from: "published" | "remote" = "published"): Promise<MergeResult> { + updateDraft(courseId: string, who: Actor, from: "published" | "remote" = "published"): Promise<MergeResult> { return lock(courseId, async () => { const p = requireManaged(courseId); const dir = await ensureDraftUnlocked(courseId, who.id); if (await isMerging(dir)) return { ok: false, reason: "merging", message: "a merge is already in progress" }; let target = main; if (from === "remote") { - if (!(await remoteUrl(p.bare))) return { ok: false, reason: "no-remote", message: "no remote is configured" }; - const f = await git(p.bare, ["fetch", "-q", "origin"]); + const ru = await usableRemote(p.bare); + if (!ru.ok) return { ok: false, reason: "no-remote", message: ru.message }; + const f = await git(p.bare, [...remoteGuard, "fetch", "-q", "origin"]); if (f.code !== 0) return { ok: false, reason: "no-remote", message: f.stderr.trim() }; target = `origin/${main}`; if ((await git(p.bare, ["rev-parse", "-q", "--verify", `refs/remotes/${target}`])).code !== 0) return { ok: true, commit: await rev(dir, "HEAD") }; @@ -364,10 +491,15 @@ export function createStudioStore(opts: StudioStoreOptions) { }, /** Writes the author's resolution (`content: null` deletes) and commits the merge once nothing is unmerged. */ - resolve(courseId: string, who: Identity & { id: string }, files: { path: string; content: string | null }[]): Promise<{ ok: true; remaining: string[]; commit: string | null }> { + resolve(courseId: string, who: Actor, files: { path: string; content: string | null }[]): Promise<{ ok: true; remaining: string[]; commit: string | null }> { return lock(courseId, async () => { const dir = await ensureDraftUnlocked(courseId, who.id); if (!(await isMerging(dir))) throw new StudioError("no merge in progress", 409); + const m = files.find((f) => checkRelPath(f.path) === MANIFEST); + if (m && !who.owner) { + const needOwner = ownerRequiredFor(await showText(requireManaged(courseId).bare, `${main}:${MANIFEST}`), m.content); + if (needOwner) throw new StudioError(needOwner, 403); + } const touched = await writeFiles(dir, files.map((f) => (f.content === null ? { path: f.path, delete: true } : { path: f.path, content: f.content }))); await gitOk(dir, ["add", "-A", "--", ...touched]); const remaining = await unmerged(dir); @@ -396,7 +528,7 @@ export function createStudioStore(opts: StudioStoreOptions) { if (cur) await gitOk(p.bare, ["remote", "remove", "origin"]); return; } - if (/^\s*-/.test(url) || /\s/.test(url.trim())) throw new StudioError("invalid remote url"); + if (!remoteOk(url.trim())) throw new StudioError("remote must be an https://, ssh:// or git@host: URL"); await gitOk(p.bare, cur ? ["remote", "set-url", "origin", url.trim()] : ["remote", "add", "origin", url.trim()]); // Bare repos created with `init --bare` have no fetch refspec for remote-tracking branches by default. await gitOk(p.bare, ["config", "remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*"]); @@ -407,13 +539,14 @@ export function createStudioStore(opts: StudioStoreOptions) { * Explicit Sync with the remote: pull (fast-forward, or a merge without conflicts) and push the published branch. * A conflicting divergence is aborted and reported; the author merges the remote into the draft to resolve it. */ - sync(courseId: string, who: Identity): Promise<SyncResult> { + sync(courseId: string, who: Actor): Promise<SyncResult> { return lock(courseId, async () => { const p = requireManaged(courseId); - if (!(await remoteUrl(p.bare))) return { ok: false, reason: "no-remote", message: "no remote is configured for this course" }; + const ru = await usableRemote(p.bare); + if (!ru.ok) return { ok: false, reason: "no-remote", message: ru.message }; const dirty = await dirtyFiles(p.published); if (dirty.length) return { ok: false, reason: "dirty", message: `uncommitted changes in the published worktree: ${dirty.join(", ")}` }; - const f = await git(p.bare, ["fetch", "-q", "origin"]); + const f = await git(p.bare, [...remoteGuard, "fetch", "-q", "origin"]); if (f.code !== 0) return { ok: false, reason: "network", message: f.stderr.trim() || "fetch failed" }; const remoteRef = `refs/remotes/origin/${main}`; const hasRemote = (await git(p.bare, ["rev-parse", "-q", "--verify", remoteRef])).code === 0; @@ -425,25 +558,34 @@ export function createStudioStore(opts: StudioStoreOptions) { if (local !== remote) { const localBehind = (await git(p.bare, ["merge-base", "--is-ancestor", local, remote])).code === 0; const remoteBehind = (await git(p.bare, ["merge-base", "--is-ancestor", remote, local])).code === 0; + const baseManifest = await showText(p.bare, `${main}:${MANIFEST}`); if (localBehind) { + const needOwner = ownerRequiredFor(baseManifest, await showText(p.bare, `${remoteRef}:${MANIFEST}`)); + if (needOwner && !who.owner) return { ok: false, reason: "forbidden", message: needOwner }; await gitOk(p.published, ["merge", "-q", "--ff-only", remoteRef]); pulled = true; } else if (!remoteBehind) { - const r = await git(p.published, ["merge", "--no-ff", "-m", `Sync with origin/${main}`, remoteRef], who); + const r = await git(p.published, ["merge", "--no-ff", "--no-commit", remoteRef], who); if (r.code !== 0) { const conflicts = await unmerged(p.published); await git(p.published, ["merge", "--abort"]); return conflicts.length ? { ok: false, reason: "conflict", conflicts } : { ok: false, reason: "network", message: r.stderr.trim() }; } + const needOwner = ownerRequiredFor(baseManifest, (await readFileIn(p.published, MANIFEST))?.content ?? null); + if (needOwner && !who.owner) { + await git(p.published, ["merge", "--abort"]); + return { ok: false, reason: "forbidden", message: needOwner }; + } + await gitOk(p.published, ["commit", "-q", "-m", `Sync with origin/${main}`], who); pulled = true; } } } const ahead = hasRemote ? await count(p.bare, `${remoteRef}..${main}`) : 1; if (ahead > 0) { - const r = await git(p.bare, ["push", "-q", "origin", `${main}:${main}`]); + const r = await git(p.bare, [...remoteGuard, "push", "-q", "origin", `${main}:${main}`]); if (r.code !== 0) return { ok: false, reason: "network", message: r.stderr.trim() || "push failed" }; - await git(p.bare, ["fetch", "-q", "origin"]); + await git(p.bare, [...remoteGuard, "fetch", "-q", "origin"]); pushed = true; } return { ok: true, pulled, pushed, head: await rev(p.published, "HEAD") }; diff --git a/packages/feature-studio/test/app-helpers.ts b/packages/feature-studio/test/app-helpers.ts index 43b1d11..e7960af 100644 --- a/packages/feature-studio/test/app-helpers.ts +++ b/packages/feature-studio/test/app-helpers.ts @@ -15,8 +15,9 @@ export const users: Record<string, AuthUser> = { }; /** A server context like main.ts builds, with header-selected users (`x-user: ada`). */ -export async function setupApp(extraCourseDirs: string[] = [], dbUrl = "sqlite://:memory:") { +export async function setupApp(extraCourseDirs: string[] = [], dbUrl = "sqlite://:memory:", prepare?: (dataDir: string) => void) { const dataDir = tempDir(); + prepare?.(dataDir); let dropSchema = async () => {}; if (dbUrl.startsWith("postgres")) { // a private schema per test, like packages/db's forEachDb diff --git a/packages/feature-studio/test/feature.test.ts b/packages/feature-studio/test/feature.test.ts index 4636333..12769c5 100644 --- a/packages/feature-studio/test/feature.test.ts +++ b/packages/feature-studio/test/feature.test.ts @@ -74,7 +74,8 @@ test("remote config and sync without a remote", async () => { env = await setupApp(); await env.call("ada", "POST", "/api/studio/courses", { id: "c1", title: "C" }); expect((await env.call("ada", "POST", "/api/studio/courses/c1/sync")).status).toBe(400); - expect((await env.call("ada", "PUT", "/api/studio/courses/c1/remote", { url: "/tmp/nowhere.git" })).status).toBe(200); - expect((await env.call("ada", "GET", "/api/studio/courses/c1")).json.status.remote).toBe("/tmp/nowhere.git"); + expect((await env.call("ada", "PUT", "/api/studio/courses/c1/remote", { url: "/tmp/nowhere.git" })).status).toBe(400); + expect((await env.call("ada", "PUT", "/api/studio/courses/c1/remote", { url: "https://nowhere.invalid/x.git" })).status).toBe(200); + expect((await env.call("ada", "GET", "/api/studio/courses/c1")).json.status.remote).toBe("https://nowhere.invalid/x.git"); expect((await env.call("ada", "POST", "/api/studio/courses/c1/sync")).status).toBe(502); }); diff --git a/packages/feature-studio/test/helpers.ts b/packages/feature-studio/test/helpers.ts index c4516e8..fe8b2c5 100644 --- a/packages/feature-studio/test/helpers.ts +++ b/packages/feature-studio/test/helpers.ts @@ -11,9 +11,9 @@ export function tempDir(prefix = "stc-studio-"): string { return realpathSync(mkdtempSync(join(tmpdir(), prefix))); } -export async function newStore() { +export async function newStore(opts: { allowLocalRemotes?: boolean } = {}) { const dataDir = tempDir(); - const store = createStudioStore({ dataDir }); + const store = createStudioStore({ dataDir, ...opts }); return { dataDir, store }; } diff --git a/packages/feature-studio/test/publish.test.ts b/packages/feature-studio/test/publish.test.ts index a32d481..391d969 100644 --- a/packages/feature-studio/test/publish.test.ts +++ b/packages/feature-studio/test/publish.test.ts @@ -3,8 +3,8 @@ import { readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { ada, bob, extGit, newStore, tempDir } from "./helpers"; -async function courseWithEdit(text = "# Draft text\n") { - const s = await newStore(); +async function courseWithEdit(text = "# Draft text\n", opts: { allowLocalRemotes?: boolean } = {}) { + const s = await newStore(opts); await s.store.createCourse({ id: "c1", title: "C" }, ada); const cur = (await s.store.readFile("c1", ada.id, "lessons/welcome.md"))!; await s.store.saveFiles("c1", ada, [{ path: "lessons/welcome.md", content: text, baseHash: cur.hash }], "Draft edit"); @@ -104,7 +104,8 @@ describe("sync", () => { }); test("push to an empty remote, pull a remote commit, report a conflicting divergence", async () => { - const { store, pub } = await courseWithEdit(); + // local bare remotes need the test-only allowLocalRemotes switch; the HTTP feature never sets it + const { store, pub } = await courseWithEdit("# Draft text\n", { allowLocalRemotes: true }); await store.publish("c1", ada); const remote = join(tempDir(), "remote.git"); await extGit(tempDir(), "init", "-q", "--bare", "-b", "main", remote); @@ -145,11 +146,9 @@ describe("sync", () => { }); }); -test("ext:: remotes cannot run commands", async () => { - const { store } = await courseWithEdit(); +test("ext:: remotes cannot run commands, even with local remotes allowed for tests", async () => { + const { store } = await courseWithEdit("# x\n", { allowLocalRemotes: true }); const marker = join(tempDir(), "pwned"); - await store.setRemote("c1", `ext::touch%s${marker}`); - const r = await store.sync("c1", ada); - expect(r.ok).toBe(false); + await expect(store.setRemote("c1", `ext::touch%s${marker}`)).rejects.toThrow(/remote/); expect(await Bun.file(marker).exists()).toBe(false); }); diff --git a/packages/feature-studio/test/security.test.ts b/packages/feature-studio/test/security.test.ts new file mode 100644 index 0000000..aac80f7 --- /dev/null +++ b/packages/feature-studio/test/security.test.ts @@ -0,0 +1,268 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { isAllowedRemote, ownerRequiredFor } from "../src/server/store"; +import { ada, extGit, newStore, tempDir } from "./helpers"; +import { setupApp } from "./app-helpers"; + +describe("C1: writes never follow symlinks", () => { + test("a dangling symlink is refused for save and resolve; the outside target is never created", async () => { + const { store } = await newStore(); + await store.createCourse({ id: "c1", title: "C" }, ada); + const draft = await store.ensureDraft("c1", ada.id); + const outside = join(tempDir(), "pwned.txt"); + symlinkSync(outside, join(draft, "evil.md")); + await expect(store.saveFiles("c1", ada, [{ path: "evil.md", content: "x" }], "x")).rejects.toThrow(/symlink/); + await expect(store.saveFiles("c1", ada, [{ path: "evil.md", content: "x", baseHash: null }], "x")).rejects.toThrow(/symlink/); + expect(existsSync(outside)).toBe(false); + }); + + test("a symlinked parent directory is refused", async () => { + const { store } = await newStore(); + await store.createCourse({ id: "c1", title: "C" }, ada); + const draft = await store.ensureDraft("c1", ada.id); + const outsideDir = tempDir(); + symlinkSync(outsideDir, join(draft, "sub")); + await expect(store.saveFiles("c1", ada, [{ path: "sub/x.md", content: "x" }], "x")).rejects.toThrow(/symlink/); + await expect(store.saveFiles("c1", ada, [{ path: "sub/deeper/x.md", content: "x" }], "x")).rejects.toThrow(/symlink/); + expect(existsSync(join(outsideDir, "x.md"))).toBe(false); + expect(existsSync(join(outsideDir, "deeper"))).toBe(false); + }); + + test("a symlink pulled in from a remote merge cannot be written through; deleting the link leaves the target", async () => { + const { store } = await newStore({ allowLocalRemotes: true }); + await store.createCourse({ id: "c1", title: "C" }, ada); + const remote = join(tempDir(), "remote.git"); + await extGit(tempDir(), "init", "-q", "--bare", "-b", "main", remote); + await store.setRemote("c1", remote); + await store.sync("c1", ada); // push main + const outsideDir = tempDir(); + const target = join(outsideDir, "target.md"); + writeFileSync(target, "outside\n"); + const other = join(tempDir(), "other"); + await extGit(tempDir(), "clone", "-q", remote, other); + symlinkSync(target, join(other, "notes.md")); + symlinkSync(join(outsideDir, "new.md"), join(other, "dangling.md")); + await extGit(other, "add", "notes.md", "dangling.md"); + await extGit(other, "commit", "-qm", "plant symlinks"); + await extGit(other, "push", "-q", "origin", "main"); + expect((await store.updateDraft("c1", ada, "remote")).ok).toBe(true); + await expect(store.saveFiles("c1", ada, [{ path: "notes.md", content: "pwned" }], "x")).rejects.toThrow(/symlink/); + await expect(store.saveFiles("c1", ada, [{ path: "dangling.md", content: "pwned" }], "x")).rejects.toThrow(/symlink/); + expect(readFileSync(target, "utf8")).toBe("outside\n"); + expect(existsSync(join(outsideDir, "new.md"))).toBe(false); + expect((await store.saveFiles("c1", ada, [{ path: "notes.md", delete: true }], "rm link")).ok).toBe(true); + expect(readFileSync(target, "utf8")).toBe("outside\n"); + }); + + test("resolve refuses to write through a symlink", async () => { + const { store } = await newStore(); + await store.createCourse({ id: "c1", title: "C" }, ada); + const pub = store.paths("c1").published; + const cur = (await store.readFile("c1", ada.id, "lessons/welcome.md"))!; + await store.saveFiles("c1", ada, [{ path: "lessons/welcome.md", content: "# mine\n", baseHash: cur.hash }], "mine"); + writeFileSync(join(pub, "lessons/welcome.md"), "# theirs\n"); + await extGit(pub, "commit", "-qam", "theirs"); + await store.updateDraft("c1", ada); + const outside = join(tempDir(), "p.txt"); + symlinkSync(outside, join(store.paths("c1").draft(ada.id), "link.md")); + await expect(store.resolve("c1", ada, [{ path: "link.md", content: "x" }])).rejects.toThrow(/symlink/); + expect(existsSync(outside)).toBe(false); + }); +}); + +describe("I1: remotes", () => { + test.each([ + "/tmp/x.git", "../x.git", "x.git", "file:///tmp/x.git", "ext::sh -c touch% /tmp/p", "fd::3", "http://host/x.git", "git://host/x.git", + "-uoption", "https://", "https:///x", "git@:x", "git@host:-x", "ssh://host", "ssh://-oProxyCommand=x/y", "https://host/x y", + ])("rejects %p", async (url) => { + expect(isAllowedRemote(url)).toBe(false); + const { store } = await newStore(); + await store.createCourse({ id: "c1", title: "C" }, ada); + await expect(store.setRemote("c1", url)).rejects.toThrow(); + expect(await store.remote("c1")).toBeNull(); + }); + + test.each(["https://github.com/a/b.git", "https://user:tok@git.example.org:8443/a/b.git", "ssh://git@host.example:22/a/b.git", "git@github.com:a/b.git"])("accepts %p", (url) => { + expect(isAllowedRemote(url)).toBe(true); + }); + + test("a remote configured outside the studio is re-checked before sync and update", async () => { + const { store } = await newStore(); + await store.createCourse({ id: "c1", title: "C" }, ada); + const remote = join(tempDir(), "remote.git"); + await extGit(tempDir(), "init", "-q", "--bare", "-b", "main", remote); + await extGit(store.paths("c1").bare, "remote", "add", "origin", remote); + expect(await store.sync("c1", ada)).toMatchObject({ ok: false, reason: "no-remote" }); + expect(await store.updateDraft("c1", ada, "remote")).toMatchObject({ ok: false, reason: "no-remote" }); + expect(await extGit(remote, "for-each-ref")).toBe(""); + }); +}); + +describe("I3: owner-only visibility", () => { + const imported = "schema: 1\nid: c1\ntitle: C\nvisibility: restricted\nsource: { kind: packt-video }\nmodules: []\n"; + test("rule", () => { + expect(ownerRequiredFor(imported, imported)).toBeNull(); + expect(ownerRequiredFor(imported, imported.replace("restricted", "public"))).toMatch(/owner/); + expect(ownerRequiredFor(imported, imported.replace("restricted", "signed-in").replace("packt-video", "authored"))).toMatch(/owner/); + expect(ownerRequiredFor(imported, imported.replace("source: { kind: packt-video }\n", ""))).toMatch(/source.kind/); + const authored = imported.replace("packt-video", "authored"); + expect(ownerRequiredFor(authored, authored.replace("restricted", "public"))).toBeNull(); + expect(ownerRequiredFor(authored, authored.replace("authored", "pdf-deck"))).toMatch(/source.kind/); + }); + + let env: Awaited<ReturnType<typeof setupApp>>; + afterEach(() => env?.stop()); + + test("publish by a non-owner is 403 when the merged manifest makes an imported course visible; the owner may", async () => { + env = await setupApp(); + const { call, feature } = env; + await call("ada", "POST", "/api/studio/courses", { id: "c1", title: "C" }); + // the published course is an import (as an importer-adopted course would be) + const pub = feature.store.paths("c1").published; + const m0 = readFileSync(join(pub, "stcourse.yaml"), "utf8").replace("kind: authored", "kind: packt-video"); + writeFileSync(join(pub, "stcourse.yaml"), m0); + await extGit(pub, "commit", "-qam", "imported"); + await feature.watcher.check(); + const ada = { id: "u-ada", name: "Ada", email: "ada@example.org" }; + + // save-time check stops the plain attempt + const st = (await call("ada", "GET", "/api/studio/courses/c1")).json; + expect((await call("ada", "POST", "/api/studio/courses/c1/update-draft")).status).toBe(200); + const st2 = (await call("ada", "GET", "/api/studio/courses/c1")).json; + expect((await call("ada", "PUT", "/api/studio/courses/c1/files", { message: "x", manifest: { ...st2.manifestRaw, visibility: "public" }, manifestBaseHash: st2.manifestHash, files: [] })).status).toBe(403); + void st; + + // a draft that got there another way (AI draft writer, merge) is stopped at publish + for (const edit of [(t: string) => t.replace("visibility: restricted", "visibility: public"), (t: string) => t.replace("visibility: restricted", "visibility: public").replace("kind: packt-video", "kind: authored")]) { + const cur = (await feature.store.readFile("c1", ada.id, "stcourse.yaml"))!; + await feature.store.saveFiles("c1", ada, [{ path: "stcourse.yaml", content: edit(cur.content), baseHash: cur.hash }], "sneak"); + const r = await call("ada", "POST", "/api/studio/courses/c1/publish"); + expect(r.status).toBe(403); + expect(r.json.reason).toBe("forbidden"); + expect(readFileSync(join(pub, "stcourse.yaml"), "utf8")).toBe(m0); + expect(await extGit(pub, "status", "--porcelain")).toBe(""); + await feature.store.saveFiles("c1", ada, [{ path: "stcourse.yaml", content: cur.content }], "undo"); + } + + // the owner may + const own = (await call("owner", "GET", "/api/studio/courses/c1")).json; + expect((await call("owner", "PUT", "/api/studio/courses/c1/files", { message: "public", manifest: { ...own.manifestRaw, visibility: "public" }, manifestBaseHash: own.manifestHash, files: [] })).status).toBe(200); + expect((await call("owner", "POST", "/api/studio/courses/c1/publish")).status).toBe(200); + expect(readFileSync(join(pub, "stcourse.yaml"), "utf8")).toContain("visibility: public"); + }); + + test("resolve of the manifest by a non-owner is checked too", async () => { + const { store } = await newStore(); + await store.createCourse({ id: "c1", title: "C" }, ada); + const pub = store.paths("c1").published; + const cur = (await store.readFile("c1", ada.id, "stcourse.yaml"))!; + await store.saveFiles("c1", ada, [{ path: "stcourse.yaml", content: cur.content.replace("title: C", "title: Mine"), baseHash: cur.hash }], "mine"); + writeFileSync(join(pub, "stcourse.yaml"), cur.content.replace("title: C", "title: Theirs").replace("kind: authored", "kind: packt-video")); + await extGit(pub, "commit", "-qam", "theirs"); + expect((await store.updateDraft("c1", ada)).ok).toBe(false); + const bad = cur.content.replace("kind: authored", "kind: packt-video").replace("visibility: restricted", "visibility: public"); + await expect(store.resolve("c1", ada, [{ path: "stcourse.yaml", content: bad }])).rejects.toThrow(/owner/); + expect((await store.resolve("c1", { ...ada, owner: true }, [{ path: "stcourse.yaml", content: bad }])).remaining).toEqual([]); + }); +}); + +describe("I2: uploads", () => { + let env: Awaited<ReturnType<typeof setupApp>>; + afterEach(() => { delete process.env.STC_UPLOAD_MAX_MB; return env?.stop(); }); + const upload = (name: string, bytes: Uint8Array<ArrayBuffer>, extra: Record<string, string> = {}) => { + const fd = new FormData(); + fd.set("file", new File([bytes], name)); + fd.set("dir", "media"); + for (const [k, v] of Object.entries(extra)) fd.set(k, v); + return env.call("ada", "POST", "/api/studio/courses/c1/upload", fd); + }; + + test("type allowlist, manifest refused, no silent overwrite", async () => { + env = await setupApp(); + await env.call("ada", "POST", "/api/studio/courses", { id: "c1", title: "C" }); + expect((await upload("tool.exe", new Uint8Array([1]))).status).toBe(415); + expect((await upload("page.HTM", new Uint8Array([1]))).status).toBe(415); + expect((await upload("x", new Uint8Array([1]), { path: "stcourse.yaml" })).status).toBe(400); + expect((await upload("slides.pdf", new Uint8Array([1]))).status).toBe(200); + const again = await upload("slides.pdf", new Uint8Array([2])); + expect(again.status).toBe(409); + expect(readFileSync(join(env.dataDir, "worktrees/c1/drafts/u-ada/media/slides.pdf"))[0]).toBe(1); + expect((await upload("slides.pdf", new Uint8Array([2]), { overwrite: "true" })).status).toBe(200); + expect(readFileSync(join(env.dataDir, "worktrees/c1/drafts/u-ada/media/slides.pdf"))[0]).toBe(2); + }); + + test("size cap from STC_UPLOAD_MAX_MB, by Content-Length and while streaming", async () => { + env = await setupApp(); + await env.call("ada", "POST", "/api/studio/courses", { id: "c1", title: "C" }); + process.env.STC_UPLOAD_MAX_MB = "1"; + expect((await upload("big.zip", new Uint8Array(1024 * 1024 + 10))).status).toBe(413); + // no Content-Length: a streamed body is cut off once it passes the cap + const fd = new FormData(); + fd.set("file", new File([new Uint8Array(2 * 1024 * 1024)], "big.zip")); + const ct = new Response(fd).headers.get("content-type")!; + const bytes = new Uint8Array(await new Response(fd).arrayBuffer()); + const stream = new ReadableStream({ start(c) { for (let i = 0; i < bytes.length; i += 65536) c.enqueue(bytes.slice(i, i + 65536)); c.close(); } }); + const r = await env.app.fetch(new Request("http://x/api/studio/courses/c1/upload", { method: "POST", headers: { "x-user": "ada", "content-type": ct }, body: stream, duplex: "half" } as RequestInit)); + expect(r.status).toBe(413); + expect(existsSync(join(env.dataDir, "worktrees/c1/drafts/u-ada/big.zip"))).toBe(false); + expect((await upload("small.zip", new Uint8Array(1000))).status).toBe(200); + }); + + test("raw draft files are served with a sandboxing CSP", async () => { + env = await setupApp(); + await env.call("ada", "POST", "/api/studio/courses", { id: "c1", title: "C" }); + await upload("x.svg", new TextEncoder().encode("<svg xmlns='http://www.w3.org/2000/svg'><script>alert(1)</script></svg>")); + const r = await env.app.fetch(new Request("http://x/api/studio/courses/c1/raw/media/x.svg", { headers: { "x-user": "ada" } })); + expect(r.headers.get("content-security-policy")).toContain("sandbox"); + }); +}); + +describe("I4: layout and imported courses", () => { + let env: Awaited<ReturnType<typeof setupApp>>; + afterEach(() => env?.stop()); + + test("imported courses in <dataDir>/courses are read-only with their own message; ids cannot collide", async () => { + env = await setupApp([], undefined, (dataDir) => { + mkdirSync(join(dataDir, "courses/imp/lessons"), { recursive: true }); + writeFileSync(join(dataDir, "courses/imp/lessons/a.md"), "# A\n"); + writeFileSync(join(dataDir, "courses/imp/stcourse.yaml"), "schema: 1\nid: imp\ntitle: Imported\nsource: { kind: packt-video }\nmodules:\n - { id: m, title: M, lessons: [{ id: a, title: A, type: text, file: lessons/a.md }] }\n"); + }); + expect(env.ctx.courses.get("imp")?.status).toBe("ok"); + const list = (await env.call("ada", "GET", "/api/studio/courses")).json; + expect(list.courses).toContainEqual(expect.objectContaining({ id: "imp", managed: false, message: "Imported course: read-only in the studio for now" })); + expect((await env.call("ada", "GET", "/api/studio/courses/imp")).json.message).toBe("Imported course: read-only in the studio for now"); + const w = await env.call("ada", "PUT", "/api/studio/courses/imp/files", { message: "x", files: [{ path: "a.md", content: "x" }] }); + expect(w.status).toBe(403); + expect(w.json.error).toBe("Imported course: read-only in the studio for now"); + expect((await env.call("ada", "POST", "/api/studio/courses", { id: "imp", title: "Dup" })).status).toBe(409); + // studio repos are not in the importer's folder + expect((await env.call("ada", "POST", "/api/studio/courses", { id: "c1", title: "C" })).status).toBe(201); + expect(existsSync(join(env.dataDir, "studio/repos/c1.git/HEAD"))).toBe(true); + expect(existsSync(join(env.dataDir, "courses/c1"))).toBe(false); + }); + + test("store refuses an id already used by an importer folder", async () => { + const { store, dataDir } = await newStore(); + mkdirSync(join(dataDir, "courses/imp"), { recursive: true }); + await expect(store.createCourse({ id: "imp", title: "x" }, ada)).rejects.toThrow(/exists/); + }); +}); + +describe("I5: listing has no side effects", () => { + let env: Awaited<ReturnType<typeof setupApp>>; + afterEach(() => env?.stop()); + + test("GET /api/studio/courses creates no draft worktree or branch; opening the editor does", async () => { + env = await setupApp(); + await env.call("owner", "POST", "/api/studio/courses", { id: "c1", title: "C" }); + const bare = env.feature.store.paths("c1").bare; + const draft = env.feature.store.paths("c1").draft("u-ada"); + const list = await env.call("ada", "GET", "/api/studio/courses"); + expect(list.json.courses[0]).toMatchObject({ id: "c1", ahead: 0, behind: 0, merging: false }); + expect(existsSync(draft)).toBe(false); + expect(await extGit(bare, "for-each-ref", "--format=%(refname)", "refs/heads/draft")).toBe(""); + await env.call("ada", "GET", "/api/studio/courses/c1"); + expect(existsSync(draft)).toBe(true); + }); +}); diff --git a/packages/feature-studio/test/store.test.ts b/packages/feature-studio/test/store.test.ts index 6377f42..c287389 100644 --- a/packages/feature-studio/test/store.test.ts +++ b/packages/feature-studio/test/store.test.ts @@ -9,8 +9,9 @@ describe("createCourse", () => { test("makes a bare repo, a published worktree on main and a valid manifest", async () => { const { dataDir, store } = await newStore(); await store.createCourse({ id: "intro-git", title: "Intro: Git & you" }, ada); - expect(existsSync(join(dataDir, "courses/intro-git/HEAD"))).toBe(true); - expect(existsSync(join(dataDir, "courses/intro-git/stcourse.yaml"))).toBe(false); + expect(existsSync(join(dataDir, "studio/repos/intro-git.git/HEAD"))).toBe(true); + expect(existsSync(join(dataDir, "studio/repos/intro-git.git/stcourse.yaml"))).toBe(false); + expect(existsSync(join(dataDir, "courses"))).toBe(false); // the importer's folder is left alone const pub = store.paths("intro-git").published; expect(pub).toBe(join(dataDir, "worktrees/intro-git/published")); expect(await extGit(pub, "rev-parse", "--abbrev-ref", "HEAD")).toBe("main"); From a13cc66ebb30770362731276a306618e41ba974d Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 04:47:25 +0100 Subject: [PATCH 49/62] fix(ai): BYOK endpoints limited to official presets and admin-allowed origins (SSRF) - A user's own key goes only to https://api.anthropic.com, https://api.openai.com/v1 or an origin listed in STC_AI_ALLOWED_BASE_URLS / the server config's allowedBaseUrls; anything else is a 400, and a stored key whose origin was later removed stops being used. - User-scope provider calls never follow redirects, and their errors reach the user only as the HTTP status plus a fixed message; the full text is logged. - The admin's server-scope provider may still use any base URL. - Admin UI: allowed origins field; MCP stdio env editable, HTTP headers kept. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- packages/feature-ai/src/endpoints.ts | 92 +++++++++++++++++++ packages/feature-ai/src/feature.ts | 56 ++++++++--- packages/feature-ai/src/settings.ts | 10 +- packages/feature-ai/src/web/AccountAiPage.tsx | 2 +- packages/feature-ai/src/web/AdminAiPage.tsx | 34 +++++-- packages/feature-ai/test/feature.test.ts | 68 +++++++++++++- packages/feature-ai/test/settings.test.ts | 2 +- 7 files changed, 236 insertions(+), 28 deletions(-) create mode 100644 packages/feature-ai/src/endpoints.ts diff --git a/packages/feature-ai/src/endpoints.ts b/packages/feature-ai/src/endpoints.ts new file mode 100644 index 0000000..e0368a4 --- /dev/null +++ b/packages/feature-ai/src/endpoints.ts @@ -0,0 +1,92 @@ +import type { AiProvider } from "@stc/contracts"; +import type { ProviderKind } from "./settings"; + +/** + * Where a user's own key (BYOK) may be sent. Learners must not be able to point the server at arbitrary hosts + * (SSRF), so BYOK uses the official endpoint for its kind, or an origin the admin allowed explicitly through + * STC_AI_ALLOWED_BASE_URLS (comma-separated origins) or the server AI config's `allowedBaseUrls`. + * The admin's own server-scope provider may use any base URL (Ollama and the like). + */ +export const PRESET_BASE_URLS: Record<ProviderKind, string> = { + anthropic: "https://api.anthropic.com", + openai: "https://api.openai.com/v1", +}; + +const trimSlash = (s: string) => s.replace(/\/+$/, ""); + +/** The origin of an http(s) URL, or null. "Exact origins" means scheme, host and port must all match. */ +export function originOf(u: string): string | null { + try { + const url = new URL(u.trim()); + if (url.protocol !== "http:" && url.protocol !== "https:") return null; + if (url.username || url.password) return null; + return url.origin; + } catch { + return null; + } +} + +/** Parses an allowlist (array or comma-separated string) into origins; entries that are not plain origins are dropped. */ +export function parseAllowedOrigins(v: unknown): string[] { + const items = Array.isArray(v) ? v.map(String) : typeof v === "string" ? v.split(",") : []; + const out = new Set<string>(); + for (const raw of items) { + const s = trimSlash(raw.trim()); + if (!s) continue; + const o = originOf(s); + if (o && o === s) out.add(o); + } + return [...out]; +} + +/** + * The base URL a BYOK config may use, or null when it is not allowed. + * - empty: the official endpoint for the kind; + * - the official endpoint for the kind (with or without a trailing slash); + * - any URL whose origin is on the allowlist. + */ +export function byokBaseUrl(kind: ProviderKind, baseUrl: string | undefined, allowed: string[]): string | null { + const preset = PRESET_BASE_URLS[kind]; + if (!baseUrl || !baseUrl.trim()) return preset; + const clean = trimSlash(baseUrl.trim()); + if (clean === preset) return preset; + const origin = originOf(clean); + if (!origin) return null; + const url = new URL(clean); + if (url.search || url.hash) return null; + return allowed.includes(origin) ? clean : null; +} + +/** fetch for user-scope calls: never follows redirects (an allowed host must not bounce the request elsewhere). */ +export function noRedirectFetch(base: typeof fetch): typeof fetch { + const f = ((input: RequestInfo | URL, init?: RequestInit) => base(input, { ...init, redirect: "manual" })) as typeof fetch; + return f; +} + +export const USER_PROVIDER_ERROR = "The AI provider returned an error. Check your key, model and endpoint under Account, AI."; + +/** The message shown for a failed user-scope provider call: the HTTP status, if any, and a fixed sentence. */ +export function publicProviderError(message: string): string { + const status = /(?:^|\s)([1-5]\d{2}):/.exec(message)?.[1]; + return status ? `${status}: ${USER_PROVIDER_ERROR}` : USER_PROVIDER_ERROR; +} + +/** + * Wraps a provider created with a user's key: any error is logged server-side with its full text and rethrown + * as the status plus a fixed message, so upstream response bodies never reach the browser. + */ +export function sanitizeProviderErrors(p: AiProvider, log: (msg: string) => void): AiProvider { + return { + id: p.id, + async *chat(messages, opts) { + try { + yield* p.chat(messages, opts); + } catch (e) { + const raw = (e as Error)?.message ?? String(e); + if ((e as Error)?.name === "AbortError") throw e; + log(raw); + throw new Error(publicProviderError(raw)); + } + }, + }; +} diff --git a/packages/feature-ai/src/feature.ts b/packages/feature-ai/src/feature.ts index 625fa23..ad521f8 100644 --- a/packages/feature-ai/src/feature.ts +++ b/packages/feature-ai/src/feature.ts @@ -12,6 +12,7 @@ import { createFsDraftSink, type DraftSink } from "./generator/draft-sink"; import { generateDraft, type GenerateRequest, type GenerateSource } from "./generator/generate"; import { connectMcp, mcpServersFor } from "./mcp/client"; import { createProvider } from "./providers"; +import { PRESET_BASE_URLS, byokBaseUrl, noRedirectFetch, parseAllowedOrigins, publicProviderError, sanitizeProviderErrors } from "./endpoints"; import { getCourseIndex, noteChunks } from "./retrieval/course-index"; import { aiSettingsRepo, type McpServerConfig, type ProviderConfig } from "./settings"; @@ -23,6 +24,10 @@ export interface AiFeatureOptions { /** Overrides provider construction (tests use a fake AiProvider). */ providerFactory?: (cfg: ProviderConfig, apiKey: string | null) => AiProvider; mcpTimeoutMs?: number; + /** Environment to read STC_AI_ALLOWED_BASE_URLS from. Default: process.env. */ + env?: Record<string, string | undefined>; + /** Server-side log for provider errors that are not shown to users. Default: console.error. */ + log?: (message: string) => void; } const json = (body: unknown, status = 200) => Response.json(body, { status }); @@ -40,7 +45,7 @@ async function body(req: Request): Promise<Record<string, unknown> | null> { } const str = (v: unknown): string | undefined => (typeof v === "string" && v.trim() ? v.trim() : undefined); -function parseProvider(v: unknown): (ProviderConfig & { apiKey?: string; clearApiKey?: boolean }) | null | "invalid" { +function parseProvider(v: unknown, scope: "server" | "user"): (ProviderConfig & { apiKey?: string; clearApiKey?: boolean }) | null | "invalid" { if (v === null || v === undefined) return null; if (typeof v !== "object") return "invalid"; const p = v as Record<string, unknown>; @@ -55,7 +60,8 @@ function parseProvider(v: unknown): (ProviderConfig & { apiKey?: string; clearAp if (str(p.effort)) out.effort = str(p.effort) as ProviderConfig["effort"]; if (str(p.apiKey)) out.apiKey = str(p.apiKey); if (p.clearApiKey === true) out.clearApiKey = true; - if (out.kind === "openai" && (!out.baseUrl || !out.model)) return "invalid"; + // Server scope: any endpoint, but OpenAI-compatible needs one. User scope: an empty base URL means the official endpoint. + if (out.kind === "openai" && (!out.model || (scope === "server" && !out.baseUrl))) return "invalid"; return out; } @@ -140,15 +146,28 @@ export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feat const notes = notesRepo(ctx.db); const progress = progressRepo(ctx.db); const sink = opts.draftSink ?? createFsDraftSink(ctx.config.dataDir); - const makeProvider = opts.providerFactory ?? ((cfg: ProviderConfig, key: string | null) => createProvider(cfg, key, opts.fetch)); + const log = opts.log ?? ((m: string) => console.error(`[ai] ${m}`)); + const env = opts.env ?? process.env; + const allowedOrigins = async () => [...new Set([...parseAllowedOrigins(env.STC_AI_ALLOWED_BASE_URLS ?? ""), ...(await settings.getServer()).allowedBaseUrls])]; + /** + * Providers for a user's own key never follow redirects, and their errors reach the user only as the HTTP + * status plus a fixed sentence (the full text is logged here). Server-scope providers are the admin's own. + */ + const makeProvider = (r: { source: "byok" | "server"; cfg: ProviderConfig; key: string | null }, userId: string): AiProvider => { + if (r.source === "server") return opts.providerFactory ? opts.providerFactory(r.cfg, r.key) : createProvider(r.cfg, r.key, opts.fetch); + const p = opts.providerFactory ? opts.providerFactory(r.cfg, r.key) : createProvider(r.cfg, r.key, noRedirectFetch(opts.fetch ?? fetch)); + return sanitizeProviderErrors(p, (m) => log(`provider error for user ${userId} (own key): ${m}`)); + }; /** BYOK first ("used only for their own requests"), then the server provider. */ const resolveProvider = async (userId: string | null): Promise<{ source: "byok" | "server"; cfg: ProviderConfig; key: string | null } | null> => { if (userId) { const u = await settings.getUser(userId); if (u.byok) { - const key = await settings.userApiKey(userId).catch(() => null); - if (key) return { source: "byok", cfg: u.byok, key }; + // Re-checked on every use: an origin the admin removed from the allowlist stops working at once. + const base = byokBaseUrl(u.byok.kind, u.byok.baseUrl, await allowedOrigins()); + const key = base ? await settings.userApiKey(userId).catch(() => null) : null; + if (key && base) return { source: "byok", cfg: { ...u.byok, baseUrl: base }, key }; } } const s = await settings.getServer(); @@ -208,7 +227,7 @@ export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feat const resolved = await resolveProvider(user.id); if (!resolved) return err(409, "AI is not set up. An admin can configure a server provider, or you can add your own key under Account, AI."); let provider: AiProvider; - try { provider = makeProvider(resolved.cfg, resolved.key); } catch (e) { return err(409, (e as Error).message); } + try { provider = makeProvider(resolved, user.id); } catch (e) { return err(409, (e as Error).message); } let conversationId = str(b?.conversationId); if (conversationId) { @@ -261,7 +280,9 @@ export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feat await addMessage(convId, "sources", JSON.stringify(cited)); } } catch (e) { - send({ type: "error", message: (e as Error).message }); + const m = (e as Error).message; + if (resolved.source === "byok") log(`chat error for user ${user.id} (own key): ${m}`); + send({ type: "error", message: resolved.source === "byok" ? publicProviderError(m) : m }); } finally { await tools?.close?.().catch(() => {}); try { controller.close(); } catch {} @@ -275,7 +296,7 @@ export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feat const p = payload as { req: GenerateRequest; userId: string }; const resolved = await resolveProvider(p.userId); if (!resolved) throw new Error("AI is not set up for this user."); - const provider = makeProvider(resolved.cfg, resolved.key); + const provider = makeProvider(resolved, p.userId); return generateDraft({ provider, repo: ctx.courses, req: p.req, sink, authorId: p.userId, dataDir: ctx.config.dataDir, signal: jc.signal, onProgress: (f, m) => jc.reportProgress(f, m) }); }; @@ -305,11 +326,18 @@ export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feat if (deny) return deny; const b = await body(req); if (!b) return err(400, "Invalid JSON."); - const provider = parseProvider(b.provider); + const provider = parseProvider(b.provider, "server"); const mcpServers = parseMcp(b.mcpServers); if (provider === "invalid") return err(400, "Invalid provider. OpenAI-compatible endpoints need a base URL and a model."); if (mcpServers === "invalid") return err(400, "Invalid MCP server list."); - await settings.saveServer({ provider, mcpServers }); + let allowedBaseUrls: string[] | undefined; + if (b.allowedBaseUrls !== undefined) { + const list = Array.isArray(b.allowedBaseUrls) ? b.allowedBaseUrls.map(String).map((x) => x.trim()).filter(Boolean) : null; + if (!list) return err(400, "allowedBaseUrls must be a list of origins."); + allowedBaseUrls = parseAllowedOrigins(list); + if (allowedBaseUrls.length !== new Set(list.map((x) => x.replace(/\/+$/, ""))).size) return err(400, "Each allowed base URL must be an origin such as http://ollama.internal:11434."); + } + await settings.saveServer({ provider, mcpServers, allowedBaseUrls }); return json(await settings.getServer()); }, }, @@ -340,9 +368,11 @@ export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feat PUT: async (req, rc) => { if (!rc.user) return err(401, "Sign in first."); const b = await body(req); - const p = b ? parseProvider(b) : "invalid"; - if (!p || p === "invalid" || !p.apiKey) return err(400, "Choose a provider and paste your key. OpenAI-compatible endpoints also need a base URL and a model."); - await settings.saveUserKey(rc.user.id, { ...p, apiKey: p.apiKey }); + const p = b ? parseProvider(b, "user") : "invalid"; + if (!p || p === "invalid" || !p.apiKey) return err(400, "Choose a provider and paste your key. OpenAI-compatible endpoints also need a model."); + const base = byokBaseUrl(p.kind, p.baseUrl, await allowedOrigins()); + if (!base) return err(400, "That base URL is not allowed for your own key. Use the official endpoint, or ask an admin to allow it."); + await settings.saveUserKey(rc.user.id, { ...p, baseUrl: base === PRESET_BASE_URLS[p.kind] ? undefined : base, apiKey: p.apiKey }); return json((await settings.getUser(rc.user.id)).byok); }, DELETE: async (_req, rc) => { diff --git a/packages/feature-ai/src/settings.ts b/packages/feature-ai/src/settings.ts index 0d1360c..ecc9776 100644 --- a/packages/feature-ai/src/settings.ts +++ b/packages/feature-ai/src/settings.ts @@ -19,14 +19,17 @@ export interface McpServerConfig { courseIds?: string[]; } -export interface ServerSettings { provider: PublicProviderConfig | null; mcpServers: McpServerConfig[] } +/** `allowedBaseUrls`: origins learners' own keys (BYOK) may be sent to, besides the official endpoints. */ +export interface ServerSettings { provider: PublicProviderConfig | null; mcpServers: McpServerConfig[]; allowedBaseUrls: string[] } export interface ServerSettingsInput { provider: (ProviderConfig & { apiKey?: string; clearApiKey?: boolean }) | null; mcpServers: McpServerConfig[]; + /** Omitted: keep the saved list. */ + allowedBaseUrls?: string[]; } export interface UserSettings { byok: PublicProviderConfig | null; mentorOff: Record<string, boolean> } -type ServerRow = { provider: ProviderConfig | null; apiKeyCipher: string | null; mcpServers: McpServerConfig[] }; +type ServerRow = { provider: ProviderConfig | null; apiKeyCipher: string | null; mcpServers: McpServerConfig[]; allowedBaseUrls?: string[] }; type UserRow = { byok: ProviderConfig | null; mentorOff: Record<string, boolean> }; const pickProvider = (p: ProviderConfig): ProviderConfig => { @@ -72,7 +75,7 @@ export function aiSettingsRepo(db: Db, box: SecretBox) { return { async getServer(): Promise<ServerSettings> { const r = await serverRow(); - return { provider: r.provider ? { ...r.provider, hasApiKey: !!r.apiKeyCipher } : null, mcpServers: r.mcpServers ?? [] }; + return { provider: r.provider ? { ...r.provider, hasApiKey: !!r.apiKeyCipher } : null, mcpServers: r.mcpServers ?? [], allowedBaseUrls: r.allowedBaseUrls ?? [] }; }, async saveServer(input: ServerSettingsInput): Promise<void> { const prev = await serverRow(); @@ -85,6 +88,7 @@ export function aiSettingsRepo(db: Db, box: SecretBox) { provider: input.provider ? pickProvider(input.provider) : null, apiKeyCipher, mcpServers: input.mcpServers ?? [], + allowedBaseUrls: input.allowedBaseUrls ?? prev.allowedBaseUrls ?? [], }); }, async serverApiKey(): Promise<string | null> { diff --git a/packages/feature-ai/src/web/AccountAiPage.tsx b/packages/feature-ai/src/web/AccountAiPage.tsx index 0b0d41f..137cfe8 100644 --- a/packages/feature-ai/src/web/AccountAiPage.tsx +++ b/packages/feature-ai/src/web/AccountAiPage.tsx @@ -66,7 +66,7 @@ export default function AccountAiPage() { </Field> </div> {kind === "openai" ? ( - <Field label="Base URL" hint="The address that ends before /chat/completions."> + <Field label="Base URL (optional)" hint="Leave empty for api.openai.com. Other endpoints work only if an admin allowed them."> <input className="ai-input ai-mono" value={baseUrl} onChange={(e) => setBaseUrl(e.target.value)} placeholder="https://api.openai.com/v1" /> </Field> ) : null} diff --git a/packages/feature-ai/src/web/AdminAiPage.tsx b/packages/feature-ai/src/web/AdminAiPage.tsx index 0c3d0ca..21c4ad5 100644 --- a/packages/feature-ai/src/web/AdminAiPage.tsx +++ b/packages/feature-ai/src/web/AdminAiPage.tsx @@ -5,14 +5,19 @@ import { Field, PageHead } from "./components"; type Kind = "" | "anthropic" | "openai"; interface ProviderOut { kind: "anthropic" | "openai"; model?: string; baseUrl?: string; effort?: string; hasApiKey: boolean } -interface McpServer { id: string; name: string; transport: "stdio" | "http"; command?: string; args?: string[]; url?: string; courseIds?: string[] } -interface Config { provider: ProviderOut | null; mcpServers: McpServer[] } +interface McpServer { id: string; name: string; transport: "stdio" | "http"; command?: string; args?: string[]; env?: Record<string, string>; url?: string; headers?: Record<string, string>; courseIds?: string[] } +interface Config { provider: ProviderOut | null; mcpServers: McpServer[]; allowedBaseUrls: string[] } -type ServerDraft = { id: string; name: string; transport: "stdio" | "http"; command: string; args: string; url: string; courses: string; test?: string }; -const toDraft = (s: McpServer): ServerDraft => ({ id: s.id, name: s.name, transport: s.transport, command: s.command ?? "", args: (s.args ?? []).join(" "), url: s.url ?? "", courses: (s.courseIds ?? []).join(", ") }); +type ServerDraft = { id: string; name: string; transport: "stdio" | "http"; command: string; args: string; env: string; url: string; headers?: Record<string, string>; courses: string; test?: string }; +const toDraft = (s: McpServer): ServerDraft => ({ + id: s.id, name: s.name, transport: s.transport, command: s.command ?? "", args: (s.args ?? []).join(" "), + env: Object.entries(s.env ?? {}).map(([k, v]) => `${k}=${v}`).join("\n"), url: s.url ?? "", headers: s.headers, courses: (s.courseIds ?? []).join(", "), +}); +const parseEnv = (text: string): Record<string, string> => + Object.fromEntries(text.split("\n").map((l) => l.trim()).filter((l) => l.includes("=")).map((l) => [l.slice(0, l.indexOf("=")).trim(), l.slice(l.indexOf("=") + 1)])); const fromDraft = (d: ServerDraft): McpServer => ({ id: d.id, name: d.name || d.id, transport: d.transport, - ...(d.transport === "stdio" ? { command: d.command, args: d.args.trim() ? d.args.trim().split(/\s+/) : [] } : { url: d.url }), + ...(d.transport === "stdio" ? { command: d.command, args: d.args.trim() ? d.args.trim().split(/\s+/) : [], env: parseEnv(d.env) } : { url: d.url, ...(d.headers ? { headers: d.headers } : {}) }), courseIds: d.courses.split(",").map((s) => s.trim()).filter(Boolean), }); @@ -26,6 +31,7 @@ export default function AdminAiPage() { const [apiKey, setApiKey] = useState(""); const [clearKey, setClearKey] = useState(false); const [servers, setServers] = useState<ServerDraft[]>([]); + const [allowed, setAllowed] = useState(""); const [msg, setMsg] = useState<{ ok: boolean; text: string } | null>(null); const apply = (c: Config) => { @@ -37,6 +43,7 @@ export default function AdminAiPage() { setApiKey(""); setClearKey(false); setServers(c.mcpServers.map(toDraft)); + setAllowed((c.allowedBaseUrls ?? []).join("\n")); }; useEffect(() => { api<Config>("/api/ai/admin/config").then(apply, (e: ApiError) => setDenied(e.status === 401 ? "Sign in as the owner to change AI settings." : e.message)); @@ -46,7 +53,7 @@ export default function AdminAiPage() { setMsg(null); try { const provider = kind ? { kind, model, baseUrl, effort, apiKey: apiKey || undefined, clearApiKey: clearKey } : null; - apply(await api<Config>("/api/ai/admin/config", { method: "PUT", body: { provider, mcpServers: servers.map(fromDraft) } })); + apply(await api<Config>("/api/ai/admin/config", { method: "PUT", body: { provider, mcpServers: servers.map(fromDraft), allowedBaseUrls: allowed.split(/[\s,]+/).filter(Boolean) } })); setMsg({ ok: true, text: kind ? "Saved. The assistant is available to learners." : "Saved. AI is off for everyone without their own key." }); } catch (e) { setMsg({ ok: false, text: (e as Error).message }); @@ -118,6 +125,14 @@ export default function AdminAiPage() { ) : null} </section> + <section className="ai-section"> + <h2>Endpoints for learners' own keys</h2> + <span className="ai-hint">Own keys go to api.anthropic.com or api.openai.com only. To let people use another endpoint, such as a shared Ollama, list its origin here (one per line, for example http://ollama.internal:11434). STC_AI_ALLOWED_BASE_URLS adds more.</span> + <Field label="Allowed origins"> + <textarea className="ai-input ai-mono" rows={2} value={allowed} onChange={(e) => setAllowed(e.target.value)} placeholder="http://ollama.internal:11434" /> + </Field> + </section> + <section className="ai-section"> <h2>MCP servers</h2> <span className="ai-hint">Extra tools for the assistant. Leave "Courses" empty to offer a server in every course.</span> @@ -141,6 +156,11 @@ export default function AdminAiPage() { <Field label="Arguments"><input className="ai-input ai-mono" value={s.args} onChange={(e) => setServer(i, { args: e.target.value })} placeholder="some-mcp-server --flag" /></Field> </div> )} + {s.transport === "stdio" ? ( + <Field label="Environment" hint="One NAME=value per line. The command gets only PATH, HOME, LANG and these; server secrets are never passed on."> + <textarea className="ai-input ai-mono" rows={2} value={s.env} onChange={(e) => setServer(i, { env: e.target.value })} placeholder="API_TOKEN=…" /> + </Field> + ) : null} <Field label="Courses" hint="Course ids, separated by commas."><input className="ai-input ai-mono" value={s.courses} onChange={(e) => setServer(i, { courses: e.target.value })} /></Field> <div className="ai-footer" style={{ margin: "0 0 10px" }}> <Pill onClick={() => testServer(i)}>Test connection</Pill> @@ -150,7 +170,7 @@ export default function AdminAiPage() { </div> ))} <div className="ai-footer"> - <Pill onClick={() => setServers((x) => [...x, { id: `server-${x.length + 1}`, name: "", transport: "http", command: "", args: "", url: "", courses: "" }])}>Add MCP server</Pill> + <Pill onClick={() => setServers((x) => [...x, { id: `server-${x.length + 1}`, name: "", transport: "http", command: "", args: "", env: "", url: "", courses: "" }])}>Add MCP server</Pill> </div> </section> diff --git a/packages/feature-ai/test/feature.test.ts b/packages/feature-ai/test/feature.test.ts index 8b657c6..5ff2293 100644 --- a/packages/feature-ai/test/feature.test.ts +++ b/packages/feature-ai/test/feature.test.ts @@ -7,7 +7,7 @@ import { createApp, createJobQueue } from "@stc/core"; import { createCourseRepository } from "@stc/course-format"; import type { Db } from "@stc/db"; import { forEachDb } from "../../db/test/helpers"; -import { aiFeature } from "../src/feature"; +import { aiFeature, type AiFeatureOptions } from "../src/feature"; import { fakeProvider } from "./helpers"; const SAMPLE = resolve(import.meta.dir, "../../../fixtures/courses/sample"); @@ -18,7 +18,7 @@ const USERS: Record<string, AuthUser> = { auth: { id: "auth", email: "w@x", name: "Writer", roles: ["author"], groups: [] }, }; -async function setup(db: Db, turns: Parameters<typeof fakeProvider>[0] = [() => ({ text: "ok" })]) { +async function setup(db: Db, turns: Parameters<typeof fakeProvider>[0] = [() => ({ text: "ok" })], extra: Partial<AiFeatureOptions> = {}) { const dir = mkdtempSync(join(tmpdir(), "stc-ai-feat-")); cpSync(SAMPLE, join(dir, "courses/sample"), { recursive: true }); cpSync(SAMPLE, join(dir, "courses/secret"), { recursive: true }); @@ -32,7 +32,7 @@ async function setup(db: Db, turns: Parameters<typeof fakeProvider>[0] = [() => config: { port: 0, databaseUrl: "", dataDir: join(dir, "data"), courseDirs: [], secret: "test-secret", publicUrl: "" }, auth: { userFromRequest: async (req: Request) => USERS[req.headers.get("x-user") ?? ""] ?? null }, } as unknown as ServerContext; - const feature = aiFeature(ctx, { providerFactory: (_cfg, key) => { keys.push(key); return fake.provider; } }); + const feature = aiFeature(ctx, { providerFactory: (_cfg, key) => { keys.push(key); return fake.provider; }, ...extra }); const jobs = createJobQueue(db, feature.jobs!); ctx.jobs = jobs; const app = createApp(ctx, [feature]); @@ -156,3 +156,65 @@ forEachDb("generation is for authors and writes drafts, not course files", async expect((await call("GET", `/api/ai/studio/jobs/${jobId}`, "auth")).status).toBe(200); expect((await call("GET", `/api/ai/studio/jobs/${jobId}`, "bob")).status).toBe(404); }); + +// SSRF guard: a learner's own key (BYOK) may only go to the official endpoints or admin-allowed origins. +forEachDb("BYOK base URLs are limited to official endpoints and allowed origins", async (db) => { + const { call } = await setup(db, undefined, { env: { STC_AI_ALLOWED_BASE_URLS: "http://ollama.internal:11434, https://llm.example.org" } }); + const put = (body: unknown) => call("PUT", "/api/ai/account/key", "ann", body); + // Internal or arbitrary hosts are refused for either kind. + for (const baseUrl of ["http://127.0.0.1:8080/v1", "http://169.254.169.254/latest", "http://localhost:11434/v1", "https://api.anthropic.com.evil.test", "http://ollama.internal:11435/v1", "https://ollama.internal:11434/v1", "file:///etc/passwd"]) { + expect((await put({ kind: "openai", model: "m", apiKey: "k", baseUrl })).status).toBe(400); + } + expect((await put({ kind: "anthropic", apiKey: "k", baseUrl: "http://127.0.0.1:9/" })).status).toBe(400); + expect((await call("GET", "/api/ai/account", "ann")).status).toBe(200); + expect((await (await call("GET", "/api/ai/account", "ann")).json()).byok).toBeNull(); + // Official endpoints and allowlisted origins (any path on them) are accepted. + expect((await put({ kind: "anthropic", apiKey: "k" })).status).toBe(200); + expect((await put({ kind: "anthropic", apiKey: "k", baseUrl: "https://api.anthropic.com/" })).status).toBe(200); + expect((await put({ kind: "openai", model: "m", apiKey: "k" })).status).toBe(200); + expect(await (await call("GET", "/api/ai/account", "ann")).json()).toMatchObject({ byok: { kind: "openai", model: "m", hasApiKey: true } }); // official endpoint: stored as "no base URL" + expect((await put({ kind: "openai", model: "m", apiKey: "k", baseUrl: "http://ollama.internal:11434/v1" })).status).toBe(200); + expect((await put({ kind: "openai", model: "m", apiKey: "k", baseUrl: "https://llm.example.org/v1/" })).status).toBe(200); + // The admin's server-scope provider may use any endpoint (Ollama on localhost). + expect((await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "openai", model: "llama", baseUrl: "http://127.0.0.1:11434/v1" }, mcpServers: [] })).status).toBe(200); +}); + +forEachDb("the admin allowlist in the server config enables an origin for BYOK, and removing it disables the key", async (db) => { + const { call } = await setup(db); + const put = () => call("PUT", "/api/ai/account/key", "ann", { kind: "openai", model: "m", apiKey: "k", baseUrl: "http://gpu-box:8000/v1" }); + expect((await put()).status).toBe(400); + expect((await call("PUT", "/api/ai/admin/config", "owner", { provider: null, mcpServers: [], allowedBaseUrls: ["http://gpu-box:8000/v1"] })).status).toBe(400); // origins only + expect((await call("PUT", "/api/ai/admin/config", "ann", { provider: null, mcpServers: [], allowedBaseUrls: ["http://gpu-box:8000"] })).status).toBe(403); + expect((await call("PUT", "/api/ai/admin/config", "owner", { provider: null, mcpServers: [], allowedBaseUrls: ["http://gpu-box:8000"] })).status).toBe(200); + expect((await (await call("GET", "/api/ai/admin/config", "owner")).json()).allowedBaseUrls).toEqual(["http://gpu-box:8000"]); + expect((await put()).status).toBe(200); + expect(await (await call("GET", "/api/ai/status?courseId=sample", "ann")).json()).toMatchObject({ enabled: true, source: "byok" }); + // Saving the config without the field keeps the list; an empty list removes it, and the saved key stops being used. + await call("PUT", "/api/ai/admin/config", "owner", { provider: null, mcpServers: [] }); + expect((await (await call("GET", "/api/ai/admin/config", "owner")).json()).allowedBaseUrls).toEqual(["http://gpu-box:8000"]); + await call("PUT", "/api/ai/admin/config", "owner", { provider: null, mcpServers: [], allowedBaseUrls: [] }); + expect(await (await call("GET", "/api/ai/status?courseId=sample", "ann")).json()).toMatchObject({ enabled: false }); +}); + +forEachDb("provider errors on a user's own key show the status and a fixed message, never the upstream body", async (db) => { + const logs: string[] = []; + const seen: { url: string; redirect?: RequestRedirect }[] = []; + const fakeFetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + seen.push({ url: String(input), redirect: init?.redirect }); + return new Response(JSON.stringify({ error: { message: "INTERNAL-SECRET-BODY from upstream" } }), { status: 502, headers: { "content-type": "application/json" } }); + }) as typeof fetch; + const { call } = await setup(db, undefined, { providerFactory: undefined, fetch: fakeFetch, log: (m) => logs.push(m), env: { STC_AI_ALLOWED_BASE_URLS: "http://ollama.internal:11434" } }); + expect((await call("PUT", "/api/ai/account/key", "ann", { kind: "openai", model: "m", apiKey: "k", baseUrl: "http://ollama.internal:11434/v1" })).status).toBe(200); + const text = await (await call("POST", "/api/ai/courses/sample/chat", "ann", { message: "hello" })).text(); + expect(text).not.toContain("INTERNAL-SECRET-BODY"); + const events = text.trim().split("\n").map((l) => JSON.parse(l)); + expect(events.find((e) => e.type === "error").message).toBe("502: The AI provider returned an error. Check your key, model and endpoint under Account, AI."); + expect(seen).toEqual([{ url: "http://ollama.internal:11434/v1/chat/completions", redirect: "manual" }]); + expect(logs.some((l) => l.includes("INTERNAL-SECRET-BODY") && l.includes("ann"))).toBe(true); + + // The admin's server provider keeps the full message for the owner to debug. + await call("DELETE", "/api/ai/account/key", "ann"); + await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "openai", model: "m", baseUrl: "http://127.0.0.1:11434/v1" }, mcpServers: [] }); + const t2 = await (await call("POST", "/api/ai/courses/sample/chat", "owner", { message: "hello" })).text(); + expect(t2).toContain("502: INTERNAL-SECRET-BODY"); +}); diff --git a/packages/feature-ai/test/settings.test.ts b/packages/feature-ai/test/settings.test.ts index 589c47f..6dec14a 100644 --- a/packages/feature-ai/test/settings.test.ts +++ b/packages/feature-ai/test/settings.test.ts @@ -7,7 +7,7 @@ const box = createSecretBox("test-secret"); forEachDb("server config is empty by default and hides the key", async (db) => { const r = aiSettingsRepo(db, box); - expect(await r.getServer()).toEqual({ provider: null, mcpServers: [] }); + expect(await r.getServer()).toEqual({ provider: null, mcpServers: [], allowedBaseUrls: [] }); expect(await r.serverApiKey()).toBeNull(); await r.saveServer({ provider: { kind: "anthropic", model: "claude-opus-5-5", apiKey: "sk-ant-SECRET" }, mcpServers: [] }); From cfc41c6a639baa40e563d782fe93757a6b56c799 Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 04:47:47 +0100 Subject: [PATCH 50/62] fix(ai): stdio MCP servers get PATH, HOME, LANG and their configured env only STC_SECRET, database URLs and API keys are no longer inherited by MCP child processes. Test: a fake server reports its environment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- packages/feature-ai/src/mcp/client.ts | 12 ++++++++- packages/feature-ai/test/fixtures/fake-mcp.ts | 1 + packages/feature-ai/test/mcp.test.ts | 25 +++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/packages/feature-ai/src/mcp/client.ts b/packages/feature-ai/src/mcp/client.ts index 1bdd0af..67cb316 100644 --- a/packages/feature-ai/src/mcp/client.ts +++ b/packages/feature-ai/src/mcp/client.ts @@ -25,13 +25,23 @@ function unwrap(r: RpcReply): any { return r.result; } +/** Inherited by stdio MCP servers. Everything else (STC_SECRET, database URLs, API keys) stays with the server. */ +const INHERITED_ENV = ["PATH", "HOME", "LANG"] as const; + +/** The environment for a stdio MCP server: PATH, HOME and LANG from the server, plus exactly the admin-configured variables. */ +export function mcpChildEnv(cfg: McpServerConfig, parent: Record<string, string | undefined> = process.env): Record<string, string> { + const env: Record<string, string> = {}; + for (const k of INHERITED_ENV) if (parent[k] !== undefined) env[k] = parent[k]!; + return { ...env, ...(cfg.env ?? {}) }; +} + function stdioTransport(cfg: McpServerConfig, timeoutMs: number): Transport { if (!cfg.command) throw new Error(`MCP server "${cfg.name}" has no command`); const proc = Bun.spawn([cfg.command, ...(cfg.args ?? [])], { stdin: "pipe", stdout: "pipe", stderr: "ignore", - env: { ...process.env, ...(cfg.env ?? {}) }, + env: mcpChildEnv(cfg), }); let nextId = 1; const pending = new Map<number, { resolve(v: RpcReply): void; reject(e: Error): void }>(); diff --git a/packages/feature-ai/test/fixtures/fake-mcp.ts b/packages/feature-ai/test/fixtures/fake-mcp.ts index 8ac7e4b..7f768dc 100644 --- a/packages/feature-ai/test/fixtures/fake-mcp.ts +++ b/packages/feature-ai/test/fixtures/fake-mcp.ts @@ -8,6 +8,7 @@ export function handle(msg: any): any | null { case "tools/list": return { jsonrpc: "2.0", id: msg.id, result: { tools: [{ name: "echo", description: "Echo text back", inputSchema: { type: "object", properties: { text: { type: "string" } }, required: ["text"] } }] } }; case "tools/call": + if (msg.params?.name === "env") return { jsonrpc: "2.0", id: msg.id, result: { content: [{ type: "text", text: JSON.stringify(process.env) }] } }; if (msg.params?.name !== "echo") return { jsonrpc: "2.0", id: msg.id, error: { code: -32602, message: `Unknown tool ${msg.params?.name}` } }; return { jsonrpc: "2.0", id: msg.id, result: { content: [{ type: "text", text: `echo: ${msg.params.arguments?.text}` }] } }; default: diff --git a/packages/feature-ai/test/mcp.test.ts b/packages/feature-ai/test/mcp.test.ts index 47b78d9..18356d5 100644 --- a/packages/feature-ai/test/mcp.test.ts +++ b/packages/feature-ai/test/mcp.test.ts @@ -55,3 +55,28 @@ test("server selection and tool names", () => { expect(mcpToolName("my server", "read file!")).toBe("mcp_my_server__read_file_"); expect(mcpToolName("a".repeat(40), "b".repeat(40)).length).toBeLessThanOrEqual(64); }); + +test("stdio: the child gets a minimal environment plus only its configured variables", async () => { + const saved = { ...process.env }; + Object.assign(process.env, { + STC_SECRET: "server-secret", STC_DATABASE_URL: "postgres://u:p@db/x", DATABASE_URL: "postgres://u:p@db/y", + ANTHROPIC_API_KEY: "sk-ant-x", MY_SECRET: "s", AWS_SECRET_ACCESS_KEY: "aws", LANG: "C.UTF-8", + }); + try { + const c = await connectMcp({ id: "fake", name: "Fake", transport: "stdio", command: process.execPath, args: [FAKE], env: { MCP_TOKEN: "t", STC_EXPLICIT: "configured" } }); + try { + const env = JSON.parse(await c.call("env", {})) as Record<string, string>; + for (const k of ["STC_SECRET", "STC_DATABASE_URL", "DATABASE_URL", "ANTHROPIC_API_KEY", "MY_SECRET", "AWS_SECRET_ACCESS_KEY"]) expect(env[k]).toBeUndefined(); + expect(env.MCP_TOKEN).toBe("t"); + expect(env.STC_EXPLICIT).toBe("configured"); // explicitly configured by the admin + expect(env.LANG).toBe("C.UTF-8"); + if (saved.PATH) expect(env.PATH).toBe(saved.PATH); + if (saved.HOME) expect(env.HOME).toBe(saved.HOME); + const unexpected = Object.keys(env).filter((k) => !["PATH", "HOME", "LANG", "MCP_TOKEN", "STC_EXPLICIT"].includes(k)); + expect(unexpected.filter((k) => /STC_|DATABASE|SECRET|_KEY$/i.test(k))).toEqual([]); + } finally { await c.close(); } + } finally { + for (const k of Object.keys(process.env)) if (!(k in saved)) delete process.env[k]; + Object.assign(process.env, saved); + } +}); From ead7327bb11d6339766ba0ccbe88bb6ce0c6fecc Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 04:48:56 +0100 Subject: [PATCH 51/62] fix(ai): solution and hidden-file guard runs on the canonical real path read_file and read_lesson URL-decode, posix-normalise, reject .. and absolute paths, resolve through CourseRepository.resolvePath (symlinks followed), then guard the real course-relative path and compare real paths with every manifest browser.solution. Hidden files are refused too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- packages/feature-ai/src/assistant/tools.ts | 20 +++--- .../feature-ai/src/retrieval/course-index.ts | 66 ++++++++++++++++-- packages/feature-ai/test/assistant.test.ts | 67 +++++++++++++++++++ 3 files changed, 139 insertions(+), 14 deletions(-) diff --git a/packages/feature-ai/src/assistant/tools.ts b/packages/feature-ai/src/assistant/tools.ts index 8ab1118..8cf603f 100644 --- a/packages/feature-ai/src/assistant/tools.ts +++ b/packages/feature-ai/src/assistant/tools.ts @@ -3,7 +3,7 @@ import type { ToolSpec } from "@stc/contracts"; import type { CourseRepository, LoadedCourse } from "@stc/course-format"; import type { McpServerConfig } from "../settings"; import { connectMcp, mcpToolName, type McpClient } from "../mcp/client"; -import { isSolutionPath, noteChunks, type Chunk, type CourseIndex } from "../retrieval/course-index"; +import { canonicalCoursePath, isGuardedFile, noteChunks, type Chunk, type CourseIndex } from "../retrieval/course-index"; export interface ToolSet { specs: ToolSpec[]; @@ -86,22 +86,24 @@ export function courseTools(o: { const id = str(a.lessonId); const ref = lessonChunk(id); if (!ref) return `No lesson with id "${id}". Lessons:\n${lessonList()}`; - if (isSolutionPath(course, ref.path)) return "That file is for authors only."; + const canon = ref.path ? canonicalCoursePath(o.repo, course, ref.path) : null; + if (canon && isGuardedFile(o.repo, course, canon)) return "That file is for authors only."; const md = await o.repo.readLessonMarkdown(course.id, id); if (md === null) return `Lesson "${id}" has no readable text (it may be a video without a transcript).`; sources.push(ref); return md.slice(0, MAX_READ); } case "read_file": { - const rel = str(a.path).replace(/^\/+/, ""); - if (isSolutionPath(course, rel)) return "That file is for authors only and is not available."; - const abs = o.repo.resolvePath(course.id, rel); - const info = abs ? await stat(abs).catch(() => null) : null; - if (!abs || !info?.isFile()) return `No such file in this course: ${rel}`; + // Canonicalise first (decode, normalise, follow symlinks), then guard the real path. + const canon = canonicalCoursePath(o.repo, course, str(a.path)); + if (!canon) return `No such file in this course: ${str(a.path)}`; + if (isGuardedFile(o.repo, course, canon)) return "That file is for authors only and is not available."; + const info = await stat(canon.abs).catch(() => null); + if (!info?.isFile()) return `No such file in this course: ${canon.rel}`; if (info.size > 2_000_000) return "That file is too large to read here."; - const text = await Bun.file(abs).text(); + const text = await Bun.file(canon.abs).text(); if (text.includes("\u0000")) return "That is a binary file."; - sources.push({ path: rel, kind: /\.(md|markdown)$/i.test(rel) ? "markdown" : "code", text: "" }); + sources.push({ path: canon.rel, kind: /\.(md|markdown)$/i.test(canon.rel) ? "markdown" : "code", text: "" }); return text.slice(0, MAX_READ); } case "list_notes": { diff --git a/packages/feature-ai/src/retrieval/course-index.ts b/packages/feature-ai/src/retrieval/course-index.ts index 0dadd53..61a71e0 100644 --- a/packages/feature-ai/src/retrieval/course-index.ts +++ b/packages/feature-ai/src/retrieval/course-index.ts @@ -1,6 +1,7 @@ +import { realpathSync } from "node:fs"; import { readdir, stat } from "node:fs/promises"; -import { basename, extname, join, relative, sep } from "node:path"; -import { slugify, type LoadedCourse } from "@stc/course-format"; +import { basename, extname, join, posix as posixPath, relative, sep } from "node:path"; +import { slugify, type CourseRepository, type LoadedCourse } from "@stc/course-format"; import { Bm25 } from "./bm25"; export type ChunkKind = "lesson" | "transcript" | "markdown" | "code" | "deck" | "note"; @@ -32,13 +33,68 @@ const posix = (p: string) => p.split(sep).join("/"); /** * Files only authors may see: manifest `browser.solution` files and anything under a `solution`, `solutions` * or `reference` folder (Pure Snake `js/solutions/`, Systemic Design `reference/` and `expansion/solution/`). - * The assistant never retrieves or reads them. + * The assistant never retrieves or reads them. `rel` is normalised first; for paths that come from a user or a + * model, use `canonicalCoursePath` + `isGuardedFile`, which also follow symlinks. */ export function isSolutionPath(course: LoadedCourse, rel: string): boolean { - const p = posix(rel).replace(/^\.\//, ""); + const p = normalizeRel(rel); + if (p === null) return true; // not a course-relative path at all: refuse if (p.split("/").slice(0, -1).some((s) => SOLUTION_SEGMENTS.has(s.toLowerCase()))) return true; if (/(^|\/)solution\.[a-z]+$/i.test(p)) return true; - for (const m of course.manifest?.modules ?? []) for (const l of m.lessons) if (l.browser?.solution && posix(l.browser.solution) === p) return true; + for (const m of course.manifest?.modules ?? []) for (const l of m.lessons) if (l.browser?.solution && normalizeRel(l.browser.solution) === p) return true; + return false; +} + +/** Posix-normalised course-relative path, or null when it is absolute or climbs out with `..`. */ +export function normalizeRel(raw: string): string | null { + if (raw.includes("\u0000")) return null; + const p = posixPath.normalize(raw.replace(/\\/g, "/")).replace(/^(\.\/)+/, ""); + if (!p || p === "." || p.startsWith("/") || /^[a-zA-Z]:/.test(p)) return null; + if (p.split("/").includes("..")) return null; + return p.replace(/\/+$/, ""); +} + +/** Undoes URL encoding (up to three layers). A malformed escape leaves the string as it is. */ +function decodeLayers(raw: string): string { + let s = raw; + for (let i = 0; i < 3 && /%[0-9a-fA-F]{2}/.test(s); i++) { + try { s = decodeURIComponent(s); } catch { break; } + } + return s; +} + +/** + * Canonical form of a path a user or the model asked for: URL-decoded, backslashes as slashes, posix-normalised, + * leading `./` and `/` stripped, `..` rejected, then resolved through `CourseRepository.resolvePath` (real path, + * symlinks followed, must stay inside the course root). `rel` is the real path relative to the real root. + * Null when the path is invalid, escapes the course or does not exist. + */ +export function canonicalCoursePath(repo: CourseRepository, course: LoadedCourse, raw: string): { rel: string; abs: string } | null { + const norm = normalizeRel(decodeLayers(raw.trim()).replace(/\\/g, "/").replace(/^\/+/, "")); + if (!norm) return null; + const abs = repo.resolvePath(course.id, norm); + if (!abs) return null; + let realRoot: string; + try { realRoot = realpathSync(course.root); } catch { return null; } + const rel = relative(realRoot, abs).split(sep).join("/"); + if (!rel || rel.startsWith("..")) return null; + return { rel, abs }; +} + +/** + * The guard for tool reads, on canonical paths only: solution files (by the folder and name rules on the real + * relative path, and by comparing real paths with every manifest `browser.solution`) and hidden files or folders. + */ +export function isGuardedFile(repo: CourseRepository, course: LoadedCourse, canonical: { rel: string; abs: string }): boolean { + if (canonical.rel.split("/").some((s) => s.startsWith("."))) return true; + if (isSolutionPath(course, canonical.rel)) return true; + for (const m of course.manifest?.modules ?? []) { + for (const l of m.lessons) { + if (!l.browser?.solution) continue; + const sol = repo.resolvePath(course.id, l.browser.solution); + if (sol && sol === canonical.abs) return true; + } + } return false; } diff --git a/packages/feature-ai/test/assistant.test.ts b/packages/feature-ai/test/assistant.test.ts index 8b533d4..c4e6307 100644 --- a/packages/feature-ai/test/assistant.test.ts +++ b/packages/feature-ai/test/assistant.test.ts @@ -102,3 +102,70 @@ test("my_progress reports the learner's progress per lesson", async () => { expect(out).toContain("Adding numbers (m2-text): not started"); expect(out).toMatch(/1 of \d+ lessons done/); }); + +// The solution guard must hold for every spelling of a path: it checks the canonical real path. +test("read_file refuses solutions however the path is written, including through symlinks", async () => { + const { cpSync, mkdtempSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } = await import("node:fs"); + const { tmpdir } = await import("node:os"); + const dir = mkdtempSync(join(tmpdir(), "stc-ai-guard-")); + try { + const root = join(dir, "courses/sample"); + cpSync(join(FIXTURES, "sample"), root, { recursive: true }); + const yaml = join(root, "stcourse.yaml"); + writeFileSync(yaml, readFileSync(yaml, "utf8").replace("solution: exercises/add/solution.js", "solution: exercises/add/answer.js")); + writeFileSync(join(root, "exercises/add/answer.js"), "export const add = (a, b) => a + b; // ANSWER\n"); + mkdirSync(join(root, "solutions")); + writeFileSync(join(root, "solutions/ref.js"), "// REFERENCE SOLUTION\n"); + writeFileSync(join(root, ".env"), "TOKEN=hidden\n"); + symlinkSync("exercises/add/answer.js", join(root, "peek.js")); + symlinkSync("solutions", join(root, "material")); + symlinkSync("../solutions/ref.js", join(root, "lessons/extra.js")); + const r = await createCourseRepository([join(dir, "courses")]); + const c = r.get("sample")!; + expect(c.status).toBe("ok"); + const t = courseTools({ repo: r, course: c, index: await buildCourseIndex(c, "/nonexistent"), listNotes: async () => [] }); + const read = (path: string) => t.run("read_file", { path }); + + const refused = [ + "exercises/add/answer.js", + "exercises/add/../add/answer.js", + "./exercises/add/answer.js", + "exercises//add/answer.js", + "./exercises/./add/answer.js", + "/exercises/add/answer.js", + "exercises\\add\\answer.js", + "exercises%2Fadd%2Fanswer.js", + "exercises/add/%2E%2E/add/answer.js", + "exercises%252Fadd%252Fanswer.js", + "%2E/exercises/add/answer.js", + "peek.js", + "material/ref.js", + "lessons/extra.js", + "lessons/../solutions/ref.js", + "exercises/add/solution.js", + ".env", + "lessons/../.env", + ]; + for (const p of refused) { + const out = await read(p); + expect({ p, out }).toEqual({ p, out: expect.stringMatching(/authors only|not available|No such file/) }); + expect(out).not.toContain("ANSWER"); + expect(out).not.toContain("REFERENCE"); + expect(out).not.toContain("hidden"); + } + expect(await read("exercises/add/answer.js")).toMatch(/authors only/); + expect(await read("peek.js")).toMatch(/authors only/); + expect(await read("../../etc/passwd")).toMatch(/No such file/); + // Ordinary files still read, in any spelling. + expect(await read("exercises/add/starter.js")).toContain("add"); + expect(await read("./exercises/./add/../add/starter.js")).toContain("add"); + expect(await read("exercises%2Fadd%2Fstarter.js")).toContain("add"); + expect(t.sources.at(-1)?.path).toBe("exercises/add/starter.js"); + // The index never holds them either. + const idx = await buildCourseIndex(c, "/nonexistent"); + const hits = idx.search("ANSWER REFERENCE SOLUTION add", 50); + expect(hits.filter((x) => /answer|ref\.js|peek|material|extra/.test(x.path) || x.text.includes("ANSWER") || x.text.includes("REFERENCE"))).toEqual([]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); From f3ce2e34bfa5658809dd3d9b9f9ceff35030bdfd Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 04:49:48 +0100 Subject: [PATCH 52/62] fix(ai): lessonUrl builder in one place; authoring follows core canView - Learner links go through lessonUrl/courseUrl (src/web/urls.ts). - Generation, the studio course list and file listing require an authoring right and core canView, so a global author needs a grant for a restricted course. Contract requests 8 (global author in canView) and 9 (STC_AI_ALLOWED_BASE_URLS in AppConfig). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- docs/contract-requests/ai.md | 24 +++++++++++++++-- packages/feature-ai/src/feature.ts | 19 ++++++++++---- packages/feature-ai/src/web/AssistantPage.tsx | 5 ++-- packages/feature-ai/src/web/index.ts | 1 + packages/feature-ai/src/web/urls.ts | 11 ++++++++ packages/feature-ai/test/feature.test.ts | 26 +++++++++++++++++++ packages/feature-ai/test/web.test.tsx | 10 +++++++ 7 files changed, 87 insertions(+), 9 deletions(-) create mode 100644 packages/feature-ai/src/web/urls.ts diff --git a/docs/contract-requests/ai.md b/docs/contract-requests/ai.md index 222d977..c29e6af 100644 --- a/docs/contract-requests/ai.md +++ b/docs/contract-requests/ai.md @@ -57,5 +57,25 @@ Proposal: export these paths from `@stc/core` (e.g. `cachePaths(dataDir)`) so bo ## 7. Learner lesson URL -Citations carry `lessonId`; the assistant links them to `/c/<courseId>/<lessonId>` (a guess at WP-A's route). Proposal: a `lessonUrl(courseId, lessonId)` -helper in `@stc/contracts/web` so WP-A owns the route shape. +Citations carry `lessonId`; the assistant links them to `/c/<courseId>/<lessonId>`. Every such link goes through +`lessonUrl(courseId, lessonId)` / `courseUrl(courseId)` in `packages/feature-ai/src/web/urls.ts` (also exported from +`@stc/feature-ai/web`), so a different WP-A route is a one-line change. Proposal: move that helper into +`@stc/contracts/web` so WP-A owns the route shape. + +## 8. A global `author` role in core's `canView` + +Generation and the studio file listing now require an authoring right (owner, the `author` role or a `course_authors` +row) and core's `canView(user, course, acl, isAuthor)`. `canView` only knows per-course authors (`isAuthor`), so a user +with the global `author` role cannot reach a `restricted` course without an ACL grant or a `course_authors` row. +Spec section 6 allows author rights "on all courses or per course", and the studio branch (`feature-studio` +`server/feature.ts:82-85`) currently grants the global role on every course. Proposal: core exports one +`canAuthor(user, course, acl, isAuthor)` (or `canView` treats `roles.includes("author")` like `isAuthor`), and both +the studio and the AI package call it. Until then the AI package follows core's `canView` (the stricter rule). + +## 9. `STC_AI_ALLOWED_BASE_URLS` in `AppConfig` + +Learners' own keys (BYOK) go only to `https://api.anthropic.com`, `https://api.openai.com/v1` or an origin on the +allowlist (SSRF guard). The allowlist is `STC_AI_ALLOWED_BASE_URLS` (comma-separated exact origins) plus the admin +setting `allowedBaseUrls` in the server AI config. `AppConfig` is frozen, so the package reads the variable from +`process.env` (injectable through `aiFeature(ctx, { env })`). Proposal: add `aiAllowedBaseUrls: string[]` to +`AppConfig` and `loadConfig`. diff --git a/packages/feature-ai/src/feature.ts b/packages/feature-ai/src/feature.ts index ad521f8..d17f3a7 100644 --- a/packages/feature-ai/src/feature.ts +++ b/packages/feature-ai/src/feature.ts @@ -184,8 +184,17 @@ export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feat const ok = canView(user, c, await acl.listForCourse(c.id), user ? await acl.isAuthor(c.id, user.id) : false); return ok ? c : null; }; - const canAuthor = async (user: AuthUser, courseId: string) => - user.roles.includes("owner") || user.roles.includes("author") || (await acl.isAuthor(courseId, user.id)); + /** + * Generation and the studio file listing: the user must hold an authoring right (owner, the `author` role, or a + * `course_authors` row) AND see the course under core's `canView`, the same visibility rule as everywhere else. + * A global `author` therefore cannot reach a restricted course without an ACL grant or a course_authors row + * (see contract request 8). + */ + const canAuthor = async (user: AuthUser, course: LoadedCourse) => { + const isAuthorRow = await acl.isAuthor(course.id, user.id); + const mayAuthor = user.roles.includes("owner") || user.roles.includes("author") || isAuthorRow; + return mayAuthor && canView(user, course, await acl.listForCourse(course.id), isAuthorRow); + }; const conversationOf = async (id: string, userId: string) => { const rows = await ctx.db.sql`SELECT id, user_id, course_id, title, created_at FROM ai_conversations WHERE id = ${id} AND user_id = ${userId}`; @@ -427,7 +436,7 @@ export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feat if (!rc.user) return err(401, "Sign in first."); const out = []; for (const c of ctx.courses.list()) { - if (c.status === "ok" && (await canAuthor(rc.user, c.id))) out.push({ id: c.id, title: c.manifest?.title ?? c.id, tracks: c.manifest?.tracks ?? [] }); + if (c.status === "ok" && (await canAuthor(rc.user, c))) out.push({ id: c.id, title: c.manifest?.title ?? c.id, tracks: c.manifest?.tracks ?? [] }); } return json(out); }, @@ -438,7 +447,7 @@ export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feat const courseId = rc.url.searchParams.get("courseId") ?? ""; const course = ctx.courses.get(courseId); if (!course || course.status !== "ok") return err(404, "Course not found."); - if (!(await canAuthor(rc.user, course.id))) return err(403, "Only authors can generate drafts."); + if (!(await canAuthor(rc.user, course))) return err(403, "Only authors can generate drafts."); const lessons = (course.manifest?.modules ?? []).flatMap((m) => m.lessons.map((l) => ({ id: l.id, title: l.title, type: l.type, module: m.title, hasText: l.type !== "video" || !!l.transcript }))); const files = await listFiles(course.root, 800); // authors may use any course file, solutions included const cache = await listFiles(join(ctx.config.dataDir, "cache", course.id), 400); @@ -453,7 +462,7 @@ export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feat if (!g) return err(400, "Choose a course, a kind (document, exercise or questions), a style and a topic."); const course = ctx.courses.get(g.courseId); if (!course || course.status !== "ok") return err(404, "Course not found."); - if (!(await canAuthor(rc.user, course.id))) return err(403, "Only authors can generate drafts."); + if (!(await canAuthor(rc.user, course))) return err(403, "Only authors can generate drafts."); if (!(await resolveProvider(rc.user.id))) return err(409, "AI is not set up."); const jobId = await ctx.jobs.enqueue("ai.generate", { req: g, userId: rc.user.id }); return json({ jobId }, 202); diff --git a/packages/feature-ai/src/web/AssistantPage.tsx b/packages/feature-ai/src/web/AssistantPage.tsx index 3d180a8..d9c761a 100644 --- a/packages/feature-ai/src/web/AssistantPage.tsx +++ b/packages/feature-ai/src/web/AssistantPage.tsx @@ -2,6 +2,7 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { Link, useParams } from "react-router-dom"; import { Pill, SegmentedToggle } from "@stc/ui"; import { api, readNdjson, type ChatEvent, type Citation, type Status } from "./api"; +import { courseUrl, lessonUrl } from "./urls"; import { Markdown, MentorBanner, PageHead } from "./components"; type Turn = { role: "user" | "assistant"; text: string; citations?: Citation[]; tools?: string[]; error?: string; notices?: string[] }; @@ -21,7 +22,7 @@ function Sources({ courseId, citations }: { courseId: string; citations: Citatio {rows.map((c) => ( <li key={c.path}> <span className="ai-path">{c.path}</span> - {c.lessonId ? <Link className="ai-meta" to={`/c/${encodeURIComponent(courseId)}/${encodeURIComponent(c.lessonId)}`}>{c.lessonTitle ?? c.lessonId}</Link> : <span className="ai-meta">{c.kind}</span>} + {c.lessonId ? <Link className="ai-meta" to={lessonUrl(courseId, c.lessonId)}>{c.lessonTitle ?? c.lessonId}</Link> : <span className="ai-meta">{c.kind}</span>} </li> ))} </ul> @@ -120,7 +121,7 @@ export default function AssistantPage() { const newConversation = () => { abortRef.current?.abort(); setConversationId(null); setTurns([]); }; const courseTitle = status?.course?.title ?? courseId; - const chap = <Link to={`/c/${encodeURIComponent(courseId)}`}>{courseTitle}</Link>; + const chap = <Link to={courseUrl(courseId)}>{courseTitle}</Link>; if (loadError && !status) return <div className="ai-page"><PageHead chap={chap} title="Course assistant" /><p className="ai-error">{loadError}</p></div>; if (!status) return <div className="ai-page"><PageHead chap={chap} title="Course assistant" /></div>; diff --git a/packages/feature-ai/src/web/index.ts b/packages/feature-ai/src/web/index.ts index b3c7387..181fc97 100644 --- a/packages/feature-ai/src/web/index.ts +++ b/packages/feature-ai/src/web/index.ts @@ -9,3 +9,4 @@ export const aiWeb: WebFeature = { { path: "/studio/generate", element: () => import("./GeneratePage") }, ], }; +export { courseUrl, lessonUrl } from "./urls"; diff --git a/packages/feature-ai/src/web/urls.ts b/packages/feature-ai/src/web/urls.ts new file mode 100644 index 0000000..d08fb57 --- /dev/null +++ b/packages/feature-ai/src/web/urls.ts @@ -0,0 +1,11 @@ +/** + * Learner URLs the AI pages link to. The learner app (WP-A) owns these routes; keep every builder here so + * Phase 2 can change the shape in one place (contract request 7). + */ +export function courseUrl(courseId: string): string { + return `/c/${encodeURIComponent(courseId)}`; +} + +export function lessonUrl(courseId: string, lessonId: string): string { + return `${courseUrl(courseId)}/${encodeURIComponent(lessonId)}`; +} diff --git a/packages/feature-ai/test/feature.test.ts b/packages/feature-ai/test/feature.test.ts index 5ff2293..96fa0bd 100644 --- a/packages/feature-ai/test/feature.test.ts +++ b/packages/feature-ai/test/feature.test.ts @@ -218,3 +218,29 @@ forEachDb("provider errors on a user's own key show the status and a fixed messa const t2 = await (await call("POST", "/api/ai/courses/sample/chat", "owner", { message: "hello" })).text(); expect(t2).toContain("502: INTERNAL-SECRET-BODY"); }); + +forEachDb("authoring uses core canView: a global author needs a grant for a restricted course", async (db) => { + const { call } = await setup(db); + await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "anthropic", apiKey: "k" }, mcpServers: [] }); + const body = { courseId: "secret", kind: "questions", style: "pure-snake", topic: "Adding", sources: [{ kind: "lesson", lessonId: "m2-text" }] }; + const courses = async (u: string) => ((await (await call("GET", "/api/ai/studio/courses", u)).json()) as { id: string }[]).map((c) => c.id).sort(); + // Public course: the author role is enough. Restricted course: not without a grant. + expect(await courses("auth")).toEqual(["sample"]); + expect((await call("GET", "/api/ai/studio/sources?courseId=secret", "auth")).status).toBe(403); + expect((await call("POST", "/api/ai/studio/generate", "auth", body)).status).toBe(403); + // The owner sees everything. + expect(await courses("owner")).toEqual(["sample", "secret"]); + expect((await call("POST", "/api/ai/studio/generate", "owner", body)).status).toBe(202); + // An ACL grant (core canView) lets the global author in. + await db.sql`INSERT INTO course_acl (course_id, principal_type, principal_id) VALUES ('secret', 'user', 'auth')`; + expect(await courses("auth")).toEqual(["sample", "secret"]); + expect((await call("POST", "/api/ai/studio/generate", "auth", body)).status).toBe(202); + // A course_authors row makes a learner an author of that course only. + expect((await call("POST", "/api/ai/studio/generate", "bob", body)).status).toBe(403); + await db.sql`INSERT INTO course_authors (course_id, user_id) VALUES ('secret', 'bob')`; + expect(await courses("bob")).toEqual(["secret"]); + expect((await call("POST", "/api/ai/studio/generate", "bob", body)).status).toBe(202); + // A grant to view alone does not make a learner an author. + await db.sql`INSERT INTO course_acl (course_id, principal_type, principal_id) VALUES ('secret', 'user', 'ann')`; + expect((await call("POST", "/api/ai/studio/generate", "ann", body)).status).toBe(403); +}); diff --git a/packages/feature-ai/test/web.test.tsx b/packages/feature-ai/test/web.test.tsx index 3f0d540..683652d 100644 --- a/packages/feature-ai/test/web.test.tsx +++ b/packages/feature-ai/test/web.test.tsx @@ -34,3 +34,13 @@ test("web routes", () => { expect(aiWeb.routes.map((r) => r.path)).toEqual(["/c/:courseId/assistant", "/admin/ai", "/account/ai", "/studio/generate"]); expect(aiWeb.routes.find((r) => r.path === "/admin/ai")!.nav).toEqual({ label: "AI", area: "admin" }); }); + +test("lesson links go through one builder", async () => { + const { lessonUrl, courseUrl } = await import("../src/web/urls"); + expect(lessonUrl("sample", "m2-ex-add")).toBe("/c/sample/m2-ex-add"); + expect(lessonUrl("S08 - Docker", "l 1/x")).toBe("/c/S08%20-%20Docker/l%201%2Fx"); + expect(courseUrl("sample")).toBe("/c/sample"); + const src = await Bun.file(new URL("../src/web/AssistantPage.tsx", import.meta.url)).text(); + expect(src).toContain("lessonUrl(courseId, c.lessonId)"); + expect(src).not.toMatch(/`\/c\//); +}); From 89c51f013dcdd66554e290cd0285846952875f86 Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 04:53:36 +0100 Subject: [PATCH 53/62] feat(learner): web shell, library, lesson views (LessonView), decks, code files, exercises, notes, account and admin pages Tasks 10-12 of the WP-A plan in one commit (the route modules import each other): - header extras portalled into the Phase 0 header: course switcher, search, Reading/Work, account - course frame with outline sidebar (mobile overlay, Escape closes), progress, gating locks - library: featured course, continue row, New and collection rows, course list, filters, search - LessonView: text (track switcher, folds, ordered hint tiers), video (resume, done near the end, timestamp notes, transcript), companion decks rendered with pdf.js (thumbs, arrows, fullscreen, per-slide notes), code files (tree, tabs, copy, zip, command), exercises (Web Worker runner or local command with optional pasted evidence), floating action bar - notes list, account (email/password, providers, SSO by email) and owner admin pages - pdf.js worker served from /api/learner/pdf.worker.mjs - browser entry @stc/feature-learner/web (the package index also carries the server half) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- apps/web/src/features.ts | 3 +- packages/feature-learner/package.json | 4 + packages/feature-learner/src/index.ts | 5 + packages/feature-learner/src/server/assets.ts | 20 + .../feature-learner/src/server/feature.ts | 2 + .../src/web/account/AccountPage.tsx | 132 +++++++ .../src/web/account/AdminPage.tsx | 207 ++++++++++ packages/feature-learner/src/web/api.ts | 197 +++++++++ .../feature-learner/src/web/course/pages.tsx | 232 +++++++++++ packages/feature-learner/src/web/icons.tsx | 15 + packages/feature-learner/src/web/index.ts | 17 + .../src/web/lesson/CodeFiles.tsx | 155 ++++++++ .../src/web/lesson/DeckViewer.tsx | 196 +++++++++ .../src/web/lesson/Exercise.tsx | 178 +++++++++ .../src/web/lesson/LessonView.tsx | 260 ++++++++++++ .../feature-learner/src/web/lesson/Prose.tsx | 72 ++++ .../src/web/lesson/TitleText.tsx | 7 + .../feature-learner/src/web/lesson/notes.tsx | 171 ++++++++ .../src/web/library/LibraryPage.tsx | 218 ++++++++++ .../src/web/shell/CourseFrame.tsx | 152 +++++++ .../src/web/shell/HeaderExtras.tsx | 138 +++++++ packages/feature-learner/src/web/styles.ts | 373 ++++++++++++++++++ .../feature-learner/test/account.test.tsx | 19 + packages/feature-learner/test/assets.test.ts | 11 + .../feature-learner/test/lesson-view.test.tsx | 108 +++++ .../feature-learner/test/web-shell.test.tsx | 99 +++++ packages/feature-learner/web/src/index.ts | 4 + 27 files changed, 2994 insertions(+), 1 deletion(-) create mode 100644 packages/feature-learner/src/server/assets.ts create mode 100644 packages/feature-learner/src/web/account/AccountPage.tsx create mode 100644 packages/feature-learner/src/web/account/AdminPage.tsx create mode 100644 packages/feature-learner/src/web/api.ts create mode 100644 packages/feature-learner/src/web/course/pages.tsx create mode 100644 packages/feature-learner/src/web/icons.tsx create mode 100644 packages/feature-learner/src/web/index.ts create mode 100644 packages/feature-learner/src/web/lesson/CodeFiles.tsx create mode 100644 packages/feature-learner/src/web/lesson/DeckViewer.tsx create mode 100644 packages/feature-learner/src/web/lesson/Exercise.tsx create mode 100644 packages/feature-learner/src/web/lesson/LessonView.tsx create mode 100644 packages/feature-learner/src/web/lesson/Prose.tsx create mode 100644 packages/feature-learner/src/web/lesson/TitleText.tsx create mode 100644 packages/feature-learner/src/web/lesson/notes.tsx create mode 100644 packages/feature-learner/src/web/library/LibraryPage.tsx create mode 100644 packages/feature-learner/src/web/shell/CourseFrame.tsx create mode 100644 packages/feature-learner/src/web/shell/HeaderExtras.tsx create mode 100644 packages/feature-learner/src/web/styles.ts create mode 100644 packages/feature-learner/test/account.test.tsx create mode 100644 packages/feature-learner/test/assets.test.ts create mode 100644 packages/feature-learner/test/lesson-view.test.tsx create mode 100644 packages/feature-learner/test/web-shell.test.tsx create mode 100644 packages/feature-learner/web/src/index.ts diff --git a/apps/web/src/features.ts b/apps/web/src/features.ts index 8b077c9..cdef4be 100644 --- a/apps/web/src/features.ts +++ b/apps/web/src/features.ts @@ -1,4 +1,5 @@ import type { WebFeature } from "@stc/contracts"; +import { learnerWeb } from "@stc/feature-learner/web"; // Phase 1 packages append one import line each and push into this list. -export const webFeatures: WebFeature[] = []; +export const webFeatures: WebFeature[] = [learnerWeb]; diff --git a/packages/feature-learner/package.json b/packages/feature-learner/package.json index 5da2328..807eab9 100644 --- a/packages/feature-learner/package.json +++ b/packages/feature-learner/package.json @@ -9,5 +9,9 @@ "better-auth": "^1.7.14", "marked": "^18.1.2", "pdfjs-dist": "^6.4.299" + }, + "exports": { + ".": "./src/index.ts", + "./web": "./web/src/index.ts" } } diff --git a/packages/feature-learner/src/index.ts b/packages/feature-learner/src/index.ts index 1988a66..c01208d 100644 --- a/packages/feature-learner/src/index.ts +++ b/packages/feature-learner/src/index.ts @@ -1,2 +1,7 @@ +// Server export { migrationsDirs, runMigrations } from "./server/migrate"; export { learnerFeature } from "./server/feature"; +// Browser (also available without the server half as "@stc/feature-learner/web") +export { learnerWeb, LessonView, type LessonViewProps, ensureLearnerStyles } from "./web"; +// Shared JSON shapes +export type * from "./shared/types"; diff --git a/packages/feature-learner/src/server/assets.ts b/packages/feature-learner/src/server/assets.ts new file mode 100644 index 0000000..8a41496 --- /dev/null +++ b/packages/feature-learner/src/server/assets.ts @@ -0,0 +1,20 @@ +import type { Feature } from "@stc/contracts"; + +/** The pdf.js worker from the installed pdfjs-dist, for client-side deck rendering. */ +export function pdfWorkerPath(): string { + return Bun.resolveSync("pdfjs-dist/build/pdf.worker.min.mjs", import.meta.dir); +} + +export function assetRoutes(): Feature["routes"] { + let path: string | null = null; + return { + "/api/learner/pdf.worker.mjs": { + GET: async () => { + path ??= pdfWorkerPath(); + return new Response(Bun.file(path), { + headers: { "Content-Type": "text/javascript; charset=utf-8", "Cache-Control": "public, max-age=86400" }, + }); + }, + }, + }; +} diff --git a/packages/feature-learner/src/server/feature.ts b/packages/feature-learner/src/server/feature.ts index 513bc79..0d2c512 100644 --- a/packages/feature-learner/src/server/feature.ts +++ b/packages/feature-learner/src/server/feature.ts @@ -8,6 +8,7 @@ import { lessonRoutes } from "./lesson"; import { userDataRoutes } from "./userdata"; import { codeRoutes } from "./code"; import { transpileRoutes } from "./transpile"; +import { assetRoutes } from "./assets"; /** * The learner feature: auth, library, reader, notes, progress and exercises. @@ -32,6 +33,7 @@ export function learnerFeature(ctx: ServerContext): Feature & { auth: AuthServic ...userDataRoutes(ctx), ...codeRoutes(ctx), ...transpileRoutes(), + ...assetRoutes(), }; // Every handler waits for the learner migrations (a no-op once Phase 2 runs them in migrate()). for (const methods of Object.values(routes)) { diff --git a/packages/feature-learner/src/web/account/AccountPage.tsx b/packages/feature-learner/src/web/account/AccountPage.tsx new file mode 100644 index 0000000..be89024 --- /dev/null +++ b/packages/feature-learner/src/web/account/AccountPage.tsx @@ -0,0 +1,132 @@ +import { useEffect, useState, type FormEvent } from "react"; +import { Link, useLocation, useNavigate } from "react-router-dom"; +import { Pill, SegmentedToggle } from "@stc/ui"; +import { api, clearCourseCache, ensureStyles, refreshLibrary, refreshMe, useApi, useMe } from "../api"; +import { HeaderExtras } from "../shell/HeaderExtras"; + +export interface ProvidersPayload { emailPassword: boolean; providers: { id: string; label: string; kind: "social" | "oidc" }[]; sso: boolean } + +async function afterAuthChange() { + clearCourseCache(); + await Promise.all([refreshMe(), refreshLibrary()]); +} + +/** Only same-site paths are allowed as a return target. */ +export function safeNext(raw: string | null): string { + return raw && raw.startsWith("/") && !raw.startsWith("//") ? raw : "/"; +} + +export default function AccountPage() { + ensureStyles(); + const { loaded, user } = useMe(); + useEffect(() => { document.title = "Account – STCourses"; }, []); + return ( + <div className="lr" data-look="work"> + <HeaderExtras course={null} lookMode={null} /> + <article style={{ maxWidth: 660, margin: "0 auto", paddingBottom: 60 }}> + {!loaded ? null : user ? <SignedIn /> : <SignInForms />} + </article> + </div> + ); +} + +function SignedIn() { + const { user } = useMe(); + const navigate = useNavigate(); + const [busy, setBusy] = useState(false); + if (!user) return null; + const signOut = async () => { + setBusy(true); + try { await api("/api/auth/sign-out", { method: "POST", body: {} }); } catch { /* the session may already be gone */ } + await afterAuthChange(); + setBusy(false); + navigate("/"); + }; + return ( + <> + <h1 className="lib-h">Your account</h1> + <dl className="acct-dl"> + <dt>Name</dt><dd>{user.name || "—"}</dd> + <dt>Email</dt><dd>{user.email}</dd> + <dt>Roles</dt><dd>{user.roles.join(", ") || "learner"}</dd> + {user.groups.length ? <><dt>Groups</dt><dd>{user.groups.join(", ")}</dd></> : null} + </dl> + <div style={{ display: "flex", gap: 8, flexWrap: "wrap" }}> + {user.roles.includes("owner") ? <Link className="stc-pill" to="/account/admin">Users, groups and library</Link> : null} + <Pill onClick={signOut} disabled={busy}>Sign out</Pill> + </div> + </> + ); +} + +export function SignInForms() { + const loc = useLocation(); + const navigate = useNavigate(); + const next = safeNext(new URLSearchParams(loc.search).get("next")); + const { data: prov } = useApi<ProvidersPayload>("/api/learner/providers"); + const [mode, setMode] = useState<"in" | "up">("in"); + const [name, setName] = useState(""); + const [email, setEmail] = useState(""); + const [password, setPassword] = useState(""); + const [ssoEmail, setSsoEmail] = useState(""); + const [err, setErr] = useState<string | null>(null); + const [busy, setBusy] = useState(false); + + const submit = async (e: FormEvent) => { + e.preventDefault(); + setErr(null); setBusy(true); + try { + if (mode === "in") await api("/api/auth/sign-in/email", { method: "POST", body: { email, password } }); + else await api("/api/auth/sign-up/email", { method: "POST", body: { name: name.trim() || email.split("@")[0], email, password } }); + await afterAuthChange(); + navigate(next); + } catch (e) { + setErr(mode === "in" ? "That email and password do not match an account." : (e as Error).message); + } finally { setBusy(false); } + }; + const redirect = async (path: string, body: Record<string, unknown>) => { + setErr(null); + try { + const callbackURL = new URL(next, location.origin).toString(); + const r = await api<{ url?: string }>(path, { method: "POST", body: { ...body, callbackURL } }); + if (r?.url) location.assign(r.url); else setErr("The sign-in provider did not answer."); + } catch (e) { setErr((e as Error).message); } + }; + + return ( + <> + <h1 className="lib-h">{mode === "in" ? "Sign in" : "Create an account"}</h1> + <p className="lib-sub">Your progress, notes and exercise code are kept with your account.</p> + <div style={{ marginBottom: 20 }}> + <SegmentedToggle label="Sign in or create an account" value={mode} onChange={(v) => { setMode(v as "in" | "up"); setErr(null); }} + options={[{ value: "in", label: "Sign in" }, { value: "up", label: "Create account" }]} /> + </div> + <form className="form" onSubmit={submit}> + {mode === "up" ? <label>Name<input value={name} onChange={(e) => setName(e.target.value)} autoComplete="name" /></label> : null} + <label>Email<input type="email" required value={email} onChange={(e) => setEmail(e.target.value)} autoComplete="email" /></label> + <label>Password<input type="password" required minLength={8} value={password} onChange={(e) => setPassword(e.target.value)} autoComplete={mode === "in" ? "current-password" : "new-password"} /></label> + {err ? <p className="err" role="alert">{err}</p> : null} + <div><Pill type="submit" variant="primary" disabled={busy}>{mode === "in" ? "Sign in" : "Create account"}</Pill></div> + {mode === "up" ? <p className="muted" style={{ fontSize: 12.5, margin: 0 }}>The first account on a new server becomes its owner.</p> : null} + </form> + {prov && (prov.providers.length || prov.sso) ? ( + <> + <div className="or">or</div> + <div className="prov"> + {prov.providers.map((p) => ( + <Pill key={p.id} onClick={() => void (p.kind === "oidc" ? redirect("/api/auth/sign-in/oauth2", { providerId: p.id }) : redirect("/api/auth/sign-in/social", { provider: p.id }))}> + Continue with {p.label} + </Pill> + ))} + </div> + {prov.sso ? ( + <form className="form" style={{ marginTop: 18 }} onSubmit={(e) => { e.preventDefault(); void redirect("/api/auth/sign-in/sso", { email: ssoEmail }); }}> + <label>Work email for single sign-on<input type="email" required value={ssoEmail} onChange={(e) => setSsoEmail(e.target.value)} /></label> + <div><Pill type="submit">Continue with single sign-on</Pill></div> + </form> + ) : null} + </> + ) : null} + </> + ); +} diff --git a/packages/feature-learner/src/web/account/AdminPage.tsx b/packages/feature-learner/src/web/account/AdminPage.tsx new file mode 100644 index 0000000..632c12b --- /dev/null +++ b/packages/feature-learner/src/web/account/AdminPage.tsx @@ -0,0 +1,207 @@ +import { useEffect, useState } from "react"; +import { Link } from "react-router-dom"; +import { Pill } from "@stc/ui"; +import type { Role } from "@stc/contracts"; +import { api, ensureStyles, refreshLibrary, toast, useApi, useLibrary, useMe } from "../api"; +import { HeaderExtras } from "../shell/HeaderExtras"; + +interface AdminUser { id: string; email: string; name: string; createdAt: string; roles: Role[]; groups: string[] } +interface AdminGroup { id: string; name: string; members: string[] } +interface Collection { id: string; title: string; sort: number; courseIds: string[] } +interface Access { visibility: string; acl: { principal_type: "user" | "group"; principal_id: string }[]; authors: string[] } + +const ROLES: Role[] = ["owner", "author", "learner"]; + +/** Owner administration: roles, groups, course access and library curation. */ +export default function AdminPage() { + ensureStyles(); + const { loaded, user } = useMe(); + useEffect(() => { document.title = "Administration – STCourses"; }, []); + const owner = !!user?.roles.includes("owner"); + return ( + <div className="lr" data-look="work"> + <HeaderExtras course={null} lookMode={null} /> + <article style={{ maxWidth: 900, margin: "0 auto", paddingBottom: 80 }}> + <p className="chap"><Link to="/account">Account</Link></p> + <h1 className="lib-h">Users, groups and library</h1> + {!loaded ? null : !owner ? <p className="lib-sub">Only the owner of this server can open this page.</p> : <AdminSections />} + </article> + </div> + ); +} + +function AdminSections() { + const users = useApi<AdminUser[]>("/api/admin/users"); + const groups = useApi<AdminGroup[]>("/api/admin/groups"); + return ( + <> + <p className="lib-sub">Sign-in providers are configured on the server with STC_AUTH_* settings.</p> + <UsersSection users={users.data ?? []} reload={users.reload} /> + <GroupsSection groups={groups.data ?? []} users={users.data ?? []} reload={() => { groups.reload(); users.reload(); }} /> + <LibrarySection /> + <AccessSection users={users.data ?? []} groups={groups.data ?? []} /> + </> + ); +} + +function UsersSection({ users, reload }: { users: AdminUser[]; reload: () => void }) { + const setRoles = async (u: AdminUser, role: Role, on: boolean) => { + const roles = on ? [...u.roles, role] : u.roles.filter((r) => r !== role); + try { await api(`/api/admin/users/${encodeURIComponent(u.id)}/roles`, { method: "PUT", body: { roles } }); reload(); } + catch (e) { toast((e as Error).message); } + }; + return ( + <section className="admin-sec" aria-labelledby="adm-users"> + <h2 id="adm-users">Users and roles</h2> + <table className="table"> + <thead><tr><th>User</th><th>Roles</th><th>Groups</th></tr></thead> + <tbody> + {users.map((u) => ( + <tr key={u.id}> + <td><div>{u.name || u.email}</div><span className="ls">{u.email}</span></td> + <td><div className="checks">{ROLES.map((r) => ( + <label key={r}><input type="checkbox" checked={u.roles.includes(r)} onChange={(e) => setRoles(u, r, e.target.checked)} />{r}</label> + ))}</div></td> + <td><span className="ls">{u.groups.join(", ")}</span></td> + </tr> + ))} + </tbody> + </table> + </section> + ); +} + +function GroupsSection({ groups, users, reload }: { groups: AdminGroup[]; users: AdminUser[]; reload: () => void }) { + const [name, setName] = useState(""); + const create = async () => { + try { await api("/api/admin/groups", { method: "POST", body: { name } }); setName(""); reload(); } catch (e) { toast((e as Error).message); } + }; + const setMembers = async (g: AdminGroup, userIds: string[]) => { + try { await api(`/api/admin/groups/${encodeURIComponent(g.id)}/members`, { method: "PUT", body: { userIds } }); reload(); } catch (e) { toast((e as Error).message); } + }; + const remove = async (g: AdminGroup) => { + if (!confirm(`Delete the group “${g.name}”? Courses shared with it lose that access.`)) return; + try { await api(`/api/admin/groups/${encodeURIComponent(g.id)}`, { method: "DELETE" }); reload(); } catch (e) { toast((e as Error).message); } + }; + return ( + <section className="admin-sec" aria-labelledby="adm-groups"> + <h2 id="adm-groups">Groups</h2> + <p className="source" style={{ marginTop: 0 }}>Groups give access to restricted courses. Groups named in OIDC or SAML claims are filled at sign-in.</p> + {groups.length ? ( + <table className="table"> + <thead><tr><th>Group</th><th>Members</th><th /></tr></thead> + <tbody> + {groups.map((g) => ( + <tr key={g.id}> + <td>{g.name}</td> + <td><div className="checks">{users.map((u) => ( + <label key={u.id}><input type="checkbox" checked={g.members.includes(u.id)} + onChange={(e) => setMembers(g, e.target.checked ? [...g.members, u.id] : g.members.filter((x) => x !== u.id))} />{u.name || u.email}</label> + ))}</div></td> + <td style={{ textAlign: "right" }}><Pill className="sm" variant="ghost" onClick={() => remove(g)}>Delete</Pill></td> + </tr> + ))} + </tbody> + </table> + ) : null} + <div className="row-form"> + <input className="field" placeholder="New group name" value={name} onChange={(e) => setName(e.target.value)} aria-label="New group name" /> + <Pill className="sm" disabled={!name.trim()} onClick={create}>Add group</Pill> + </div> + </section> + ); +} + +function LibrarySection() { + const lib = useLibrary(); + const cur = useApi<{ featured: string | null; collections: Collection[] }>("/api/admin/library"); + const [featured, setFeatured] = useState<string>(""); + const [cols, setCols] = useState<Collection[]>([]); + const [dirty, setDirty] = useState(false); + useEffect(() => { + if (!cur.data) return; + setFeatured(cur.data.featured ?? ""); setCols(cur.data.collections); setDirty(false); + }, [cur.data]); + const courses = lib?.courses ?? []; + const save = async () => { + try { + await api("/api/admin/library", { method: "PUT", body: { featured: featured || null, collections: cols.filter((c) => c.title.trim()).map((c, i) => ({ ...c, sort: i })) } }); + cur.reload(); void refreshLibrary(); toast("Library saved"); + } catch (e) { toast((e as Error).message); } + }; + const edit = (i: number, c: Partial<Collection>) => { setCols(cols.map((x, k) => (k === i ? { ...x, ...c } : x))); setDirty(true); }; + return ( + <section className="admin-sec" aria-labelledby="adm-lib"> + <h2 id="adm-lib">Library</h2> + <div className="row-form" style={{ marginTop: 0 }}> + <label className="ls" htmlFor="adm-feat">Featured course</label> + <select id="adm-feat" className="field" value={featured} onChange={(e) => { setFeatured(e.target.value); setDirty(true); }}> + <option value="">Automatic (what you were last working on, else the newest)</option> + {courses.map((c) => <option key={c.id} value={c.id}>{c.title}</option>)} + </select> + </div> + <h3 className="sec-h" style={{ marginTop: 24 }}>Collections</h3> + <p className="source" style={{ marginTop: 0 }}>Each collection is a row in the library. A course can be in several. Courses can also name collections in their manifest.</p> + <table className="table"> + <tbody> + {cols.map((c, i) => ( + <tr key={c.id || `new-${i}`}> + <td style={{ width: 220 }}><input className="field" value={c.title} aria-label="Collection title" onChange={(e) => edit(i, { title: e.target.value })} /></td> + <td><div className="checks">{courses.map((co) => ( + <label key={co.id}><input type="checkbox" checked={c.courseIds.includes(co.id)} + onChange={(e) => edit(i, { courseIds: e.target.checked ? [...c.courseIds, co.id] : c.courseIds.filter((x) => x !== co.id) })} />{co.title}</label> + ))}</div></td> + <td style={{ textAlign: "right" }}><Pill className="sm" variant="ghost" onClick={() => { setCols(cols.filter((_, k) => k !== i)); setDirty(true); }}>Remove</Pill></td> + </tr> + ))} + </tbody> + </table> + <div className="row-form"> + <Pill className="sm" onClick={() => { setCols([...cols, { id: "", title: "", sort: cols.length, courseIds: [] }]); setDirty(true); }}>Add collection</Pill> + <Pill className="sm" variant="primary" disabled={!dirty} onClick={save}>Save library</Pill> + </div> + </section> + ); +} + +function AccessSection({ users, groups }: { users: AdminUser[]; groups: AdminGroup[] }) { + const lib = useLibrary(); + const [courseId, setCourseId] = useState(""); + const access = useApi<Access>(courseId ? `/api/admin/courses/${encodeURIComponent(courseId)}/access` : null); + const [draft, setDraft] = useState<Access | null>(null); + useEffect(() => { setDraft(access.data); }, [access.data]); + const has = (t: "user" | "group", id: string) => !!draft?.acl.some((a) => a.principal_type === t && a.principal_id === id); + const toggle = (t: "user" | "group", id: string, on: boolean) => draft && setDraft({ + ...draft, acl: on ? [...draft.acl, { principal_type: t, principal_id: id }] : draft.acl.filter((a) => !(a.principal_type === t && a.principal_id === id)), + }); + const save = async () => { + if (!draft) return; + try { await api(`/api/admin/courses/${encodeURIComponent(courseId)}/access`, { method: "PUT", body: { acl: draft.acl, authors: draft.authors } }); access.reload(); toast("Access saved"); } + catch (e) { toast((e as Error).message); } + }; + return ( + <section className="admin-sec" aria-labelledby="adm-access"> + <h2 id="adm-access">Course access</h2> + <p className="source" style={{ marginTop: 0 }}>Restricted courses are visible to the users and groups listed here. Authors can edit a course in the studio.</p> + <div className="row-form" style={{ marginTop: 0 }}> + <select className="field" value={courseId} onChange={(e) => setCourseId(e.target.value)} aria-label="Course"> + <option value="">Choose a course</option> + {(lib?.courses ?? []).map((c) => <option key={c.id} value={c.id}>{c.title}</option>)} + </select> + {draft ? <span className="ls">Visibility in the manifest: {draft.visibility}</span> : null} + </div> + {draft ? ( + <> + <table className="table" style={{ marginTop: 14 }}> + <tbody> + <tr><td style={{ width: 120 }} className="ls">Users</td><td><div className="checks">{users.map((u) => <label key={u.id}><input type="checkbox" checked={has("user", u.id)} onChange={(e) => toggle("user", u.id, e.target.checked)} />{u.name || u.email}</label>)}</div></td></tr> + <tr><td className="ls">Groups</td><td><div className="checks">{groups.length ? groups.map((g) => <label key={g.id}><input type="checkbox" checked={has("group", g.id)} onChange={(e) => toggle("group", g.id, e.target.checked)} />{g.name}</label>) : <span className="ls">No groups yet</span>}</div></td></tr> + <tr><td className="ls">Authors</td><td><div className="checks">{users.map((u) => <label key={u.id}><input type="checkbox" checked={draft.authors.includes(u.id)} onChange={(e) => setDraft({ ...draft, authors: e.target.checked ? [...draft.authors, u.id] : draft.authors.filter((x) => x !== u.id) })} />{u.name || u.email}</label>)}</div></td></tr> + </tbody> + </table> + <div className="row-form"><Pill className="sm" variant="primary" onClick={save}>Save access</Pill></div> + </> + ) : null} + </section> + ); +} diff --git a/packages/feature-learner/src/web/api.ts b/packages/feature-learner/src/web/api.ts new file mode 100644 index 0000000..9826e1a --- /dev/null +++ b/packages/feature-learner/src/web/api.ts @@ -0,0 +1,197 @@ +import { useCallback, useEffect, useRef, useState, useSyncExternalStore } from "react"; +import type { AuthUser } from "@stc/contracts"; +import type { CoursePayload, LibraryPayload } from "../shared/types"; +import { learnerCss } from "./styles"; + +/** Injects the learner stylesheet once (the module graph stays importable on the server). */ +export function ensureStyles(): void { + if (typeof document === "undefined" || document.getElementById("stc-learner-css")) return; + const el = document.createElement("style"); + el.id = "stc-learner-css"; + el.textContent = learnerCss; + document.head.append(el); +} +export { learnerCss }; + +export class HttpError extends Error { + constructor(public status: number, message: string) { super(message); } +} + +export async function api<T>(path: string, init: { method?: string; body?: unknown } = {}): Promise<T> { + const r = await fetch(path, { + method: init.method ?? "GET", + credentials: "same-origin", + headers: init.body !== undefined ? { "content-type": "application/json" } : undefined, + body: init.body !== undefined ? JSON.stringify(init.body) : undefined, + }); + const text = await r.text(); + let data: unknown = null; + try { data = text ? JSON.parse(text) : null; } catch { data = text; } + if (!r.ok) { + const msg = (data && typeof data === "object" && ("error" in data || "message" in data)) + ? String((data as { error?: string; message?: string }).error ?? (data as { message?: string }).message) + : r.status === 404 ? "Not found" : `Request failed (${r.status})`; + throw new HttpError(r.status, msg); + } + return data as T; +} + +/** Loads `path` (null skips). `reload()` refetches without clearing the current data. */ +export function useApi<T>(path: string | null): { data: T | null; error: HttpError | null; loading: boolean; reload: () => void; set: (v: T) => void } { + const [state, setState] = useState<{ path: string | null; data: T | null; error: HttpError | null; loading: boolean }>({ path, data: null, error: null, loading: !!path }); + const [n, setN] = useState(0); + const current = state.path === path ? state : { path, data: null, error: null, loading: !!path }; + useEffect(() => { + if (!path) return; + let live = true; + setState((s) => (s.path === path ? { ...s, loading: true } : { path, data: null, error: null, loading: true })); + api<T>(path).then( + (data) => live && setState({ path, data, error: null, loading: false }), + (e: unknown) => live && setState({ path, data: null, error: e instanceof HttpError ? e : new HttpError(0, String(e)), loading: false }), + ); + return () => { live = false; }; + }, [path, n]); + const reload = useCallback(() => setN((x) => x + 1), []); + const set = useCallback((v: T) => setState((s) => ({ ...s, data: v })), []); + return { data: current.data, error: current.error, loading: current.loading, reload, set }; +} + +/* ---------- tiny shared stores ---------- */ + +function store<T>(initial: T) { + let value = initial; + const subs = new Set<() => void>(); + return { + get: () => value, + set(v: T) { if (v !== value) { value = v; subs.forEach((f) => f()); } }, + subscribe(f: () => void) { subs.add(f); return () => { subs.delete(f); }; }, + }; +} +function useStore<T>(s: ReturnType<typeof store<T>>): T { + return useSyncExternalStore(s.subscribe, s.get, s.get); +} + +const ls = { + get(k: string): string | null { try { return localStorage.getItem(k); } catch { return null; } }, + set(k: string, v: string) { try { localStorage.setItem(k, v); } catch { /* private mode */ } }, +}; + +export type Look = "reading" | "work"; +const lookStore = store<Look>((typeof window !== "undefined" && ls.get("stc.look") === "work") ? "work" : "reading"); +export function useLook(): [Look, (l: Look) => void] { + const v = useStore(lookStore); + return [v, (l) => { ls.set("stc.look", l); lookStore.set(l); }]; +} + +const searchStore = store(""); +export function useSearch(): [string, (q: string) => void] { + return [useStore(searchStore), searchStore.set]; +} + +const trackStore = store(0); +export function useTrack(courseId: string, tracks: { id: string }[]): [string | null, (t: string) => void] { + useStore(trackStore); + const saved = typeof window !== "undefined" ? ls.get(`stc.track.${courseId}`) : null; + const value = tracks.find((t) => t.id === saved)?.id ?? tracks[0]?.id ?? null; + return [value, (t) => { ls.set(`stc.track.${courseId}`, t); trackStore.set(trackStore.get() + 1); }]; +} + +/* ---------- session and caches ---------- */ + +const meStore = store<{ loaded: boolean; user: AuthUser | null }>({ loaded: false, user: null }); +let meLoading: Promise<void> | null = null; +export function refreshMe(): Promise<void> { + meLoading = api<{ user: AuthUser | null }>("/api/me").then( + (r) => meStore.set({ loaded: true, user: r.user }), + () => meStore.set({ loaded: true, user: null }), + ); + return meLoading; +} +export function useMe(): { loaded: boolean; user: AuthUser | null } { + const v = useStore(meStore); + useEffect(() => { if (!v.loaded && !meLoading) void refreshMe(); }, [v.loaded]); + return v; +} + +const libraryStore = store<LibraryPayload | null>(null); +let libraryLoading: Promise<void> | null = null; +export function refreshLibrary(): Promise<void> { + libraryLoading = api<LibraryPayload>("/api/library").then((l) => libraryStore.set(l), () => { /* keep stale */ }); + return libraryLoading; +} +export function useLibrary(): LibraryPayload | null { + const v = useStore(libraryStore); + useEffect(() => { if (!libraryLoading) void refreshLibrary(); }, []); + return v; +} + +const courseCache = new Map<string, CoursePayload>(); +const courseVersion = store(0); +/** The course outline with progress. Cached across route changes; `reload` refetches. */ +export function useCourse(courseId: string): { course: CoursePayload | null; error: HttpError | null; reload: () => Promise<void> } { + useStore(courseVersion); + const [error, setError] = useState<HttpError | null>(null); + const live = useRef(true); + const reload = useCallback(async () => { + try { + const c = await api<CoursePayload>(`/api/courses/${encodeURIComponent(courseId)}`); + courseCache.set(courseId, c); + if (live.current) setError(null); + courseVersion.set(courseVersion.get() + 1); + } catch (e) { + if (live.current) setError(e instanceof HttpError ? e : new HttpError(0, String(e))); + } + }, [courseId]); + useEffect(() => { + live.current = true; + setError(null); + void reload(); + return () => { live.current = false; }; + }, [reload]); + return { course: courseCache.get(courseId) ?? null, error, reload }; +} +export function clearCourseCache(): void { courseCache.clear(); courseVersion.set(courseVersion.get() + 1); } + +/* ---------- toast ---------- */ + +const toastStore = store<string | null>(null); +let toastTimer: ReturnType<typeof setTimeout> | undefined; +export function toast(msg: string): void { + toastStore.set(msg); + clearTimeout(toastTimer); + toastTimer = setTimeout(() => toastStore.set(null), 2600); +} +export function useToast(): string | null { return useStore(toastStore); } + +/* ---------- formatting ---------- */ + +export function fmtTime(s: number): string { + const t = Math.max(0, Math.floor(s)); + const h = Math.floor(t / 3600), m = Math.floor((t % 3600) / 60), sec = t % 60; + return h ? `${h}:${String(m).padStart(2, "0")}:${String(sec).padStart(2, "0")}` : `${m}:${String(sec).padStart(2, "0")}`; +} +export function fmtMinutes(min: number | null | undefined): string { + if (!min) return ""; + if (min < 60) return `${Math.round(min)} min`; + const h = Math.floor(min / 60), m = Math.round(min % 60); + return m ? `${h} h ${m} min` : `${h} h`; +} +export function fmtSize(n: number | null | undefined): string { + if (n === null || n === undefined) return ""; + if (n < 1024) return `${n} B`; + if (n < 1024 * 1024) return `${Math.round(n / 1024)} KB`; + return `${(n / 1024 / 1024).toFixed(1)} MB`; +} +export const lessonHref = (courseId: string, lessonId: string) => `/c/${encodeURIComponent(courseId)}/${encodeURIComponent(lessonId)}`; +export const courseHref = (courseId: string) => `/c/${encodeURIComponent(courseId)}`; + +export async function copyText(text: string): Promise<boolean> { + try { await navigator.clipboard.writeText(text); return true; } catch { + const t = document.createElement("textarea"); + t.value = text; document.body.append(t); t.select(); + let ok = false; + try { ok = document.execCommand("copy"); } catch { /* unsupported */ } + t.remove(); + return ok; + } +} diff --git a/packages/feature-learner/src/web/course/pages.tsx b/packages/feature-learner/src/web/course/pages.tsx new file mode 100644 index 0000000..39eb4c9 --- /dev/null +++ b/packages/feature-learner/src/web/course/pages.tsx @@ -0,0 +1,232 @@ +import { useCallback, useEffect, useMemo, type CSSProperties } from "react"; +import { Link, useLocation, useNavigate, useParams } from "react-router-dom"; +import { CourseCover, FloatingActionBar, Pill, ProgressBar } from "@stc/ui"; +import type { CoursePayload, LessonPayload, Note, ProgressState } from "../../shared/types"; +import { api, ensureStyles, fmtMinutes, fmtTime, lessonHref, refreshLibrary, toast, useApi, useCourse, useLook, useMe, useTrack } from "../api"; +import { Check, Next, Prev } from "../icons"; +import { CourseFrame, courseDone, moduleNumber } from "../shell/CourseFrame"; +import { HeaderExtras } from "../shell/HeaderExtras"; +import { LessonView, TitleText } from "../lesson/LessonView"; +import { NoteItem } from "../lesson/notes"; + +function Problem({ title, children }: { title: string; children?: React.ReactNode }) { + ensureStyles(); + return ( + <div className="lr" data-look="work"> + <HeaderExtras course={null} lookMode={null} /> + <article className="col" style={{ maxWidth: 660, margin: "0 auto" }}> + <h1 className="lib-h">{title}</h1> + <div className="lib-sub">{children}</div> + <Link className="stc-pill" to="/">Back to the library</Link> + </article> + </div> + ); +} + +function CourseMissing({ status, signedIn }: { status: number; signedIn: boolean }) { + if (status === 404) { + return ( + <Problem title="Course not found"> + This course does not exist, or you do not have access to it.{signedIn ? "" : <> <Link to="/account">Signing in</Link> may help.</>} + </Problem> + ); + } + return <Problem title="Something went wrong">The course could not be loaded. Try again in a moment.</Problem>; +} + +/** First lesson that is not done (in an unlocked module), or the first lesson. */ +export function nextLesson(c: CoursePayload): { id: string; title: string } | null { + for (const m of c.modules) { + if (m.locked) break; + for (const l of m.lessons) if (c.progress[l.id]?.state !== "done") return l; + } + return c.modules[0]?.lessons[0] ?? null; +} + +/* ---------- /c/:courseId ---------- */ + +export function CoursePage() { + const { courseId = "" } = useParams(); + const { course, error } = useCourse(courseId); + const { user } = useMe(); + const [look] = useLook(); + useEffect(() => { if (course) document.title = `${course.title} – STCourses`; }, [course]); + if (error && !course) return <CourseMissing status={error.status} signedIn={!!user} />; + if (!course) return null; + const { done, total } = courseDone(course); + const next = nextLesson(course); + const started = Object.keys(course.progress).length > 0; + const minutes = course.modules.flatMap((m) => m.lessons).reduce((s, l) => s + (l.minutes ?? 0), 0); + return ( + <CourseFrame course={course} currentLessonId={null} look={look} lookMode={null}> + <article className="col" data-look="work" style={{ paddingBottom: 60 }}> + <div style={{ display: "flex", gap: 24, alignItems: "flex-end" }}> + <span style={{ "--cw": "120px" } as CSSProperties}><CourseCover title={course.title} image={course.coverUrl ?? undefined} accent={course.accent ?? undefined} /></span> + <div style={{ minWidth: 0 }}> + <h1 className="title" style={{ fontFamily: "var(--sans)" }}>{course.title}</h1> + {course.authors.length ? <p className="source">{course.authors.join(", ")}</p> : null} + </div> + </div> + {course.pitch ? <p style={{ fontSize: 16, color: "var(--muted)", margin: "22px 0 0" }}>{course.pitch}</p> : null} + <dl className="meta"> + <div><dt>Lessons</dt><dd className="tnum">{total}</dd></div> + {minutes ? <div><dt>Time</dt><dd>About {fmtMinutes(minutes)}</dd></div> : null} + {course.tracks.length > 1 ? <div><dt>Tracks</dt><dd>{course.tracks.map((t) => t.label).join(", ")}</dd></div> : null} + {course.gating === "sequential" ? <div><dt>Order</dt><dd>Each module opens when the one before it is done</dd></div> : null} + {course.signedIn ? <div style={{ minWidth: 180 }}><dt>Progress</dt><dd className="lr-prog" style={{ marginTop: 4 }}><ProgressBar value={total ? done / total : 0} label="Course progress" /><span className="tnum">{done} of {total}</span></dd></div> : null} + </dl> + {course.status === "error" ? <div className="notice"><strong>This course has problems.</strong> {course.canEdit ? "The list in the sidebar names them." : "Some lessons may not open."}</div> : null} + <div style={{ display: "flex", gap: 8, marginTop: 24, alignItems: "center", flexWrap: "wrap" }}> + {next ? <Link className="stc-pill stc-pill--primary" to={lessonHref(course.id, next.id)}>{started ? "Continue" : "Start"}{started ? <span style={{ fontWeight: 400 }}>: {next.title.replace(/`/g, "")}</span> : null}</Link> : null} + {!course.signedIn ? <span className="source" style={{ margin: 0 }}><Link to="/account">Sign in</Link> to keep your progress and notes.</span> : null} + </div> + <h2 className="sec-h" style={{ marginTop: 40 }}>Modules</h2> + <div className="lib"> + {course.modules.map((m) => { + const d = m.lessons.filter((l) => course.progress[l.id]?.state === "done").length; + return ( + <Link key={m.id} className="lib-row" to={lessonHref(course.id, m.lessons[0]?.id ?? "")} style={{ gridTemplateColumns: "32px minmax(0,1fr) 120px" }}> + <span className="lnum tnum">{moduleNumber(course, m.index)}</span> + <span style={{ minWidth: 0 }}><span className="lt">{m.title}</span><span className="ls">{m.lessons.length === 1 ? "1 lesson" : `${m.lessons.length} lessons`}{m.locked ? " · locked" : ""}</span></span> + {course.signedIn ? <span className="lr-prog" style={{ margin: 0 }}><ProgressBar value={m.lessons.length ? d / m.lessons.length : 0} dim label={`${m.title} progress`} /></span> : <span />} + </Link> + ); + })} + </div> + </article> + </CourseFrame> + ); +} + +/* ---------- /c/:courseId/:lessonId ---------- */ + +export function LessonPage() { + const { courseId = "", lessonId = "" } = useParams(); + const loc = useLocation(); + const navigate = useNavigate(); + const { course, error: courseError, reload } = useCourse(courseId); + const lessonApi = useApi<LessonPayload>(`/api/courses/${encodeURIComponent(courseId)}/lessons/${encodeURIComponent(lessonId)}`); + const lesson = lessonApi.data; + const { user } = useMe(); + const [look] = useLook(); + const [track, setTrack] = useTrack(courseId, course?.tracks ?? lesson?.tracks ?? []); + const q = useMemo(() => new URLSearchParams(loc.search), [loc.search]); + const initialTime = q.has("t") ? Number(q.get("t")) : null; + const initialSlide = q.has("slide") && q.has("deck") ? { deckId: q.get("deck")!, page: Number(q.get("slide")) } : null; + const progress = course?.progress[lessonId] ?? null; + const signedIn = !!user; + + useEffect(() => { if (lesson) document.title = `${lesson.lesson.title.replace(/`/g, "")} – ${lesson.courseTitle}`; }, [lesson]); + useEffect(() => { window.scrollTo(0, 0); }, [lessonId]); + + const setState = useCallback(async (state: ProgressState, positionSeconds?: number) => { + if (!signedIn) return; + try { + await api(`/api/courses/${encodeURIComponent(courseId)}/progress/${encodeURIComponent(lessonId)}`, { method: "PUT", body: { state, positionSeconds } }); + if (state === "done" || progress?.state !== state) { await reload(); void refreshLibrary(); } + } catch (e) { toast(`Progress was not saved: ${(e as Error).message}`); } + }, [courseId, lessonId, signedIn, progress?.state, reload]); + + if (courseError && !course) return <CourseMissing status={courseError.status} signedIn={signedIn} />; + if (lessonApi.error && course) { + return ( + <CourseFrame course={course} currentLessonId={lessonId} look={look} lookMode={null}> + <article className="col"><h1 className="title">Lesson not found</h1><p className="source">This course has no lesson “{lessonId}”.</p></article> + </CourseFrame> + ); + } + if (!course || !lesson || lesson.lesson.id !== lessonId) { + return course ? <CourseFrame course={course} currentLessonId={lessonId} look={look} lookMode={null}><div className="col" /></CourseFrame> : null; + } + const isDone = progress?.state === "done"; + const isExercise = lesson.lesson.type === "exercise"; + const n = moduleNumber(course, lesson.module.index); + return ( + <CourseFrame course={course} currentLessonId={lessonId} look={look} lookMode={isExercise ? "work" : "toggle"}> + <LessonView + lesson={lesson} look={look} track={track} onTrackChange={setTrack} progress={progress} signedIn={signedIn} + onProgress={setState} onSaved={() => { void reload(); void refreshLibrary(); }} + initialTime={initialTime} initialSlide={initialSlide} + moduleLabel={`${/^(section|part|chapter)\b/i.test(lesson.module.title) ? "" : "Module "}${n}`.trim()} + /> + <FloatingActionBar> + <Pill variant="ghost" disabled={!lesson.prev} title={lesson.prev ? `Previous: ${lesson.prev.title}` : "First lesson"} onClick={() => lesson.prev && navigate(lessonHref(courseId, lesson.prev.id))}> + <Prev /><span className="sr">Previous lesson</span> + </Pill> + <Pill variant="ghost" aria-pressed={isDone} disabled={!signedIn || lesson.locked} title={signedIn ? undefined : "Sign in to keep your progress"} + onClick={() => void setState(isDone ? "not-started" : "done")}> + <Check size={14} />{isDone ? "Done" : "Mark done"} + </Pill> + <span className="sep" aria-hidden="true" /> + <Pill variant="primary" disabled={!lesson.next} title={lesson.next ? `Next: ${lesson.next.title}` : "Last lesson"} onClick={() => lesson.next && navigate(lessonHref(courseId, lesson.next.id))}> + <span>Next<span className="lbl-long"> lesson</span></span><Next /> + </Pill> + </FloatingActionBar> + </CourseFrame> + ); +} + +/* ---------- /c/:courseId/notes ---------- */ + +export function NotesPage() { + const { courseId = "" } = useParams(); + const { course, error } = useCourse(courseId); + const { user, loaded } = useMe(); + const [look] = useLook(); + const notesApi = useApi<Note[]>(user ? `/api/courses/${encodeURIComponent(courseId)}/notes` : null); + useEffect(() => { if (course) document.title = `Your notes – ${course.title}`; }, [course]); + if (error && !course) return <CourseMissing status={error.status} signedIn={!!user} />; + if (!course) return null; + const titles = new Map(course.modules.flatMap((m) => m.lessons.map((l) => [l.id, { title: l.title, module: m.title }] as const))); + const order = new Map(course.modules.flatMap((m) => m.lessons).map((l, i) => [l.id, i])); + const notes = notesApi.data ?? []; + const groups = new Map<string, Note[]>(); + for (const n of [...notes].sort((a, b) => (order.get(a.lessonId) ?? 1e9) - (order.get(b.lessonId) ?? 1e9) || anchorNum(a) - anchorNum(b) || a.createdAt.localeCompare(b.createdAt))) { + if (!groups.has(n.lessonId)) groups.set(n.lessonId, []); + groups.get(n.lessonId)!.push(n); + } + const remove = async (id: string) => { + try { + await api(`/api/notes/${encodeURIComponent(id)}`, { method: "DELETE" }); + notesApi.set(notes.filter((n) => n.id !== id)); + } catch (e) { toast(`The note was not deleted: ${(e as Error).message}`); } + }; + const update = async (id: string, body: string) => { + try { + const n = await api<Note>(`/api/notes/${encodeURIComponent(id)}`, { method: "PATCH", body: { body } }); + notesApi.set(notes.map((x) => (x.id === id ? n : x))); + } catch (e) { toast(`The note was not saved: ${(e as Error).message}`); } + }; + return ( + <CourseFrame course={course} currentLessonId={null} look={look} lookMode={null}> + <article className="col" data-look="work" style={{ paddingBottom: 60 }}> + <p className="chap">{course.title}</p> + <h1 className="title">Your notes</h1> + <p className="source">Notes on lessons, video moments and slides, in course order. Only you see them.</p> + {!loaded ? null : !user ? ( + <div className="notice"><Link to="/account">Sign in</Link> to keep notes.</div> + ) : notesApi.loading && !notesApi.data ? <p className="muted">Loading…</p> + : groups.size === 0 ? <div className="notice">No notes yet. Add them under a video, on a slide or at the end of a lesson.</div> + : [...groups].map(([lid, ns]) => ( + <section key={lid} className="notes-group"> + <h2><Link to={lessonHref(courseId, lid)}><TitleText text={titles.get(lid)?.title ?? lid} /></Link></h2> + {titles.get(lid) ? <p className="source" style={{ margin: 0 }}>{titles.get(lid)!.module}</p> : null} + <ul className="note-list"> + {ns.map((n) => <NoteItem key={n.id} note={n} notes={{ update, remove }} courseId={courseId} extra={<AnchorLink note={n} />} />)} + </ul> + </section> + ))} + </article> + </CourseFrame> + ); +} + +const anchorNum = (n: Note) => (n.anchorKind === "none" ? -1 : Number(n.anchorValue ?? 0)); + +function AnchorLink({ note }: { note: Note }) { + const base = lessonHref(note.courseId, note.lessonId); + if (note.anchorKind === "video" && note.anchorValue !== null) return <Link className="anchor" to={`${base}?t=${Math.floor(Number(note.anchorValue))}`}>Video {fmtTime(Number(note.anchorValue))}</Link>; + if (note.anchorKind === "slide" && note.anchorValue !== null && note.deckId) return <Link className="anchor" to={`${base}?deck=${encodeURIComponent(note.deckId)}&slide=${note.anchorValue}`}>Slide {note.anchorValue}</Link>; + return <span className="anchor">Lesson</span>; +} + diff --git a/packages/feature-learner/src/web/icons.tsx b/packages/feature-learner/src/web/icons.tsx new file mode 100644 index 0000000..8553689 --- /dev/null +++ b/packages/feature-learner/src/web/icons.tsx @@ -0,0 +1,15 @@ +const A = { fill: "none", stroke: "currentColor", "aria-hidden": true } as const; + +export const Chev = () => <svg width="10" height="10" viewBox="0 0 10 10" strokeWidth="1.6" {...A}><path d="M3.5 2l3 3-3 3" /></svg>; +export const Down = () => <svg width="10" height="10" viewBox="0 0 10 10" strokeWidth="1.5" {...A}><path d="M2 3.5l3 3 3-3" /></svg>; +export const Check = ({ size = 12 }: { size?: number }) => <svg width={size} height={size} viewBox="0 0 16 16" strokeWidth="1.8" {...A}><path d="M3 8.5l3.2 3L13 4.5" /></svg>; +export const Lock = () => <svg width="11" height="11" viewBox="0 0 16 16" strokeWidth="1.5" {...A}><rect x="3" y="7" width="10" height="7.5" rx="1.5" /><path d="M5.5 7V5a2.5 2.5 0 015 0v2" /></svg>; +export const FileI = () => <svg width="13" height="13" viewBox="0 0 16 16" strokeWidth="1.3" {...A}><path d="M4 1.5h5.5L13 5v9.5H4z" /><path d="M9.5 1.5V5H13" /></svg>; +export const DirI = () => <svg width="13" height="13" viewBox="0 0 16 16" strokeWidth="1.3" {...A}><path d="M1.5 3.5h5l1.5 1.5h6.5v8.5h-13z" /></svg>; +export const PdfI = () => <svg width="14" height="14" viewBox="0 0 16 16" strokeWidth="1.3" {...A}><path d="M3.5 1.5h6L13 5v9.5H3.5z" /><path d="M5.5 9h5M5.5 11.5h3.5" /></svg>; +export const SearchI = () => <svg width="14" height="14" viewBox="0 0 16 16" strokeWidth="1.5" {...A}><circle cx="7" cy="7" r="4.5" /><path d="M10.5 10.5L14 14" /></svg>; +export const MenuI = () => <svg width="16" height="16" viewBox="0 0 16 16" strokeWidth="1.5" {...A}><path d="M2 4h12M2 8h12M2 12h8" /></svg>; +export const Prev = () => <svg width="12" height="12" viewBox="0 0 12 12" strokeWidth="1.6" {...A}><path d="M7.5 2.5L4 6l3.5 3.5" /></svg>; +export const Next = () => <svg width="12" height="12" viewBox="0 0 12 12" strokeWidth="1.6" {...A}><path d="M4.5 2.5L8 6 4.5 9.5" /></svg>; +export const Full = () => <svg width="12" height="12" viewBox="0 0 12 12" strokeWidth="1.5" {...A}><path d="M1.5 4.5v-3h3M7.5 1.5h3v3M10.5 7.5v3h-3M4.5 10.5h-3v-3" /></svg>; +export const DownloadI = () => <svg width="12" height="12" viewBox="0 0 12 12" strokeWidth="1.5" {...A}><path d="M6 1.5v6.5M3.2 5.5L6 8.2l2.8-2.7M1.5 10.5h9" /></svg>; diff --git a/packages/feature-learner/src/web/index.ts b/packages/feature-learner/src/web/index.ts new file mode 100644 index 0000000..4fcd26f --- /dev/null +++ b/packages/feature-learner/src/web/index.ts @@ -0,0 +1,17 @@ +import type { WebFeature } from "@stc/contracts"; + +export { LessonView, type LessonViewProps } from "./lesson/LessonView"; +export { ensureStyles as ensureLearnerStyles } from "./api"; + +/** Learner routes: library, course pages, lessons, notes and the account pages. */ +export const learnerWeb: WebFeature = { + name: "learner", + routes: [ + { path: "/", element: () => import("./library/LibraryPage") }, + { path: "/c/:courseId", element: () => import("./course/pages").then((m) => ({ default: m.CoursePage })) }, + { path: "/c/:courseId/notes", element: () => import("./course/pages").then((m) => ({ default: m.NotesPage })) }, + { path: "/c/:courseId/:lessonId", element: () => import("./course/pages").then((m) => ({ default: m.LessonPage })) }, + { path: "/account", element: () => import("./account/AccountPage") }, + { path: "/account/admin", element: () => import("./account/AdminPage") }, + ], +}; diff --git a/packages/feature-learner/src/web/lesson/CodeFiles.tsx b/packages/feature-learner/src/web/lesson/CodeFiles.tsx new file mode 100644 index 0000000..913ff2a --- /dev/null +++ b/packages/feature-learner/src/web/lesson/CodeFiles.tsx @@ -0,0 +1,155 @@ +import { useEffect, useMemo, useState } from "react"; +import { Pill } from "@stc/ui"; +import type { CodeListing, FileRef } from "../../shared/types"; +import { mediaUrl } from "../../shared/types"; +import { copyText, fmtSize, useApi } from "../api"; +import { DirI, DownloadI, FileI, PdfI } from "../icons"; + +const MAX_VIEW = 400 * 1024; +const BINARY = /\.(png|jpe?g|gif|webp|ico|pdf|zip|gz|tgz|jar|class|exe|dll|so|dylib|bin|mp4|mp3|wav|woff2?|ttf|otf|sqlite|db|pyc)$/i; + +export const zipUrl = (courseId: string, moduleId: string) => `/api/courses/${encodeURIComponent(courseId)}/code/${encodeURIComponent(moduleId)}/zip`; + +interface TreeNode { name: string; path: string; dirs: Map<string, TreeNode>; files: { name: string; path: string; size: number }[] } + +export function buildTree(files: { path: string; size: number }[]): TreeNode { + const root: TreeNode = { name: "", path: "", dirs: new Map(), files: [] }; + for (const f of files) { + const parts = f.path.split("/"); + let node = root; + for (const p of parts.slice(0, -1)) { + if (!node.dirs.has(p)) node.dirs.set(p, { name: p, path: node.path ? `${node.path}/${p}` : p, dirs: new Map(), files: [] }); + node = node.dirs.get(p)!; + } + node.files.push({ name: parts[parts.length - 1]!, path: f.path, size: f.size }); + } + return root; +} + +/** Picks the first file to show: a README or the entry point, else the first text file. */ +function firstFile(files: { path: string; size: number }[]): string | null { + const text = files.filter((f) => !BINARY.test(f.path) && f.size <= MAX_VIEW); + const pref = [/(^|\/)dockerfile[^/]*$/i, /(^|\/)(main|index|app)\.[^/]+$/i, /(^|\/)readme(\.md)?$/i]; + for (const r of pref) { const f = text.find((x) => r.test(x.path)); if (f) return f.path; } + return text[0]?.path ?? null; +} + +/** + * Companion code for a section: file tree, tabs, line numbers, copy, the zip download and the command + * that runs it locally. Files are read through the media route, so visibility checks apply. + */ +export function CodeFiles({ courseId, moduleId, dir, title = "Files for this section" }: { courseId: string; moduleId: string; dir?: string; title?: string }) { + const { data: listing, error } = useApi<CodeListing>(`/api/courses/${encodeURIComponent(courseId)}/code/${encodeURIComponent(moduleId)}`); + const [tabs, setTabs] = useState<string[]>([]); + const [current, setCurrent] = useState<string | null>(null); + const [content, setContent] = useState<Record<string, string>>({}); + useEffect(() => { + if (!listing) return; + const f = firstFile(listing.files); + setTabs(f ? [f] : []); setCurrent(f); setContent({}); + }, [listing]); + useEffect(() => { + if (!listing || !current || current in content) return; + const f = listing.files.find((x) => x.path === current); + if (!f) return; + if (BINARY.test(f.path) || f.size > MAX_VIEW) { setContent((c) => ({ ...c, [current]: "" })); return; } + let live = true; + fetch(mediaUrl(courseId, `${listing.dir}/${f.path}`), { credentials: "same-origin" }) + .then((r) => (r.ok ? r.text() : Promise.reject(new Error(String(r.status))))) + .then((t) => live && setContent((c) => ({ ...c, [current]: t })), () => live && setContent((c) => ({ ...c, [current]: "\u0000error" }))); + return () => { live = false; }; + }, [listing, current, content, courseId]); + const tree = useMemo(() => buildTree(listing?.files ?? []), [listing]); + + const folder = (listing?.dir ?? dir ?? "").split("/").filter(Boolean).pop() ?? moduleId; + const head = ( + <div className="blockhead"> + <h2>{title}</h2> + <a className="stc-pill sm" href={zipUrl(courseId, moduleId)} download={`${folder}.zip`}><DownloadI />Download files for this section (zip)</a> + </div> + ); + if (error) return null; + if (!listing) return <section className="lr-block" aria-label={title}>{head}<p className="muted">Loading files…</p></section>; + const open = (p: string) => { setTabs((t) => (t.includes(p) ? t : [...t.slice(-5), p])); setCurrent(p); }; + const file = listing.files.find((f) => f.path === current); + const text = current ? content[current] : undefined; + const lines = text && !text.startsWith("\u0000") ? text.replace(/\n$/, "").split("\n") : null; + + return ( + <section className="lr-block" aria-label={title}> + {head} + {listing.files.length === 0 ? <p className="muted">This folder is empty.</p> : ( + <div className="files"> + <nav className="tree" aria-label="Files"> + <div className="dir"><DirI />{folder}</div> + <TreeLevel node={tree} depth={1} current={current} onOpen={open} /> + {listing.truncated ? <div className="dir">Only the first {listing.files.length} files are listed.</div> : null} + </nav> + <div className="viewer"> + <div className="tabs" role="tablist"> + {tabs.map((t) => ( + <button key={t} type="button" role="tab" aria-selected={t === current} onClick={() => setCurrent(t)}>{t.split("/").pop()}</button> + ))} + {lines ? <CopyButton text={text!} label="Copy" /> : null} + </div> + {file ? <div className="path">{folder}/{file.path} · {fmtSize(file.size)}</div> : null} + {!file ? <p className="path">Pick a file on the left.</p> + : text === undefined ? <p className="path">Loading…</p> + : lines ? <pre className="code"><code>{lines.map((l, i) => <span key={i} className="ln">{l || " "}</span>)}</code></pre> + : <p className="path">{text === "" ? "This file is not shown here (binary or large). It is in the zip." : "This file could not be loaded."} <a href={mediaUrl(courseId, `${listing.dir}/${file.path}`)} download>Download it</a></p>} + </div> + </div> + )} + {listing.command ? <CommandBlock command={listing.command} intro={`From the folder you unzipped (${folder}):`} /> : null} + </section> + ); +} + +function TreeLevel({ node, depth, current, onOpen }: { node: TreeNode; depth: number; current: string | null; onOpen: (p: string) => void }) { + const pad = { paddingLeft: 12 + depth * 14 }; + return ( + <> + {[...node.dirs.values()].map((d) => ( + <div key={d.path}> + <div className="dir" style={pad}><DirI />{d.name}</div> + <TreeLevel node={d} depth={depth + 1} current={current} onOpen={onOpen} /> + </div> + ))} + {node.files.map((f) => ( + <button key={f.path} type="button" style={pad} aria-current={f.path === current} onClick={() => onOpen(f.path)}><FileI />{f.name}</button> + ))} + </> + ); +} + +export function CopyButton({ text, label = "Copy" }: { text: string; label?: string }) { + const [done, setDone] = useState(false); + return ( + <Pill className="sm" variant="ghost" onClick={async () => { if (await copyText(text)) { setDone(true); setTimeout(() => setDone(false), 1400); } }}> + {done ? "Copied" : label} + </Pill> + ); +} + +export function CommandBlock({ command, intro }: { command: string; intro?: string }) { + return ( + <div style={{ marginTop: 18 }}> + {intro ? <p className="muted" style={{ fontSize: 13, margin: "0 0 8px" }}>{intro}</p> : null} + <div className="cmd"><pre className="code"><code>{command}</code></pre><CopyButton text={command} /></div> + </div> + ); +} + +export function Attachments({ files, title = "Attachments", note }: { files: FileRef[]; title?: string; note?: string }) { + if (!files.length) return null; + return ( + <section className="lr-block" aria-label={title}> + <div className="blockhead"><h2>{title}</h2>{note ? <span>{note}</span> : null}</div> + <ul className="rows"> + {files.map((f) => ( + <li key={f.file}><PdfI /><a href={f.url} download={f.file.split("/").pop()}>{f.title}</a><span className="sz tnum">{fmtSize(f.size)}</span></li> + ))} + </ul> + </section> + ); +} diff --git a/packages/feature-learner/src/web/lesson/DeckViewer.tsx b/packages/feature-learner/src/web/lesson/DeckViewer.tsx new file mode 100644 index 0000000..11d367a --- /dev/null +++ b/packages/feature-learner/src/web/lesson/DeckViewer.tsx @@ -0,0 +1,196 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { Pill } from "@stc/ui"; +import type { DeckRef } from "../../shared/types"; +import { Full, Next, Prev } from "../icons"; +import type { NotesApi } from "./notes"; + +/** The pdf.js worker is served by the learner API from the installed pdfjs-dist. */ +export const PDF_WORKER_URL = "/api/learner/pdf.worker.mjs"; + +// biome-ignore lint/suspicious/noExplicitAny: pdf.js document proxy (types are loaded lazily with the library) +type PdfDoc = any; + +let pdfjsPromise: Promise<typeof import("pdfjs-dist")> | null = null; +function loadPdfjs() { + pdfjsPromise ??= import("pdfjs-dist").then((m) => { m.GlobalWorkerOptions.workerSrc = PDF_WORKER_URL; return m; }); + return pdfjsPromise; +} + +/** Renders `page` of `doc` into `canvas` at `cssWidth` pixels wide (scaled for the device pixel ratio). */ +async function renderPage(doc: PdfDoc, page: number, canvas: HTMLCanvasElement, cssWidth: number, cssMaxHeight?: number) { + const p = await doc.getPage(page); + const base = p.getViewport({ scale: 1 }); + let scale = cssWidth / base.width; + if (cssMaxHeight && base.height * scale > cssMaxHeight) scale = cssMaxHeight / base.height; + const dpr = Math.min(window.devicePixelRatio || 1, 2); + const vp = p.getViewport({ scale: scale * dpr }); + canvas.width = Math.floor(vp.width); + canvas.height = Math.floor(vp.height); + canvas.style.width = `${Math.floor(vp.width / dpr)}px`; + canvas.style.height = `${Math.floor(vp.height / dpr)}px`; + const task = p.render({ canvasContext: canvas.getContext("2d")!, viewport: vp, canvas }); + await task.promise; +} + +const isTyping = (t: EventTarget | null) => !!(t as Element | null)?.closest?.("input, textarea, select, video, audio, [contenteditable], [role='slider']"); + +/** + * Slide deck viewer: 16:9 stage, previous/next, page counter, thumbnails, fullscreen, arrow keys (when + * `keyboard` is set) and a note per slide. Pages are rendered in the browser with pdf.js. + */ +export function DeckViewer({ deck, notes, initialPage = 1, keyboard = true, title }: { + deck: DeckRef; notes: NotesApi | null; initialPage?: number; keyboard?: boolean; title?: string; +}) { + const [doc, setDoc] = useState<PdfDoc | null>(null); + const [pages, setPages] = useState(0); + const [page, setPage] = useState(initialPage); + const [error, setError] = useState<string | null>(null); + const stage = useRef<HTMLElement>(null); + const canvas = useRef<HTMLCanvasElement>(null); + + useEffect(() => { + let live = true, loaded: PdfDoc | null = null; + setDoc(null); setError(null); setPages(0); + loadPdfjs() + .then((pdfjs) => pdfjs.getDocument({ url: deck.url, withCredentials: true }).promise) + .then((d: PdfDoc) => { + loaded = d; + if (!live) { void d.destroy(); return; } + setDoc(d); setPages(d.numPages); setPage((p) => Math.min(Math.max(1, p), d.numPages)); + }) + .catch((e: unknown) => live && setError(`This deck could not be opened. ${(e as Error).message ?? ""}`.trim())); + return () => { live = false; if (loaded) void loaded.destroy(); }; + }, [deck.url]); + + // Render the current page to fit the stage. + useEffect(() => { + if (!doc || !canvas.current || !stage.current) return; + let live = true; + const draw = () => { + const r = stage.current!.getBoundingClientRect(); + if (live) renderPage(doc, page, canvas.current!, r.width, r.height).catch(() => {}); + }; + draw(); + const ro = new ResizeObserver(() => draw()); + ro.observe(stage.current); + return () => { live = false; ro.disconnect(); }; + }, [doc, page]); + + const go = useCallback((p: number) => setPage((cur) => (pages ? Math.min(Math.max(1, p), pages) : cur)), [pages]); + + useEffect(() => { + if (!keyboard) return; + const onKey = (e: KeyboardEvent) => { + if (e.ctrlKey || e.metaKey || e.altKey || isTyping(e.target)) return; + if (e.key === "ArrowRight") { e.preventDefault(); setPage((p) => Math.min(p + 1, pages || p)); } + else if (e.key === "ArrowLeft") { e.preventDefault(); setPage((p) => Math.max(p - 1, 1)); } + }; + document.addEventListener("keydown", onKey); + return () => document.removeEventListener("keydown", onKey); + }, [keyboard, pages]); + + useEffect(() => { + document.querySelector(`[data-deck="${CSS.escape(deck.id)}"] .thumbs [aria-current="true"]`)?.scrollIntoView({ block: "nearest", inline: "nearest" }); + }, [page, deck.id]); + + const fullscreen = () => { + if (document.fullscreenElement) void document.exitFullscreen(); + else stage.current?.requestFullscreen?.().catch(() => {}); + }; + const slideNotes = notes?.notes.filter((n) => n.anchorKind === "slide" && n.deckId === deck.id) ?? []; + const withNote = new Set(slideNotes.map((n) => Number(n.anchorValue))); + + return ( + <section className="deck" data-deck={deck.id} aria-label={`Slides: ${title ?? deck.title}`}> + <figure className="stage" ref={stage}> + {error ? <span className="msg">{error} <a href={deck.url} target="_blank" rel="noopener">Open the PDF</a></span> : null} + {!error && !doc ? <span className="msg">Loading slides…</span> : null} + <canvas ref={canvas} hidden={!doc || !!error} role="img" aria-label={`Slide ${page} of ${pages}`} /> + </figure> + <div className="deckbar"> + <Pill className="sm" aria-label="Previous slide" disabled={page <= 1} onClick={() => go(page - 1)}><Prev /></Pill> + <span className="count tnum" aria-live="polite">{pages ? `${page} / ${pages}` : "– / –"}</span> + <Pill className="sm" aria-label="Next slide" disabled={!pages || page >= pages} onClick={() => go(page + 1)}><Next /></Pill> + <span className="gap" /> + {keyboard ? <span className="hint">Arrow keys change slides</span> : null} + <a className="stc-pill stc-pill--ghost sm" href={deck.url} target="_blank" rel="noopener">PDF</a> + <Pill className="sm" onClick={fullscreen}><Full />Fullscreen</Pill> + </div> + {doc ? <Thumbs doc={doc} pages={pages} page={page} onPick={go} withNote={withNote} /> : null} + {notes ? <SlideNote notes={notes} deckId={deck.id} page={page} existing={slideNotes.filter((n) => Number(n.anchorValue) === page)} /> : null} + </section> + ); +} + +function Thumbs({ doc, pages, page, onPick, withNote }: { doc: PdfDoc; pages: number; page: number; onPick: (p: number) => void; withNote: Set<number> }) { + const ref = useRef<HTMLDivElement>(null); + useEffect(() => { + const root = ref.current; + if (!root) return; + const queue: HTMLCanvasElement[] = []; + let running = false, live = true; + const pump = async () => { + if (running) return; + running = true; + while (live && queue.length) { + const c = queue.shift()!; + await renderPage(doc, Number(c.dataset.p), c, 104).catch(() => {}); + } + running = false; + }; + const io = new IntersectionObserver((entries) => { + for (const e of entries) { + if (!e.isIntersecting) continue; + const c = e.target as HTMLCanvasElement; + io.unobserve(c); + queue.push(c); + } + void pump(); + }, { root, rootMargin: "0px 400px" }); + root.querySelectorAll("canvas").forEach((c) => io.observe(c)); + return () => { live = false; io.disconnect(); }; + }, [doc, pages]); + return ( + <div className="thumbs" ref={ref} role="group" aria-label="Slides"> + {Array.from({ length: pages }, (_, i) => ( + <button key={i} type="button" aria-label={`Slide ${i + 1}${withNote.has(i + 1) ? ", has a note" : ""}`} aria-current={page === i + 1} className={withNote.has(i + 1) ? "has-note" : undefined} onClick={() => onPick(i + 1)}> + <canvas data-p={i + 1} /> + </button> + ))} + </div> + ); +} + +function SlideNote({ notes, deckId, page, existing }: { notes: NotesApi; deckId: string; page: number; existing: { id: string; body: string }[] }) { + const first = existing[0]; + const [text, setText] = useState(first?.body ?? ""); + const [saved, setSaved] = useState<"idle" | "saving" | "saved">("idle"); + const busy = useRef(false); + useEffect(() => { setText(first?.body ?? ""); setSaved("idle"); }, [page, deckId, first?.id, first?.body]); + if (!notes.enabled) return null; + const dirty = text.trim() !== (first?.body ?? "").trim(); + const save = async () => { + if (!dirty || busy.current) return; + busy.current = true; + setSaved("saving"); + const body = text.trim(); + try { + if (first && !body) await notes.remove(first.id); + else if (first) await notes.update(first.id, body); + else if (body) await notes.create({ anchorKind: "slide", anchorValue: page, deckId, body }); + setSaved("saved"); + } finally { busy.current = false; } + }; + return ( + <div className="slide-note"> + <label htmlFor={`sn-${deckId}`}>Your note on slide {page}</label> + <textarea id={`sn-${deckId}`} className="note-ta" value={text} onChange={(e) => { setText(e.target.value); setSaved("idle"); }} onBlur={save} + placeholder="Only you see this. It comes back when you return to this slide." /> + <div className="note-actions"> + <span className="msg">{saved === "saving" ? "Saving…" : saved === "saved" ? "Saved" : ""}</span> + {existing.length > 1 ? <span className="msg">{existing.length - 1} more on this slide in your notes list</span> : null} + <Pill className="sm" disabled={!dirty} onClick={save}>Save note</Pill> + </div> + </div> + ); +} diff --git a/packages/feature-learner/src/web/lesson/Exercise.tsx b/packages/feature-learner/src/web/lesson/Exercise.tsx new file mode 100644 index 0000000..6a6d05a --- /dev/null +++ b/packages/feature-learner/src/web/lesson/Exercise.tsx @@ -0,0 +1,178 @@ +import { useEffect, useRef, useState } from "react"; +import { Pill } from "@stc/ui"; +import type { ExerciseRunner, TestResult } from "@stc/contracts"; +import type { LessonPayload } from "../../shared/types"; +import { api, toast } from "../api"; +import { createBrowserRunner } from "../runner/runner"; +import { CommandBlock } from "./CodeFiles"; +import { Chev } from "../icons"; + +export const RUN_TIMEOUT_MS = 3000; + +export function localCommandFor(cmd: LessonPayload["localCommand"], track: string | null): string | null { + if (!cmd) return null; + if (typeof cmd === "string") return cmd; + return (track && cmd[track]) || Object.values(cmd)[0] || null; +} + +const attemptUrl = (l: LessonPayload, track: string) => + `/api/courses/${encodeURIComponent(l.courseId)}/lessons/${encodeURIComponent(l.lesson.id)}/attempt?track=${encodeURIComponent(track)}`; + +/** + * The right-hand side of an exercise page. With a browser exercise: editor, Run tests, Reset and per-test + * results (run in a Web Worker). Without one: the local command, plus optional pasted evidence. + */ +export function ExercisePanel({ lesson, track, signedIn, preview, onSaved, runner }: { + lesson: LessonPayload; + track: string | null; + signedIn: boolean; + preview?: boolean; + onSaved?: () => void; + runner?: ExerciseRunner; +}) { + if (lesson.browser) return <BrowserExercise lesson={lesson} track={track} save={signedIn && !preview} onSaved={onSaved} runner={runner} />; + return <LocalExercise lesson={lesson} track={track} save={signedIn && !preview} onSaved={onSaved} />; +} + +function BrowserExercise({ lesson, track, save, onSaved, runner }: { + lesson: LessonPayload; track: string | null; save: boolean; onSaved?: () => void; runner?: ExerciseRunner; +}) { + const b = lesson.browser!; + const key = b.language; + const [code, setCode] = useState(b.starter); + const [results, setResults] = useState<TestResult[] | null>(null); + const [running, setRunning] = useState(false); + const ta = useRef<HTMLTextAreaElement>(null); + const gutter = useRef<HTMLPreElement>(null); + + useEffect(() => { + setCode(b.starter); setResults(null); + if (!save) return; + let live = true; + api<{ code: string; result: TestResult[] | null } | null>(attemptUrl(lesson, key)).then((a) => { + if (!live || !a) return; + setCode(a.code); + if (Array.isArray(a.result)) setResults(a.result); + }, () => {}); + return () => { live = false; }; + }, [lesson.courseId, lesson.lesson.id, b.starter, key, save]); + + const run = async () => { + setRunning(true); + try { + const r = runner ?? createBrowserRunner(b.language); + const res = await r.run(code, b.tests, RUN_TIMEOUT_MS); + setResults(res); + const passed = res.length > 0 && res.every((t) => t.passed); + if (save) { + try { + await api(attemptUrl(lesson, key).split("?")[0]!, { method: "PUT", body: { track: key, code, result: res, passed } }); + onSaved?.(); + } catch (e) { toast(`Your code was not saved: ${(e as Error).message}`); } + } + } finally { setRunning(false); } + }; + const reset = () => { + if (code !== b.starter && !confirm("Reset the editor to the starting code? Your changes are lost.")) return; + setCode(b.starter); setResults(null); toast("Code reset to the starting code"); + }; + const lines = code.split("\n").length; + const n = results?.filter((t) => t.passed).length ?? 0; + const all = !!results && results.length > 0 && n === results.length; + + return ( + <div className="edit"> + <div className="edit-bar"> + <span className="f">{b.starterFile}</span> + <Pill className="sm" variant="ghost" onClick={reset}>Reset</Pill> + <Pill className="sm" variant="primary" onClick={run} disabled={running}>{running ? "Running…" : "Run tests"}</Pill> + </div> + <div className="editor"> + <pre className="gutter" ref={gutter} aria-hidden="true">{Array.from({ length: lines }, (_, i) => i + 1).join("\n")}</pre> + <textarea + ref={ta} value={code} spellCheck={false} aria-label={`Code editor, ${b.starterFile}`} + onChange={(e) => setCode(e.target.value)} + onScroll={(e) => { if (gutter.current) gutter.current.scrollTop = (e.target as HTMLTextAreaElement).scrollTop; }} + onKeyDown={(e) => { + // Tab indents inside the editor; Escape leaves it so Tab moves focus again. + if (e.key === "Tab" && !e.shiftKey && !e.ctrlKey && !e.metaKey && !e.altKey) { + e.preventDefault(); + const t = e.currentTarget; + t.setRangeText(" ", t.selectionStart, t.selectionEnd, "end"); + setCode(t.value); + } else if (e.key === "Escape") e.currentTarget.blur(); + }} + /> + </div> + <div className="results" aria-live="polite"> + {!results ? ( + <div className="res-sum"><span>Not run yet. The tests run in your browser, in a sandboxed worker.</span></div> + ) : ( + <> + <div className="res-sum"><span><b className="tnum">{n} of {results.length}</b> passed{all ? ". This exercise is done." : ""}</span></div> + <ul className="res-list"> + {results.map((t, i) => ( + <li key={i} className={t.passed ? "pass" : "fail"}> + <div className="res-row"><span className="s">{t.passed ? "Pass" : "Fail"}</span><code>{t.name}</code></div> + {t.passed ? null : ( + <dl className="diff"> + {t.expected !== undefined ? <><dt>Expected</dt><dd>{t.expected}</dd></> : null} + {t.actual !== undefined ? <><dt>Actual</dt><dd>{t.actual}</dd></> : null} + {t.message && t.expected === undefined ? <><dt>Error</dt><dd>{t.message}</dd></> : null} + </dl> + )} + </li> + ))} + </ul> + </> + )} + </div> + {b.solution ? ( + <details className="fold" style={{ margin: 0, borderBottom: 0, padding: "0 14px" }}> + <summary><Chev /><span className="sum">Solution (only authors see this)</span></summary> + <div className="in"><pre className="code"><code>{b.solution}</code></pre></div> + </details> + ) : null} + </div> + ); +} + +function LocalExercise({ lesson, track, save, onSaved }: { lesson: LessonPayload; track: string | null; save: boolean; onSaved?: () => void }) { + const cmd = localCommandFor(lesson.localCommand, track); + const key = track ?? "default"; + const [evidence, setEvidence] = useState(""); + const [saved, setSaved] = useState(false); + useEffect(() => { + setEvidence(""); setSaved(false); + if (!save) return; + let live = true; + api<{ code: string } | null>(attemptUrl(lesson, key)).then((a) => { if (live && a) { setEvidence(a.code); setSaved(true); } }, () => {}); + return () => { live = false; }; + }, [lesson.courseId, lesson.lesson.id, key, save]); + const submit = async () => { + try { + // Pasted passing output counts as a pass: the lesson is marked done. + await api(attemptUrl(lesson, key).split("?")[0]!, { method: "PUT", body: { track: key, code: evidence, result: { kind: "evidence" }, passed: true } }); + setSaved(true); + onSaved?.(); + toast("Output saved. The exercise is marked done."); + } catch (e) { toast(`Not saved: ${(e as Error).message}`); } + }; + return ( + <div className="local-panel"> + <h2>Run it on your machine</h2> + <p>This exercise runs locally, in the course’s own project. Run the tests from the course folder; when they pass, mark the exercise done.</p> + {cmd ? <CommandBlock command={cmd} /> : <p>The course does not name a command for this exercise. Its instructions on the left say how to check your work.</p>} + {save ? ( + <div style={{ marginTop: 18 }}> + <label className="muted" htmlFor="lr-evidence" style={{ display: "block", fontSize: 12.5, marginBottom: 6 }}>Passing output (optional). Paste it here to keep it with your progress.</label> + <textarea id="lr-evidence" className="note-ta" value={evidence} onChange={(e) => { setEvidence(e.target.value); setSaved(false); }} placeholder="# tests 5 / # pass 5" /> + <div className="note-actions"> + <span className="msg">{saved && evidence ? "Saved" : ""}</span> + <Pill className="sm" disabled={!evidence.trim() || saved} onClick={submit}>Save output and mark done</Pill> + </div> + </div> + ) : null} + </div> + ); +} diff --git a/packages/feature-learner/src/web/lesson/LessonView.tsx b/packages/feature-learner/src/web/lesson/LessonView.tsx new file mode 100644 index 0000000..2e4735b --- /dev/null +++ b/packages/feature-learner/src/web/lesson/LessonView.tsx @@ -0,0 +1,260 @@ +import { useEffect, useMemo, useRef, useState, type ReactNode } from "react"; +import { Link } from "react-router-dom"; +import { SegmentedToggle } from "@stc/ui"; +import type { ExerciseRunner } from "@stc/contracts"; +import type { LessonPayload, LessonProgress, ProgressState } from "../../shared/types"; +import { ensureStyles, fmtMinutes, fmtTime, type Look } from "../api"; +import { renderHintsHtml, renderLessonHtml } from "../markdown"; +import { Attachments, CodeFiles, CommandBlock, zipUrl } from "./CodeFiles"; +import { DeckViewer } from "./DeckViewer"; +import { ExercisePanel, localCommandFor } from "./Exercise"; +import { NotesPanel, useLessonNotes, type NotesApi } from "./notes"; +import { Prose } from "./Prose"; +import { DownloadI } from "../icons"; +import { TitleText } from "./TitleText"; + +export interface LessonViewProps { + lesson: LessonPayload; + /** Reading or Work. Exercise pages always use Work. */ + look?: Look; + track?: string | null; + onTrackChange?: (track: string) => void; + progress?: LessonProgress | null; + /** Signed-in learners get notes and saved attempts. */ + signedIn?: boolean; + /** Studio preview: no notes, no saved attempts, no progress reports. */ + preview?: boolean; + onProgress?: (state: ProgressState, positionSeconds?: number) => void; + /** An exercise attempt was saved (the server already updated progress). */ + onSaved?: () => void; + /** Start the video here (seconds), e.g. from a timestamp note. */ + initialTime?: number | null; + /** Open a deck at this page, e.g. from a slide note. */ + initialSlide?: { deckId: string; page: number } | null; + /** Replaces the Web Worker runner (tests). */ + runner?: ExerciseRunner; + /** Label for the module line above the title, e.g. "Module 3". */ + moduleLabel?: string; +} + +export { TitleText }; + +/** + * Renders one lesson of any type from its `LessonPayload`: text (track switcher, folds, hint tiers), + * video (player, notes with timestamps, transcript, companion deck), exercise (instructions, hints, + * browser runner or local command), plus decks, code files and attachments. + * Presentational: it fetches only notes, saved attempts and code listings. The studio uses it for previews. + */ +export function LessonView(props: LessonViewProps) { + ensureStyles(); + const { lesson, preview = false, signedIn = false } = props; + const notes = useLessonNotes(lesson.courseId, lesson.lesson.id, signedIn && !preview && !lesson.locked); + const look: Look = lesson.lesson.type === "exercise" ? "work" : props.look ?? "reading"; + const track = props.track ?? lesson.tracks[0]?.id ?? null; + const chap = props.moduleLabel ? `${props.moduleLabel}: ${lesson.module.title}` : lesson.module.title; + + let body: ReactNode; + if (lesson.locked) body = <Locked lesson={lesson} chap={chap} />; + else if (lesson.lesson.type === "video") body = <VideoLesson {...props} chap={chap} notes={notes} />; + else if (lesson.lesson.type === "exercise") body = <ExerciseLesson {...props} chap={chap} track={track} />; + else body = <TextLesson {...props} chap={chap} track={track} notes={notes} />; + + return <div className="lr lr-lesson" data-look={look} data-type={lesson.lesson.type}>{body}</div>; +} + +function Locked({ lesson, chap }: { lesson: LessonPayload; chap: string }) { + return ( + <article className="col"> + <p className="chap">{chap}</p> + <h1 className="title"><TitleText text={lesson.lesson.title} /></h1> + <div className="notice" role="note"> + <strong>This module is locked.</strong> It opens when every lesson in the modules before it is done. + {lesson.prev ? <div><Link className="stc-pill" to={`/c/${encodeURIComponent(lesson.courseId)}/${encodeURIComponent(lesson.prev.id)}`}>Back to {lesson.prev.title}</Link></div> : null} + </div> + </article> + ); +} + +function TrackSwitch({ lesson, track, onChange }: { lesson: LessonPayload; track: string | null; onChange?: (t: string) => void }) { + if (lesson.tracks.length < 2 || !track) return null; + return ( + <div> + <dt>Track</dt> + <dd><SegmentedToggle label="Track" value={track} options={lesson.tracks.map((t) => ({ value: t.id, label: t.label }))} onChange={(v) => onChange?.(v)} /></dd> + </div> + ); +} + +function useLessonHtml(lesson: LessonPayload, track: string | null): string | null { + return useMemo(() => { + const md = (track && lesson.trackMarkdown[track]) || lesson.markdown; + if (!md) return null; + return renderLessonHtml(md, { courseId: lesson.courseId, baseFile: lesson.baseFile, tracks: lesson.tracks, track, lessonFiles: lesson.lessonFiles, stripFirstHeading: true }); + }, [lesson, track]); +} + +function Decks({ lesson, notes, initialSlide, keyboard }: { lesson: LessonPayload; notes: NotesApi | null; initialSlide?: { deckId: string; page: number } | null; keyboard: boolean }) { + if (!lesson.decks.length) return null; + return ( + <> + {lesson.decks.map((d, i) => ( + <div key={d.id} className="lr-block"> + {lesson.decks.length > 1 || lesson.lesson.type !== "text" ? <div className="blockhead"><h2>{d.title}</h2><span>Slides</span></div> : null} + <DeckViewer deck={d} notes={notes} keyboard={keyboard && i === 0} initialPage={initialSlide?.deckId === d.id ? initialSlide.page : 1} /> + </div> + ))} + </> + ); +} + +function TextLesson(p: LessonViewProps & { chap: string; track: string | null; notes: NotesApi }) { + const { lesson, track } = p; + const html = useLessonHtml(lesson, track); + const cmd = localCommandFor(lesson.localCommand, track); + const minutes = lesson.lesson.minutes; + const hasMeta = !!minutes || lesson.tracks.length > 1; + const decks = <Decks lesson={lesson} notes={p.preview ? null : p.notes} initialSlide={p.initialSlide} keyboard />; + return ( + <article className="wide"> + <div className="col"> + <p className="chap">{p.chap}</p> + <h1 className="title"><TitleText text={lesson.lesson.title} /></h1> + {hasMeta ? ( + <dl className="meta"> + {minutes ? <div><dt>Time</dt><dd>About {fmtMinutes(minutes)}</dd></div> : null} + <TrackSwitch lesson={lesson} track={track} onChange={p.onTrackChange} /> + </dl> + ) : null} + {cmd ? <CommandBlock command={cmd} intro="Start this module on your machine:" /> : null} + </div> + {!html ? decks : null} + {html ? <div className="col"><Prose html={html} style={{ marginTop: 30 }} /></div> : null} + {!html && !lesson.decks.length ? <div className="col"><p className="muted" style={{ marginTop: 30 }}>This lesson has no text yet.</p></div> : null} + {html ? decks : null} + {lesson.code ? <CodeFiles courseId={lesson.courseId} moduleId={lesson.code.moduleId} dir={lesson.code.dir} /> : null} + <div className="col"> + <Attachments files={lesson.attachments} note="This section" /> + {!p.preview ? <div className="lr-block"><NotesPanel notes={p.notes} /></div> : null} + </div> + </article> + ); +} + +function VideoLesson(p: LessonViewProps & { chap: string; notes: NotesApi }) { + const { lesson } = p; + const v = useRef<HTMLVideoElement>(null); + const [browse, setBrowse] = useState(false); + const transcriptHtml = useMemo(() => (lesson.transcript + ? renderLessonHtml(lesson.transcript, { courseId: lesson.courseId, baseFile: lesson.baseFile, tracks: lesson.tracks, lessonFiles: lesson.lessonFiles, stripFirstHeading: true }) + : null), [lesson]); + const control = useMemo(() => ({ + currentTime: () => v.current?.currentTime ?? 0, + seek: (s: number) => { const el = v.current; if (el) { el.currentTime = s; void el.play().catch(() => {}); el.scrollIntoView({ block: "nearest", behavior: "smooth" }); } }, + }), []); + useVideoProgress(v, p); + const dur = lesson.video?.duration ?? (lesson.lesson.minutes ? lesson.lesson.minutes * 60 : null); + const folder = lesson.code?.dir.split("/").filter(Boolean).pop(); + return ( + <article className="wide"> + <p className="chap">{p.chap}</p> + <h1 className="title"><TitleText text={lesson.lesson.title} /></h1> + <p className="source"><strong>Video</strong>{dur ? `, ${fmtTime(dur)}` : ""}.{p.progress?.positionSeconds && p.progress.state !== "done" ? ` You stopped at ${fmtTime(p.progress.positionSeconds)}.` : ""}</p> + {lesson.video ? ( + <div className="player"><video ref={v} controls preload="metadata" src={lesson.video.url} /></div> + ) : <div className="notice">The video file for this lesson is missing.</div>} + <Decks lesson={lesson} notes={p.preview ? null : p.notes} initialSlide={p.initialSlide} keyboard /> + <div className="split"> + <div> + {!p.preview ? <NotesPanel notes={p.notes} video={lesson.video ? control : null} /> : null} + {transcriptHtml ? ( + <details className="fold transcript" open={!!p.preview}> + <summary><svg width="10" height="10" viewBox="0 0 10 10" fill="none" stroke="currentColor" strokeWidth="1.6" aria-hidden="true"><path d="M3.5 2l3 3-3 3" /></svg><span className="sum">Transcript</span></summary> + <div className="in"><Prose html={transcriptHtml} style={{ fontSize: "calc(var(--ps) * .9)" }} /></div> + </details> + ) : null} + </div> + <aside> + <div className="blockhead"><h2>Files for this section</h2><span>{lesson.module.title}</span></div> + {lesson.attachments.length || lesson.code ? ( + <ul className="rows"> + {lesson.code ? ( + <li><DownloadI /><a href={zipUrl(lesson.courseId, lesson.code.moduleId)} download={`${folder}.zip`}>{folder}.zip</a><button type="button" className="stc-pill stc-pill--ghost sm" onClick={() => setBrowse(!browse)} aria-expanded={browse}>{browse ? "Hide" : "Browse"}</button></li> + ) : null} + {lesson.attachments.map((f) => ( + <li key={f.file}><DownloadI /><a href={f.url} download={f.file.split("/").pop()}>{f.title}</a><span className="sz tnum">{f.size ? `${Math.max(1, Math.round(f.size / 1024))} KB` : ""}</span></li> + ))} + </ul> + ) : <p className="source" style={{ marginTop: 0 }}>This section has no files.</p>} + </aside> + </div> + {browse && lesson.code ? <CodeFiles courseId={lesson.courseId} moduleId={lesson.code.moduleId} dir={lesson.code.dir} title="Code for this section" /> : null} + </article> + ); +} + +/** Resume position, periodic position reports, and "done" near the end (spec 4.4). */ +function useVideoProgress(v: React.RefObject<HTMLVideoElement | null>, p: LessonViewProps) { + const report = useRef(p.onProgress); + report.current = p.onProgress; + const doneRef = useRef(p.progress?.state === "done"); + doneRef.current = p.progress?.state === "done"; + const start = p.initialTime ?? (p.progress && p.progress.state !== "done" ? p.progress.positionSeconds : null); + useEffect(() => { + const el = v.current; + if (!el || p.preview) return; + let lastSent = -1; + const onMeta = () => { if (start && start > 0 && (!el.duration || start < el.duration - 2)) el.currentTime = start; }; + const send = () => { + if (!el.duration || doneRef.current) return; + const t = el.currentTime; + if (t >= el.duration * 0.95 || el.duration - t < 10) { doneRef.current = true; report.current?.("done", t); return; } + if (Math.abs(t - lastSent) >= 5) { lastSent = t; report.current?.("in-progress", t); } + }; + let tick = 0; + const onTime = () => { if (++tick % 40 === 0 || (el.duration && el.duration - el.currentTime < 10)) send(); }; + if (el.readyState >= 1) onMeta(); else el.addEventListener("loadedmetadata", onMeta, { once: true }); + el.addEventListener("pause", send); + el.addEventListener("ended", send); + el.addEventListener("timeupdate", onTime); + return () => { + el.removeEventListener("loadedmetadata", onMeta); + el.removeEventListener("pause", send); + el.removeEventListener("ended", send); + el.removeEventListener("timeupdate", onTime); + if (el.currentTime > 0 && !el.ended) send(); + }; + }, [p.lesson.courseId, p.lesson.lesson.id, p.preview, start, v]); +} + +function ExerciseLesson(p: LessonViewProps & { chap: string; track: string | null }) { + const { lesson, track } = p; + const html = useLessonHtml(lesson, track); + const hintsHtml = useMemo(() => (lesson.hints.length + ? renderHintsHtml(lesson.hints, { courseId: lesson.courseId, baseFile: lesson.baseFile, tracks: lesson.tracks, track, lessonFiles: lesson.lessonFiles }) + : null), [lesson, track]); + const cmd = localCommandFor(lesson.localCommand, track); + return ( + <article className="wide"> + <div className="ex"> + <div> + <p className="chap">{p.chap}</p> + <h1 className="title"><TitleText text={lesson.lesson.title} /></h1> + {lesson.tracks.length > 1 && !lesson.browser ? <dl className="meta" style={{ paddingBottom: 16 }}><TrackSwitch lesson={lesson} track={track} onChange={p.onTrackChange} /></dl> : null} + {html ? <Prose html={html} style={{ marginTop: 20 }} /> : <p className="muted" style={{ marginTop: 20 }}>The instructions for this exercise are missing.</p>} + {hintsHtml ? ( + <div className="hints"> + <h2 className="sec-h" style={{ marginTop: 26 }}>Hints</h2> + <Prose html={hintsHtml} /> + </div> + ) : null} + {lesson.browser && cmd ? ( + <p className="local"><strong>Runs in your browser; the full project runs on your machine.</strong> To run the whole module locally:<code>{cmd}</code></p> + ) : null} + </div> + <ExercisePanel lesson={lesson} track={track} signedIn={!!p.signedIn} preview={p.preview} runner={p.runner} + onSaved={p.onSaved} /> + </div> + <Attachments files={lesson.attachments} note="This section" /> + </article> + ); +} diff --git a/packages/feature-learner/src/web/lesson/Prose.tsx b/packages/feature-learner/src/web/lesson/Prose.tsx new file mode 100644 index 0000000..9c75ad7 --- /dev/null +++ b/packages/feature-learner/src/web/lesson/Prose.tsx @@ -0,0 +1,72 @@ +import { useEffect, useRef, type CSSProperties } from "react"; +import { useInRouterContext, useNavigate } from "react-router-dom"; + +/** Router navigation when a router is present (the studio may embed LessonView without one). */ +export function useSafeNavigate(): ((to: string) => void) | null { + const inRouter = useInRouterContext(); + // The router context of a mounted tree never changes, so the hook order is stable. + // eslint-disable-next-line react-hooks/rules-of-hooks + return inRouter ? useNavigate() : null; +} + +/** + * Sets up hint tiers inside `root`: later tiers of a group wait (`data-wait`) until the previous tier was opened. + * Returns a cleanup function. + */ +export function wireHintTiers(root: HTMLElement): () => void { + const tiers = (group: string) => Array.from(root.querySelectorAll<HTMLDetailsElement>(`details.fold[data-group="${CSS.escape(group)}"]`)); + for (const d of root.querySelectorAll<HTMLDetailsElement>("details.fold[data-tier]")) { + if (Number(d.dataset.tier) > 1 && !d.dataset.seen) { + d.dataset.wait = "1"; + const sum = d.querySelector("summary"); + sum?.setAttribute("title", `Open hint ${Number(d.dataset.tier) - 1} first`); + sum?.setAttribute("aria-disabled", "true"); + } + } + const onClick = (e: MouseEvent) => { + const sum = (e.target as Element).closest?.("summary"); + const d = sum?.parentElement as HTMLDetailsElement | null; + if (!d || !root.contains(d) || !d.matches("details.fold[data-wait]")) return; + e.preventDefault(); + }; + const onToggle = (e: Event) => { + const d = e.target as HTMLDetailsElement; + if (!d.matches?.("details.fold[data-group]") || !d.open) return; + if (d.dataset.wait) { d.open = false; return; } + d.dataset.seen = "1"; + const next = tiers(d.dataset.group!).find((x) => Number(x.dataset.tier) === Number(d.dataset.tier) + 1); + if (next && next.dataset.wait) { + delete next.dataset.wait; + const sum = next.querySelector("summary"); + sum?.removeAttribute("title"); + sum?.removeAttribute("aria-disabled"); + } + }; + root.addEventListener("click", onClick); + root.addEventListener("toggle", onToggle, true); + return () => { root.removeEventListener("click", onClick); root.removeEventListener("toggle", onToggle, true); }; +} + +/** Sanitized lesson HTML (from `renderLessonHtml`) with hint tiers and in-app links. */ +export function Prose({ html, className = "prose", style }: { html: string; className?: string; style?: CSSProperties }) { + const ref = useRef<HTMLDivElement>(null); + const navigate = useSafeNavigate(); + useEffect(() => { + const root = ref.current; + if (!root) return; + const off = wireHintTiers(root); + const onLink = (e: MouseEvent) => { + const a = (e.target as Element).closest?.("a"); + if (!a || !navigate || e.defaultPrevented || e.button !== 0 || e.metaKey || e.ctrlKey || e.shiftKey || e.altKey) return; + const href = a.getAttribute("href") ?? ""; + if (href.startsWith("/c/")) { e.preventDefault(); navigate(href); } + else if (href.startsWith("#")) { + const el = document.getElementById(decodeURIComponent(href.slice(1))); + if (el) { e.preventDefault(); el.scrollIntoView({ behavior: "smooth", block: "start" }); } + } + }; + root.addEventListener("click", onLink); + return () => { off(); root.removeEventListener("click", onLink); }; + }, [html, navigate]); + return <div ref={ref} className={className} style={style} dangerouslySetInnerHTML={{ __html: html }} />; +} diff --git a/packages/feature-learner/src/web/lesson/TitleText.tsx b/packages/feature-learner/src/web/lesson/TitleText.tsx new file mode 100644 index 0000000..ca98413 --- /dev/null +++ b/packages/feature-learner/src/web/lesson/TitleText.tsx @@ -0,0 +1,7 @@ +/** Inline `code` spans in titles such as "Exercise 1.1 — `range`". */ +export function TitleText({ text }: { text: string }) { + const parts = text.split(/(`[^`]+`)/g); + return <>{parts.map((p, i) => (p.startsWith("`") && p.endsWith("`") && p.length > 2 ? <code key={i}>{p.slice(1, -1)}</code> : p))}</>; +} + +export const plainTitle = (t: string) => t.replace(/`/g, ""); diff --git a/packages/feature-learner/src/web/lesson/notes.tsx b/packages/feature-learner/src/web/lesson/notes.tsx new file mode 100644 index 0000000..a6d6697 --- /dev/null +++ b/packages/feature-learner/src/web/lesson/notes.tsx @@ -0,0 +1,171 @@ +import { useCallback, useEffect, useRef, useState, type ReactNode } from "react"; +import { Link } from "react-router-dom"; +import { Pill } from "@stc/ui"; +import type { Note } from "../../shared/types"; +import { api, fmtTime, lessonHref, toast } from "../api"; + +export interface NotesApi { + enabled: boolean; + notes: Note[]; + create(n: { anchorKind: Note["anchorKind"]; anchorValue?: number | null; deckId?: string | null; body: string }): Promise<Note | null>; + update(id: string, body: string): Promise<void>; + remove(id: string): Promise<void>; +} + +/** The signed-in user's notes for one lesson. Disabled (no requests) when signed out or in a preview. */ +export function useLessonNotes(courseId: string, lessonId: string, enabled: boolean): NotesApi { + const [notes, setNotes] = useState<Note[]>([]); + useEffect(() => { + setNotes([]); + if (!enabled) return; + let live = true; + api<Note[]>(`/api/courses/${encodeURIComponent(courseId)}/notes?lesson=${encodeURIComponent(lessonId)}`).then((n) => live && setNotes(n), () => {}); + return () => { live = false; }; + }, [courseId, lessonId, enabled]); + const create = useCallback<NotesApi["create"]>(async (n) => { + try { + const note = await api<Note>(`/api/courses/${encodeURIComponent(courseId)}/notes`, { method: "POST", body: { lessonId, ...n } }); + setNotes((xs) => [...xs, note]); + return note; + } catch (e) { toast(`The note was not saved: ${(e as Error).message}`); return null; } + }, [courseId, lessonId]); + const update = useCallback<NotesApi["update"]>(async (id, body) => { + try { + const note = await api<Note>(`/api/notes/${encodeURIComponent(id)}`, { method: "PATCH", body: { body } }); + setNotes((xs) => xs.map((x) => (x.id === id ? note : x))); + } catch (e) { toast(`The note was not saved: ${(e as Error).message}`); } + }, []); + const remove = useCallback<NotesApi["remove"]>(async (id) => { + try { + await api(`/api/notes/${encodeURIComponent(id)}`, { method: "DELETE" }); + setNotes((xs) => xs.filter((x) => x.id !== id)); + } catch (e) { toast(`The note was not deleted: ${(e as Error).message}`); } + }, []); + return { enabled, notes, create, update, remove }; +} + +const STAMP = /\[(?:(\d+):)?(\d{1,2}):(\d{2})\]/g; + +export function parseStamp(h: string | undefined, m: string, s: string): number { + return (h ? Number(h) * 3600 : 0) + Number(m) * 60 + Number(s); +} + +/** First "[m:ss]" timestamp in a note body, in seconds. */ +export function firstStamp(body: string): number | null { + const m = new RegExp(STAMP.source).exec(body); + return m ? parseStamp(m[1], m[2]!, m[3]!) : null; +} + +/** Note text with "[m:ss]" timestamps turned into seek buttons (or links to the lesson at that time). */ +export function NoteBody({ body, onSeek, courseId, lessonId }: { body: string; onSeek?: (s: number) => void; courseId?: string; lessonId?: string }) { + const parts: ReactNode[] = []; + let last = 0; + for (const m of body.matchAll(STAMP)) { + const t = parseStamp(m[1], m[2]!, m[3]!); + parts.push(body.slice(last, m.index)); + parts.push(onSeek + ? <button key={m.index} type="button" className="stamp" onClick={() => onSeek(t)} aria-label={`Play from ${fmtTime(t)}`}>{m[0]}</button> + : courseId && lessonId + ? <Link key={m.index} className="stamp" to={`${lessonHref(courseId, lessonId)}?t=${t}`}>{m[0]}</Link> + : m[0]); + last = m.index! + m[0].length; + } + parts.push(body.slice(last)); + return <div className="nb">{parts}</div>; +} + +/** Your notes for a lesson: a composer (with "Insert time" for video) and the list of lesson and video notes. */ +export function NotesPanel({ notes, video, label = "Your notes" }: { + notes: NotesApi; + video?: { currentTime: () => number; seek: (s: number) => void } | null; + label?: string; +}) { + const [draft, setDraft] = useState(""); + const [now, setNow] = useState(0); + const [busy, setBusy] = useState(false); + const ta = useRef<HTMLTextAreaElement>(null); + useEffect(() => { + if (!video) return; + const t = setInterval(() => setNow(video.currentTime()), 500); + return () => clearInterval(t); + }, [video]); + + if (!notes.enabled) { + return ( + <section> + <div className="blockhead"><h2>{label}</h2></div> + <p className="muted" style={{ fontSize: 13.5, margin: 0 }}><Link to="/account">Sign in</Link> to keep notes on this lesson. Only you see them.</p> + </section> + ); + } + + const insertTime = () => { + const el = ta.current; + const ins = `[${fmtTime(video!.currentTime())}] `; + const p = el?.selectionStart ?? draft.length, e = el?.selectionEnd ?? draft.length; + const pre = draft.slice(0, p) + (p && draft[p - 1] !== "\n" ? "\n" : ""); + const next = pre + ins + draft.slice(e); + setDraft(next); + requestAnimationFrame(() => { el?.focus(); const c = pre.length + ins.length; el?.setSelectionRange(c, c); }); + }; + const save = async () => { + const body = draft.trim(); + if (!body) return; + setBusy(true); + const t = video ? firstStamp(body) : null; + const ok = await notes.create(t !== null ? { anchorKind: "video", anchorValue: t, body } : { anchorKind: "none", body }); + setBusy(false); + if (ok) setDraft(""); + }; + const list = notes.notes.filter((n) => n.anchorKind !== "slide"); + + return ( + <section> + <div className="blockhead"> + <h2>{label}</h2> + {video ? <Pill className="sm" onClick={insertTime}>Insert time <span className="tnum">{fmtTime(now)}</span></Pill> : null} + </div> + <textarea + ref={ta} className="note-ta" value={draft} onChange={(e) => setDraft(e.target.value)} + aria-label="New note" placeholder={video ? "Only you see these. Insert time adds a link back to this point in the video." : "Only you see these."} + /> + <div className="note-actions"> + <Pill className="sm" variant="primary" disabled={!draft.trim() || busy} onClick={save}>Save note</Pill> + </div> + {list.length ? ( + <ul className="note-list"> + {list.map((n) => <NoteItem key={n.id} note={n} notes={notes} onSeek={video?.seek} />)} + </ul> + ) : null} + </section> + ); +} + +export function NoteItem({ note, notes, onSeek, courseId, extra }: { note: Note; notes: Pick<NotesApi, "update" | "remove">; onSeek?: (s: number) => void; courseId?: string; extra?: ReactNode }) { + const [editing, setEditing] = useState(false); + const [text, setText] = useState(note.body); + useEffect(() => { setText(note.body); }, [note.body]); + if (editing) { + return ( + <li> + <textarea className="note-ta" value={text} onChange={(e) => setText(e.target.value)} aria-label="Edit note" autoFocus + onKeyDown={(e) => { if (e.key === "Escape") { setEditing(false); setText(note.body); } }} /> + <div className="note-actions"> + <Pill className="sm" variant="ghost" onClick={() => { setEditing(false); setText(note.body); }}>Cancel</Pill> + <Pill className="sm" variant="primary" disabled={!text.trim()} onClick={async () => { await notes.update(note.id, text.trim()); setEditing(false); }}>Save</Pill> + </div> + </li> + ); + } + return ( + <li> + <NoteBody body={note.body} onSeek={onSeek} courseId={courseId ?? note.courseId} lessonId={note.lessonId} /> + <div className="nm"> + {extra} + <span>{new Date(note.updatedAt).toLocaleDateString(undefined, { day: "numeric", month: "short", year: "numeric" })}</span> + <button type="button" onClick={() => setEditing(true)}>Edit</button> + <button type="button" onClick={() => { if (confirm("Delete this note?")) void notes.remove(note.id); }}>Delete</button> + </div> + </li> + ); +} diff --git a/packages/feature-learner/src/web/library/LibraryPage.tsx b/packages/feature-learner/src/web/library/LibraryPage.tsx new file mode 100644 index 0000000..3b9131a --- /dev/null +++ b/packages/feature-learner/src/web/library/LibraryPage.tsx @@ -0,0 +1,218 @@ +import { useEffect, useMemo, useState, type CSSProperties } from "react"; +import { Link } from "react-router-dom"; +import { CourseCover, ProgressBar, SegmentedToggle } from "@stc/ui"; +import type { CourseCard, ContinueItem, Format, LibraryPayload, SearchHit } from "../../shared/types"; +import { api, courseHref, ensureStyles, fmtMinutes, lessonHref, refreshLibrary, useLibrary, useSearch } from "../api"; +import { HeaderExtras } from "../shell/HeaderExtras"; +import { accentStyle } from "../shell/CourseFrame"; + +const FORMAT_LABEL: Record<Format, string> = { text: "Text", video: "Video", slides: "Slides", exercises: "Exercises" }; +type LengthFilter = "any" | "short" | "medium" | "long"; +const LENGTHS: { value: LengthFilter; label: string; test: (m: number) => boolean }[] = [ + { value: "any", label: "Any length", test: () => true }, + { value: "short", label: "Under 2 h", test: (m) => m < 120 }, + { value: "medium", label: "2–10 h", test: (m) => m >= 120 && m <= 600 }, + { value: "long", label: "Over 10 h", test: (m) => m > 600 }, +]; + +export interface LibraryFilters { format: Format | "all"; length: LengthFilter; inProgress: boolean } + +export function filterCourses(courses: CourseCard[], f: LibraryFilters): CourseCard[] { + const len = LENGTHS.find((l) => l.value === f.length)!; + return courses.filter((c) => (f.format === "all" || c.formats.includes(f.format)) && len.test(c.minutes) && (!f.inProgress || (c.started && c.progress < 1))); +} + +export default function LibraryPage() { + ensureStyles(); + const lib = useLibrary(); + useEffect(() => { void refreshLibrary(); }, []); + useEffect(() => { document.title = "Library – STCourses"; }, []); + return ( + <div className="lr" data-look="work"> + <HeaderExtras course={null} lookMode={null} /> + {lib ? <LibraryView lib={lib} /> : <div className="lr-lib"><p className="lib-sub">Loading the library…</p></div>} + </div> + ); +} + +export function LibraryView({ lib, initialQuery }: { lib: LibraryPayload; initialQuery?: string }) { + const [storeQ] = useSearch(); + const q = (initialQuery ?? storeQ).trim(); + const [filters, setFilters] = useState<LibraryFilters>({ format: "all", length: "any", inProgress: false }); + const [hits, setHits] = useState<SearchHit[] | null>(null); + useEffect(() => { + if (!q) { setHits(null); return; } + let live = true; + const t = setTimeout(() => { + api<SearchHit[]>(`/api/library/search?q=${encodeURIComponent(q)}`).then((h) => live && setHits(h), () => live && setHits([])); + }, 180); + return () => { live = false; clearTimeout(t); }; + }, [q]); + + const byId = useMemo(() => new Map(lib.courses.map((c) => [c.id, c])), [lib]); + const filtering = filters.format !== "all" || filters.length !== "any" || filters.inProgress; + const browsing = !q && !filtering; + const formats = useMemo(() => (["text", "video", "slides", "exercises"] as Format[]).filter((f) => lib.courses.some((c) => c.formats.includes(f))), [lib]); + const n = lib.courses.length; + + let results: CourseCard[] = filterCourses(lib.courses, filters); + if (q && hits) { + const matched = new Set(hits.filter((h) => h.course).map((h) => h.courseId)); + results = results.filter((c) => matched.has(c.id)); + } + const lessonHits = (hits ?? []).filter((h) => h.lessons.length && byId.has(h.courseId) && filterCourses([byId.get(h.courseId)!], filters).length); + + return ( + <div className="lr-lib"> + <h1 className="lib-h">Library</h1> + <p className="lib-sub">{n === 0 ? "No courses you can open yet." : n === 1 ? "One course on this server." : `${n} courses on this server.`}{!lib.signedIn && n > 0 ? " Sign in to keep your progress and notes." : ""}</p> + + {browsing && lib.featured ? <Featured c={lib.featured} cont={lib.continue.find((x) => x.courseId === lib.featured!.id) ?? null} /> : null} + + {browsing && lib.continue.length ? ( + <section aria-labelledby="lr-cont"> + <h2 className="sec-h" id="lr-cont">Continue</h2> + <div className="continue-list"> + {lib.continue.slice(0, 3).map((c) => <ContinueRow key={c.courseId} item={c} course={byId.get(c.courseId)} />)} + </div> + </section> + ) : null} + + {n > 0 ? ( + <div className="filters" role="group" aria-label="Filters"> + {formats.length > 1 ? ( + <SegmentedToggle label="Format" value={filters.format} onChange={(v) => setFilters({ ...filters, format: v as LibraryFilters["format"] })} + options={[{ value: "all", label: "All formats" }, ...formats.map((f) => ({ value: f, label: FORMAT_LABEL[f] }))]} /> + ) : null} + <SegmentedToggle label="Length" value={filters.length} onChange={(v) => setFilters({ ...filters, length: v as LengthFilter })} + options={LENGTHS.map((l) => ({ value: l.value, label: l.label }))} /> + {lib.signedIn ? ( + <SegmentedToggle label="Progress" value={filters.inProgress ? "in" : "all"} onChange={(v) => setFilters({ ...filters, inProgress: v === "in" })} + options={[{ value: "all", label: "Everything" }, { value: "in", label: "In progress" }]} /> + ) : null} + </div> + ) : null} + + {browsing ? ( + <> + <Strip title="New in the library" courses={lib.newest.map((id) => byId.get(id)!).filter(Boolean).slice(0, 12)} signedIn={lib.signedIn} /> + {lib.collections.map((col) => ( + <Strip key={col.id} title={col.title} courses={col.courseIds.map((id) => byId.get(id)!).filter(Boolean)} signedIn={lib.signedIn} /> + ))} + <h2 className="sec-h">All courses</h2> + <CourseTable courses={lib.courses} signedIn={lib.signedIn} /> + </> + ) : ( + <> + <h2 className="sec-h">{q ? `Courses matching “${q}”` : "Courses"}</h2> + {q && hits === null ? <p className="lr-nohit">Searching…</p> : results.length ? <CourseTable courses={results} signedIn={lib.signedIn} /> : <p className="lr-nohit">No courses match.</p>} + {lessonHits.length ? ( + <> + <h2 className="sec-h" style={{ marginTop: 36 }}>Lessons</h2> + <ul className="hits"> + {lessonHits.flatMap((h) => h.lessons.map((l) => ( + <li key={`${h.courseId}/${l.id}`}> + <Link to={lessonHref(h.courseId, l.id)}> + <span><span className="h1">{l.title}</span><span className="h2" style={{ display: "block" }}>{h.courseTitle} · {l.moduleTitle}</span></span> + </Link> + </li> + )))} + </ul> + </> + ) : null} + </> + )} + </div> + ); +} + +function facts(c: CourseCard): string[] { + const out = [c.lessonCount === 1 ? "1 lesson" : `${c.lessonCount} lessons`]; + if (c.minutes) out.push(`about ${fmtMinutes(c.minutes)}`); + if (c.tracks.length > 1) out.push(`${c.tracks.length} tracks`); + return out; +} + +function Featured({ c, cont }: { c: CourseCard; cont: ContinueItem | null }) { + const target = cont ? lessonHref(c.id, cont.lessonId) : courseHref(c.id); + return ( + <section className="feat" aria-label="Featured course" style={accentStyle(c.accent)}> + <Link to={courseHref(c.id)} tabIndex={-1} aria-hidden="true" style={{ "--cw": "168px" } as CSSProperties}> + <CourseCover title={c.title} image={c.coverUrl ?? undefined} accent={c.accent ?? undefined} /> + </Link> + <div> + <p className="k">Featured</p> + <h2><Link to={courseHref(c.id)}>{c.title}</Link></h2> + {c.pitch ? <p className="pitch">{c.pitch}</p> : null} + <div className="facts">{facts(c).map((f) => <span key={f}>{f}</span>)}{c.formats.map((f) => <span key={f}>{FORMAT_LABEL[f]}</span>)}</div> + <div className="actions"> + <Link className="stc-pill stc-pill--primary" to={target}>{cont ? "Continue" : c.started ? "Open" : "Start"}</Link> + {c.started ? <span className="lr-prog" style={{ margin: 0, minWidth: 160 }}><ProgressBar value={c.progress} label="Progress" /><span className="tnum">{Math.round(c.progress * 100)}%</span></span> : null} + </div> + </div> + </section> + ); +} + +function ContinueRow({ item, course }: { item: ContinueItem; course: CourseCard | undefined }) { + const title = course?.title ?? item.courseId; + return ( + <div className="continue" style={accentStyle(course?.accent)}> + <span style={{ "--cw": "56px" } as CSSProperties}><CourseCover title={title} image={course?.coverUrl ?? undefined} accent={course?.accent ?? undefined} /></span> + <div style={{ minWidth: 0 }}> + <div className="t">{title}</div> + <div className="s">{item.moduleTitle}. Next up: {item.lessonTitle}</div> + <div className="lr-prog"><ProgressBar value={item.moduleTotal ? item.moduleDone / item.moduleTotal : 0} label="Module progress" /><span className="tnum">{item.moduleDone} of {item.moduleTotal} in this module</span></div> + </div> + <Link className="stc-pill stc-pill--primary" to={lessonHref(item.courseId, item.lessonId)}>Continue</Link> + </div> + ); +} + +function Strip({ title, courses, signedIn }: { title: string; courses: CourseCard[]; signedIn: boolean }) { + if (!courses.length) return null; + return ( + <section className="strip-wrap" aria-label={title}> + <h2 className="sec-h">{title}</h2> + <div className="strip"> + {courses.map((c) => ( + <Link key={c.id} className="tile" to={courseHref(c.id)} style={accentStyle(c.accent)}> + <CourseCover title={c.title} image={c.coverUrl ?? undefined} accent={c.accent ?? undefined} /> + <span className="tt">{c.title}</span> + <span className="ts">{facts(c).slice(0, 2).join(", ")}</span> + {signedIn && c.started ? <ProgressBar value={c.progress} label={`${c.title} progress`} /> : null} + </Link> + ))} + </div> + </section> + ); +} + +function CourseTable({ courses, signedIn }: { courses: CourseCard[]; signedIn: boolean }) { + return ( + <> + <div className="lib-cols" aria-hidden="true"><span>Course</span><span>Lessons</span><span>Contains</span><span>{signedIn ? "Progress" : "Length"}</span></div> + <div className="lib"> + {courses.map((c) => ( + <Link key={c.id} className="lib-row" to={courseHref(c.id)} style={accentStyle(c.accent)}> + <span className="lib-main"> + <span style={{ "--cw": "44px" } as CSSProperties}><CourseCover title={c.title} image={c.coverUrl ?? undefined} accent={c.accent ?? undefined} /></span> + <span style={{ minWidth: 0 }}> + <span className="lt">{c.title}</span> + <span className="ls">{c.pitch ?? c.authors.join(", ")}</span> + </span> + </span> + <span className="lnum tnum">{c.lessonCount}</span> + <span className="tags"> + {c.status === "error" ? <span className="tag bad">Has problems</span> : null} + {c.formats.map((f) => <span key={f} className="tag">{FORMAT_LABEL[f]}</span>)} + </span> + {signedIn + ? <span className="lr-prog"><ProgressBar value={c.progress} label={`${c.title} progress`} /><span className="tnum" style={{ minWidth: "3ch", textAlign: "right" }}>{Math.round(c.progress * 100)}%</span></span> + : <span className="lnum">{c.minutes ? fmtMinutes(c.minutes) : ""}</span>} + </Link> + ))} + </div> + </> + ); +} diff --git a/packages/feature-learner/src/web/shell/CourseFrame.tsx b/packages/feature-learner/src/web/shell/CourseFrame.tsx new file mode 100644 index 0000000..c3e472a --- /dev/null +++ b/packages/feature-learner/src/web/shell/CourseFrame.tsx @@ -0,0 +1,152 @@ +import { useEffect, useMemo, useState, type CSSProperties, type ReactNode } from "react"; +import { Link, useLocation } from "react-router-dom"; +import { CourseCover, ProgressBar } from "@stc/ui"; +import type { CoursePayload, OutlineLesson, OutlineModule } from "../../shared/types"; +import { courseHref, ensureStyles, fmtMinutes, fmtTime, lessonHref, useSearch, type Look } from "../api"; +import { Check, Chev, Lock } from "../icons"; +import { HeaderExtras } from "./HeaderExtras"; +import { TitleText } from "../lesson/TitleText"; + +const fold = (s: string) => s.normalize("NFKD").replace(/[̀-ͯ]/g, "").toLowerCase(); + +export function lessonKind(l: OutlineLesson): string { + if (l.type === "video") return l.minutes ? (l.minutes < 60 ? fmtTime(l.minutes * 60) : fmtMinutes(l.minutes)) : "Video"; + if (l.type === "exercise") return "Exercise"; + return l.hasDeck ? "Slides" : "Text"; +} + +export function courseDone(c: CoursePayload): { done: number; total: number } { + const all = c.modules.flatMap((m) => m.lessons); + return { done: all.filter((l) => c.progress[l.id]?.state === "done").length, total: all.length }; +} + +/** Display number of a module: courses whose first module id ends in 0 ("m00") count from 0. */ +export function moduleNumber(course: { modules: { id: string }[] }, index: number): number { + return /(^|\D)0+$/.test(course.modules[0]?.id ?? "") ? index : index + 1; +} + +export function accentStyle(accent: string | null | undefined): CSSProperties | undefined { + return accent ? ({ "--accent": accent } as CSSProperties) : undefined; +} + +/** The course page frame: outline sidebar (an overlay on narrow screens) and the main column. */ +export function CourseFrame({ course, currentLessonId, look, lookMode, children }: { + course: CoursePayload; + currentLessonId: string | null; + look: Look; + lookMode: "toggle" | "work" | null; + children: ReactNode; +}) { + ensureStyles(); + const [open, setOpen] = useState(false); + const loc = useLocation(); + useEffect(() => { setOpen(false); }, [loc.pathname]); + useEffect(() => { + if (!open) return; + const onKey = (e: KeyboardEvent) => { if (e.key === "Escape") setOpen(false); }; + document.addEventListener("keydown", onKey); + return () => document.removeEventListener("keydown", onKey); + }, [open]); + return ( + <div className={open ? "lr lr-shell side-open" : "lr lr-shell"} data-look={look} style={accentStyle(course.accent)}> + <HeaderExtras + course={{ id: course.id, title: course.title, coverUrl: course.coverUrl, accent: course.accent }} + lookMode={lookMode} + outline={{ open, toggle: () => setOpen(!open) }} + /> + <aside className="lr-side" id="lr-side" aria-label="Course outline"> + <Outline course={course} currentLessonId={currentLessonId} /> + </aside> + <div className="lr-scrim" onClick={() => setOpen(false)} /> + <main className="lr-main">{children}</main> + </div> + ); +} + +export function Outline({ course, currentLessonId }: { course: CoursePayload; currentLessonId: string | null }) { + const [q] = useSearch(); + const query = fold(q.trim()); + const current = course.modules.find((m) => m.lessons.some((l) => l.id === currentLessonId)); + const [openMods, setOpenMods] = useState<Set<string>>(() => new Set(current ? [current.id] : course.modules[0] ? [course.modules[0].id] : [])); + useEffect(() => { if (current) setOpenMods((s) => (s.has(current.id) ? s : new Set([...s, current.id]))); }, [current?.id]); + const loc = useLocation(); + useEffect(() => { + document.querySelector(".lr-les[aria-current='page']")?.scrollIntoView({ block: "nearest" }); + }, [currentLessonId]); + const { done, total } = courseDone(course); + const pct = total ? done / total : 0; + const notesPath = `${courseHref(course.id)}/notes`; + + const filtered = useMemo(() => course.modules + .map((m) => ({ m, lessons: query ? m.lessons.filter((l) => fold(l.title).includes(query)) : m.lessons })) + .filter(({ m, lessons }) => !query || lessons.length || fold(m.title).includes(query)), [course, query]); + + const toggle = (id: string) => setOpenMods((s) => { const n = new Set(s); if (n.has(id)) n.delete(id); else n.add(id); return n; }); + const sourceLine = course.authors.length ? course.authors.join(", ") : course.tracks.length > 1 ? `Tracks: ${course.tracks.map((t) => t.label).join(", ")}` : course.pitch ?? ""; + const errors = course.diagnostics.filter((d) => d.level === "error"); + + return ( + <> + <div className="lr-side-head"> + <span style={{ "--cw": "36px" } as CSSProperties}><CourseCover title={course.title} image={course.coverUrl ?? undefined} accent={course.accent ?? undefined} /></span> + <div style={{ flex: 1, minWidth: 0 }}> + <h2><Link to={courseHref(course.id)}>{course.title}</Link></h2> + {sourceLine ? <div className="src">{sourceLine}</div> : null} + {course.signedIn ? ( + <div className="lr-prog"><ProgressBar value={pct} label="Course progress" /><span className="tnum">{Math.round(pct * 100)}%</span></div> + ) : null} + <div className="lr-side-links"> + <Link to={courseHref(course.id)} aria-current={loc.pathname === courseHref(course.id) ? "page" : undefined}>Overview</Link> + {course.signedIn ? <Link to={notesPath} aria-current={loc.pathname === notesPath ? "page" : undefined}>Your notes</Link> : null} + </div> + </div> + </div> + {errors.length ? ( + <div className="lr-diag" role="note"> + <b>{errors.length === 1 ? "1 problem" : `${errors.length} problems`} in this course</b> (only authors see this) + <ul>{errors.slice(0, 6).map((d, i) => <li key={i}>{d.lessonId ? `${d.lessonId}: ` : ""}{d.message}{d.path ? ` (${d.path})` : ""}</li>)}</ul> + </div> + ) : null} + {filtered.map(({ m, lessons }) => ( + <ModuleBlock key={m.id} course={course} m={m} lessons={lessons} open={!!query || openMods.has(m.id)} current={m.id === current?.id} currentLessonId={currentLessonId} onToggle={() => toggle(m.id)} /> + ))} + {query && filtered.length === 0 ? <p className="lr-nohit">No lessons match “{q.trim()}”.</p> : null} + </> + ); +} + +function ModuleBlock({ course, m, lessons, open, current, currentLessonId, onToggle }: { + course: CoursePayload; m: OutlineModule; lessons: OutlineLesson[]; open: boolean; current: boolean; currentLessonId: string | null; onToggle: () => void; +}) { + const d = m.lessons.filter((l) => course.progress[l.id]?.state === "done").length; + const pct = m.lessons.length ? d / m.lessons.length : 0; + const number = String(moduleNumber(course, m.index)); + return ( + <div className={current ? "lr-mod is-current" : "lr-mod"}> + <button type="button" className="lr-mod-head" aria-expanded={open} onClick={onToggle}> + <span className="n tnum">{m.locked ? <Lock /> : number}</span> + <span>{m.title}{m.locked ? <span className="sr"> (locked)</span> : null}</span> + <span className="c tnum">{course.signedIn ? `${d}/${m.lessons.length}` : m.lessons.length}</span> + <Chev /> + </button> + {course.signedIn ? <ProgressBar value={pct} dim={pct === 1 || !current} label={`${m.title} progress`} /> : <span style={{ display: "block", height: 4 }} />} + {open ? ( + <ol className="lr-lessons"> + {lessons.map((l) => { + const isDone = course.progress[l.id]?.state === "done"; + const cls = ["lr-les", isDone && "done", m.locked && "locked"].filter(Boolean).join(" "); + return ( + <li key={l.id}> + <Link className={cls} to={lessonHref(course.id, l.id)} aria-current={l.id === currentLessonId ? "page" : undefined}> + <span className="m">{isDone ? <Check /> : m.locked ? <Lock /> : null}</span> + <span><TitleText text={l.title} /><span className="sr">{isDone ? ", done" : l.id === currentLessonId ? ", current" : ""}</span></span> + <span className="d tnum">{lessonKind(l)}</span> + </Link> + </li> + ); + })} + </ol> + ) : null} + </div> + ); +} diff --git a/packages/feature-learner/src/web/shell/HeaderExtras.tsx b/packages/feature-learner/src/web/shell/HeaderExtras.tsx new file mode 100644 index 0000000..f0cdc9a --- /dev/null +++ b/packages/feature-learner/src/web/shell/HeaderExtras.tsx @@ -0,0 +1,138 @@ +import { useEffect, useLayoutEffect, useRef, useState } from "react"; +import { createPortal } from "react-dom"; +import { Link, useLocation } from "react-router-dom"; +import { CourseCover, SegmentedToggle } from "@stc/ui"; +import { courseHref, toast, useLibrary, useLook, useMe, useSearch, useToast } from "../api"; +import { DirI, Down, MenuI, SearchI } from "../icons"; + +export interface HeaderCourse { id: string; title: string; coverUrl: string | null; accent: string | null } + +const useIsoLayoutEffect = typeof window === "undefined" ? useEffect : useLayoutEffect; + +/** + * Course switcher, search, Reading/Work and the account link, portalled into the Phase 0 header (`.stc-top`). + * Rendered only after mount, so server-side rendering of a page skips it. + */ +export function HeaderExtras({ course, lookMode, outline }: { + course: HeaderCourse | null; + /** "toggle": show Reading/Work; "work": shown but fixed to Work (exercise pages); null: hidden. */ + lookMode: "toggle" | "work" | null; + outline?: { open: boolean; toggle: () => void } | null; +}) { + const [host, setHost] = useState<Element | null>(null); + useIsoLayoutEffect(() => { setHost(document.querySelector(".stc-top")); }, []); + const msg = useToast(); + if (!host) return null; + return ( + <> + {createPortal( + <div className="lr lr-h"> + {outline ? ( + <button type="button" className="lr-icon" aria-label="Course outline" aria-expanded={outline.open} aria-controls="lr-side" onClick={outline.toggle}><MenuI /></button> + ) : null} + <CourseSwitcher course={course} /> + <SearchBox placeholder={course ? "Search this course" : "Search the library"} /> + {lookMode ? <LookToggle mode={lookMode} /> : null} + <AccountLink /> + </div>, + host, + )} + {msg ? <div className="lr lr-toast" role="status">{msg}</div> : null} + </> + ); +} + +function LookToggle({ mode }: { mode: "toggle" | "work" }) { + const [look, setLook] = useLook(); + const value = mode === "work" ? "work" : look; + return ( + <SegmentedToggle + label="Look" + value={value} + options={[{ value: "reading", label: "Reading" }, { value: "work", label: "Work" }]} + onChange={(v) => { + setLook(v as "reading" | "work"); + if (mode === "work" && v === "reading") toast("Exercises always use Work. Reading applies to text lessons."); + }} + /> + ); +} + +function SearchBox({ placeholder }: { placeholder: string }) { + const [q, setQ] = useSearch(); + const loc = useLocation(); + const path = useRef(loc.pathname.split("/").slice(0, 3).join("/")); + // Clear the query when moving to another course or to the library. + useEffect(() => { + const p = loc.pathname.split("/").slice(0, 3).join("/"); + if (p !== path.current) { path.current = p; setQ(""); } + }, [loc.pathname, setQ]); + return ( + <label className="lr-search"> + <SearchI /> + <span className="sr">Search</span> + <input + type="search" value={q} placeholder={placeholder} autoComplete="off" + onChange={(e) => setQ(e.target.value)} + onKeyDown={(e) => { if (e.key === "Escape") { setQ(""); (e.target as HTMLInputElement).blur(); } }} + /> + </label> + ); +} + +function CourseSwitcher({ course }: { course: HeaderCourse | null }) { + const [open, setOpen] = useState(false); + const lib = useLibrary(); + const ref = useRef<HTMLDivElement>(null); + const loc = useLocation(); + useEffect(() => { setOpen(false); }, [loc.pathname]); + useEffect(() => { + if (!open) return; + const onDoc = (e: MouseEvent) => { if (!ref.current?.contains(e.target as Node)) setOpen(false); }; + const onKey = (e: KeyboardEvent) => { + if (e.key === "Escape") { setOpen(false); ref.current?.querySelector<HTMLButtonElement>("button")?.focus(); } + }; + document.addEventListener("mousedown", onDoc); + document.addEventListener("keydown", onKey); + ref.current?.querySelector<HTMLAnchorElement>(".lr-menu a")?.focus(); + return () => { document.removeEventListener("mousedown", onDoc); document.removeEventListener("keydown", onKey); }; + }, [open]); + const n = lib?.courses.length ?? 0; + return ( + <div className="lr-switch" ref={ref}> + <button type="button" className="lr-switch-btn" aria-haspopup="true" aria-expanded={open} onClick={() => setOpen(!open)}> + {course + ? <span className="lr-cover-tiny"><CourseCover title={course.title} image={course.coverUrl ?? undefined} accent={course.accent ?? undefined} /></span> + : <span className="lr-tile"><DirI /></span>} + <span className="t">{course ? course.title : "All courses"}</span> + <Down /> + </button> + {open ? ( + <ul className="lr-menu"> + <li> + <Link to="/" aria-current={course ? undefined : "page"}> + <span className="lr-tile"><DirI /></span> + <span><span className="mt">Library</span><span className="ms">{n === 1 ? "1 course" : `${n} courses`}</span></span> + </Link> + </li> + {(lib?.courses ?? []).map((c) => ( + <li key={c.id}> + <Link to={courseHref(c.id)} aria-current={course?.id === c.id ? "page" : undefined}> + <span className="lr-cover-tiny"><CourseCover title={c.title} image={c.coverUrl ?? undefined} accent={c.accent ?? undefined} /></span> + <span style={{ minWidth: 0 }}><span className="mt">{c.title}</span><span className="ms">{c.authors.join(", ") || (c.source === "authored" ? "Authored course" : c.source ? `Imported (${c.source})` : "")}</span></span> + </Link> + </li> + ))} + </ul> + ) : null} + </div> + ); +} + +function AccountLink() { + const { loaded, user } = useMe(); + if (!loaded) return null; + return user + ? <Link className="stc-pill stc-pill--ghost lr-acct" to="/account">{user.name || user.email}</Link> + : <Link className="stc-pill lr-acct" to="/account">Sign in</Link>; +} diff --git a/packages/feature-learner/src/web/styles.ts b/packages/feature-learner/src/web/styles.ts new file mode 100644 index 0000000..65e7288 --- /dev/null +++ b/packages/feature-learner/src/web/styles.ts @@ -0,0 +1,373 @@ +// The learner stylesheet, injected once by ensureStyles(). Kept as a string (not a .css text import) because +// Bun's dev server loses text imports on hot rebuilds. Scoped under .lr; no colours beyond the @stc/ui tokens. +export const learnerCss = String.raw` +/* Learner app. Scoped under .lr so the studio can embed LessonView without clashes. + Tokens come from @stc/ui (tokens.css); this file adds no colours of its own. */ + +.lr { --accent-dim: color-mix(in srgb, var(--accent) 55%, var(--ground)); } +.lr[data-look="reading"], .lr [data-look="reading"] { --pf: var(--serif); --ps: 19px; --pl: 1.75; --pc: var(--read); --col: 660px; --wide: 900px; --tf: var(--serif); --h1: 40px; } +.lr[data-look="work"], .lr [data-look="work"] { --pf: var(--sans); --ps: 15.5px; --pl: 1.6; --pc: var(--text); --col: 760px; --wide: 1180px; --tf: var(--sans); --h1: 28px; } +.lr [hidden] { display: none !important; } +.lr .tnum { font-variant-numeric: tabular-nums; } + +/* ---------- header extras (portalled into .stc-top) ---------- */ +.lr-h { display: contents; } +.stc-top > a:first-child { flex: 0 0 calc(var(--sidew) - 30px); } +.lr-icon { display: none; width: 34px; height: 34px; border-radius: 18px; border: 1px solid var(--line2); background: none; place-items: center; color: var(--muted); flex: none; } +.lr-icon:hover { background: var(--raised); } +.lr-switch { position: relative; flex: none; } +.lr-switch-btn { display: flex; align-items: center; gap: 10px; height: 34px; padding: 0 12px 0 5px; border: 1px solid var(--line2); border-radius: 18px; background: none; font-weight: 500; font-size: 13px; max-width: 300px; } +.lr-switch-btn:hover { background: var(--raised); } +.lr-switch-btn .t { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.lr-switch-btn svg { color: var(--faint); flex: none; } +.lr-menu { position: absolute; top: 42px; left: 0; width: 330px; max-height: calc(100vh - 80px); overflow-y: auto; margin: 0; padding: 6px; list-style: none; background: var(--side); border: 1px solid var(--line2); border-radius: 14px; z-index: 50; } +.lr-menu a { display: flex; gap: 12px; align-items: center; padding: 8px 10px; border-radius: 10px; text-decoration: none; } +.lr-menu a:hover, .lr-menu a:focus-visible { background: var(--raised); } +.lr-menu a[aria-current] .mt { color: var(--head); } +.lr-menu .mt { font-weight: 500; font-size: 13.5px; display: block; } +.lr-menu .ms { display: block; font-size: 12px; color: var(--faint); } +.lr-tile { width: 24px; display: grid; place-items: center; color: var(--faint); flex: none; } +.lr-search { flex: 1; max-width: 380px; display: flex; align-items: center; gap: 8px; height: 34px; padding: 0 12px; border: 1px solid var(--line2); border-radius: 18px; color: var(--faint); margin-left: auto; } +.lr-search:focus-within { border-color: var(--faint); } +.lr-search input { flex: 1; min-width: 0; border: 0; background: none; outline: 0; color: var(--text); font-size: 13px; } +.lr-search input::placeholder { color: var(--faint); } +.lr-search input::-webkit-search-cancel-button { filter: grayscale(1); } +.lr-acct { flex: none; } +.lr-cover-tiny { --cw: 24px; } +.lr-cover-tiny .stc-cover { border-radius: 4px; font-size: 9px; } + +/* ---------- course shell ---------- */ +.stc-main:has(> .lr-shell) { padding: 0; } +.lr-shell { display: flex; align-items: flex-start; } +.lr-side { position: sticky; top: var(--top); flex: none; width: var(--sidew); height: calc(100vh - var(--top)); overflow-y: auto; background: var(--side); border-right: 1px solid var(--line); overscroll-behavior: contain; } +.lr-scrim { display: none; } +.lr-main { flex: 1; min-width: 0; padding: 44px 40px 0; } +.lr-side-head { display: flex; gap: 12px; padding: 18px 16px 16px; border-bottom: 1px solid var(--line); } +.lr-side-head h2 { margin: 0; font-size: 14.5px; font-weight: 600; color: var(--head); line-height: 1.3; } +.lr-side-head h2 a { text-decoration: none; } +.lr-side-head .src { font-size: 12px; color: var(--faint); margin-top: 2px; } +.lr-side-links { display: flex; gap: 14px; margin-top: 8px; font-size: 12px; } +.lr-side-links a { color: var(--muted); text-decoration: none; } +.lr-side-links a:hover, .lr-side-links a[aria-current] { color: var(--head); } +.lr-prog { display: flex; align-items: center; gap: 10px; margin-top: 10px; font-size: 12px; color: var(--muted); } +.lr-mod { border-bottom: 1px solid var(--line); } +.lr-mod-head { width: 100%; display: grid; grid-template-columns: 22px 1fr auto 12px; gap: 6px; align-items: baseline; padding: 11px 16px 7px; border: 0; background: none; text-align: left; font-size: 13.5px; color: var(--text); } +.lr-mod-head:hover { color: var(--head); } +.lr-mod-head .n, .lr-mod-head .c { color: var(--faint); font-size: 12px; } +.lr-mod-head svg { color: var(--faint); align-self: center; } +.lr-mod-head > svg { transition: transform var(--fast); } +.lr-mod-head[aria-expanded="true"] > svg { transform: rotate(90deg); } +.lr-mod.is-current .lr-mod-head { color: var(--head); font-weight: 500; } +.lr-mod > .stc-bar { margin: 0 16px 11px 44px; flex: none; } +.lr-lessons { list-style: none; margin: -3px 0 0; padding: 0 0 8px; } +.lr-les { display: grid; grid-template-columns: 22px 1fr auto; gap: 6px; align-items: center; padding: 6px 16px; font-size: 13px; line-height: 1.35; color: var(--muted); text-decoration: none; } +.lr-les:hover { background: color-mix(in srgb, var(--raised) 60%, transparent); color: var(--text); } +.lr-les .m { width: 8px; height: 8px; border-radius: 50%; border: 1.5px solid var(--line2); justify-self: center; display: grid; place-items: center; } +.lr-les.done .m { border: 0; width: 12px; height: 12px; color: var(--faint); } +.lr-les.locked { color: var(--faint); } +.lr-les.locked .m { border: 0; width: 12px; height: 12px; color: var(--faint); } +.lr-les .d { font-size: 12px; color: var(--faint); } +.lr-les code, .lr-menu code { font-size: .92em; } +.lr-les[aria-current="page"] { background: var(--raised); color: var(--head); box-shadow: inset 2px 0 0 var(--accent); } +.lr-les[aria-current="page"] .m { border: 0; background: var(--accent); width: 8px; height: 8px; } +.lr-les[aria-current="page"] .m svg { display: none; } +.lr-les[aria-current="page"].done .m svg { display: block; } +.lr-les[aria-current="page"].done .m { background: none; color: var(--accent); width: 12px; height: 12px; } +.lr-nohit { padding: 16px; color: var(--faint); font-size: 13px; } +.lr-diag { margin: 12px 16px; padding: 10px 12px; border: 1px solid var(--line2); border-radius: 10px; font-size: 12px; color: var(--muted); } +.lr-diag b { color: var(--bad); font-weight: 500; } +.lr-diag ul { margin: 6px 0 0; padding-left: 16px; } + +/* ---------- columns and headings ---------- */ +.lr .col { max-width: var(--col); margin: 0 auto; } +.lr .wide { max-width: var(--wide); margin: 0 auto; } +.lr .chap { font-size: 13px; font-weight: 500; color: var(--muted); margin: 0 0 10px; } +.lr h1.title { font: 600 var(--h1)/1.12 var(--tf); color: var(--head); margin: 0; letter-spacing: -.012em; text-wrap: balance; } +.lr h1.title code { font-size: .86em; } +.lr .meta { display: flex; flex-wrap: wrap; gap: 14px 30px; margin: 22px 0 0; padding-bottom: 22px; border-bottom: 1px solid var(--line); align-items: end; } +.lr .meta div { min-width: 0; } +.lr .meta dt { font-size: 12px; color: var(--faint); margin-bottom: 2px; } +.lr .meta dd { margin: 0; font-size: 13.5px; color: var(--text); } +.lr .meta dd code { font-size: 12.5px; } +.lr .source { font-size: 12.5px; color: var(--faint); margin: 14px 0 0; } +.lr .source strong { font-weight: 500; color: var(--muted); } +.lr .blockhead { display: flex; align-items: baseline; justify-content: space-between; gap: 12px; margin: 0 0 10px; flex-wrap: wrap; } +.lr .blockhead h2 { margin: 0; font: 600 15px/1.3 var(--sans); color: var(--head); } +.lr .blockhead > span { font-size: 12px; color: var(--faint); } +.lr .sec-h { font: 600 14px var(--sans); color: var(--muted); margin: 0 0 10px; } +.lr .muted { color: var(--faint); } +.lr .lr-block { margin-top: 40px; } + +/* ---------- prose ---------- */ +.lr .prose { font-family: var(--pf); font-size: var(--ps); line-height: var(--pl); color: var(--pc); overflow-wrap: break-word; } +.lr .prose p, .lr .prose ul, .lr .prose ol, .lr .prose dl, .lr .prose table { margin: 0 0 1em; } +.lr .prose > :first-child { margin-top: 0; } +.lr .prose h2 { font: 600 1.32em/1.25 var(--tf); color: var(--head); margin: 2em 0 .55em; letter-spacing: -.005em; } +.lr .prose h3 { font: 600 1.08em/1.3 var(--tf); color: var(--head); margin: 1.7em 0 .45em; } +.lr .prose h4, .lr .prose h5, .lr .prose h6 { font: 600 1em/1.35 var(--tf); color: var(--head); margin: 1.4em 0 .4em; } +.lr .prose strong { font-weight: 600; color: var(--head); } +.lr .prose li { margin-bottom: .35em; padding-left: .2em; } +.lr .prose a { color: var(--head); text-decoration-color: var(--line2); text-underline-offset: 3px; } +.lr .prose a:hover { text-decoration-color: var(--accent); } +.lr .prose img { max-width: 100%; height: auto; border-radius: 8px; } +.lr .prose blockquote { border-left: 2px solid var(--line2); padding-left: 18px; color: var(--muted); font-size: .92em; margin: 0 0 1.2em; } +.lr .prose blockquote strong { color: var(--text); } +.lr .prose hr { border: 0; border-top: 1px solid var(--line); margin: 2em 0; } +.lr .prose table { border-collapse: collapse; font-family: var(--sans); font-size: 14px; display: block; overflow-x: auto; } +.lr .prose th, .lr .prose td { border-bottom: 1px solid var(--line); padding: 6px 12px 6px 0; text-align: left; vertical-align: top; } +.lr .prose th { color: var(--muted); font-weight: 500; } +.lr .prose :not(pre) > code { font-size: .78em; padding: .08em .32em; background: color-mix(in srgb, var(--raised) 75%, transparent); border-radius: 4px; color: var(--text); } +.lr pre.code { font-size: 13.5px; line-height: 1.6; background: var(--side); border: 1px solid var(--line); border-radius: 10px; padding: 14px 16px; margin: 0 0 1.2em; overflow-x: auto; color: var(--text); tab-size: 2; } +.lr pre.code code { font-size: inherit; padding: 0; background: none; } +.lr .prose kbd { font: 500 .7em/1 var(--mono); padding: 2px 5px; border: 1px solid var(--line2); border-radius: 4px; } + +/* folds and hint tiers */ +.lr details.fold { border-top: 1px solid var(--line); border-bottom: 1px solid var(--line); margin: 0 0 1.4em; } +.lr details.fold + details.fold { margin-top: -1.4em; border-top: 0; } +.lr details.fold > summary { list-style: none; display: flex; align-items: center; gap: 10px; padding: 12px 2px; font: 500 14px/1.4 var(--sans); color: var(--text); cursor: pointer; } +.lr details.fold > summary::-webkit-details-marker { display: none; } +.lr details.fold > summary svg { color: var(--faint); transition: transform var(--fast); flex: none; } +.lr details.fold[open] > summary svg { transform: rotate(90deg); } +.lr details.fold > summary:hover { color: var(--head); } +.lr details.fold > summary .tier { margin-left: auto; font-size: 12px; color: var(--faint); font-weight: 400; white-space: nowrap; } +.lr details.fold[data-wait] > summary { color: var(--faint); cursor: default; } +.lr details.fold > .in { padding: 2px 0 14px 24px; } +.lr .hints .fold > .in { font-size: .95em; } + +/* ---------- pills ---------- */ +.lr .stc-pill.sm { height: 28px; padding: 0 11px; font-size: 12.5px; } +.lr a.stc-pill { color: inherit; } +.lr a.stc-pill--primary { color: var(--on-accent); } + +/* ---------- floating action bar ---------- */ +.lr .stc-dock [aria-pressed="true"] { color: var(--accent); } +.lr .stc-dock .sep { width: 1px; background: var(--line2); margin: 7px 2px; } +.lr .stc-dock a.stc-pill[aria-disabled="true"] { opacity: .45; pointer-events: none; } + +/* ---------- video ---------- */ +.lr .player { aspect-ratio: 16 / 9; background: #000; border-radius: 10px; overflow: hidden; margin-top: 22px; } +.lr .player video { display: block; width: 100%; height: 100%; } +.lr .split { display: grid; grid-template-columns: minmax(0, 1fr) 300px; gap: 40px; margin-top: 28px; } +.lr .note-ta { width: 100%; min-height: 110px; resize: vertical; background: var(--side); border: 1px solid var(--line); border-radius: 10px; padding: 12px 14px; color: var(--text); font-size: 14px; line-height: 1.55; display: block; } +.lr .note-ta:focus { outline: 0; border-color: var(--faint); } +.lr .note-actions { display: flex; gap: 6px; justify-content: flex-end; margin-top: 8px; align-items: center; } +.lr .note-actions .msg { margin-right: auto; font-size: 12px; color: var(--faint); } +.lr .note-list { list-style: none; margin: 18px 0 0; padding: 0; border-top: 1px solid var(--line); } +.lr .note-list > li { padding: 12px 2px; border-bottom: 1px solid var(--line); font-size: 14px; line-height: 1.55; } +.lr .note-list .nb { white-space: pre-wrap; overflow-wrap: anywhere; } +.lr .note-list .nm { display: flex; gap: 10px; align-items: center; margin-top: 6px; font-size: 12px; color: var(--faint); } +.lr .note-list .nm button { border: 0; background: none; padding: 0; color: var(--faint); font-size: 12px; } +.lr .note-list .nm button:hover { color: var(--text); } +.lr .stamp { border: 0; background: none; padding: 0 2px; color: var(--accent); font: inherit; font-variant-numeric: tabular-nums; text-decoration: underline; text-decoration-color: var(--accent-dim); text-underline-offset: 3px; } +.lr .transcript { margin-top: 22px; } + +/* rows (attachments) */ +.lr .rows { list-style: none; margin: 0; padding: 0; border-top: 1px solid var(--line); } +.lr .rows li { display: grid; grid-template-columns: 20px minmax(0, 1fr) auto; gap: 10px; align-items: center; padding: 10px 2px; border-bottom: 1px solid var(--line); font-size: 13.5px; } +.lr .rows li svg { color: var(--faint); } +.lr .rows a { overflow-wrap: anywhere; text-decoration: none; } +.lr .rows a:hover { text-decoration: underline; text-underline-offset: 3px; } +.lr .rows .sz { white-space: nowrap; font-size: 12px; color: var(--faint); } + +/* ---------- slides ---------- */ +.lr .deck { margin-top: 28px; } +.lr .stage { position: relative; aspect-ratio: 16 / 9; background: var(--side); border: 1px solid var(--line); border-radius: 8px; overflow: hidden; display: grid; place-items: center; margin: 0; } +.lr .stage canvas { display: block; max-width: 100%; max-height: 100%; } +.lr .stage .msg { color: var(--faint); font-size: 13px; padding: 20px; text-align: center; } +.lr .stage:fullscreen { border-radius: 0; background: #000; border: 0; } +.lr .deckbar { display: flex; align-items: center; gap: 8px; margin: 12px 0 0; flex-wrap: wrap; } +.lr .deckbar .count { font-size: 13px; color: var(--muted); margin: 0 6px; min-width: 58px; text-align: center; } +.lr .deckbar .gap { flex: 1; } +.lr .deckbar .hint { font-size: 12.5px; color: var(--faint); } +.lr .thumbs { display: flex; gap: 8px; margin: 16px 0 0; overflow-x: auto; padding: 3px 3px 6px; } +.lr .thumbs button { flex: none; width: 104px; aspect-ratio: 16 / 9; padding: 0; border: 1px solid var(--line2); border-radius: 5px; overflow: hidden; background: var(--side); opacity: .6; transition: opacity var(--fast); display: grid; place-items: center; color: var(--faint); font-size: 11px; position: relative; } +.lr .thumbs button:hover { opacity: .9; } +.lr .thumbs button[aria-current="true"] { opacity: 1; outline: 2px solid var(--accent); outline-offset: 2px; } +.lr .thumbs canvas { width: 100%; height: 100%; object-fit: contain; display: block; } +.lr .thumbs .has-note::after { content: ""; position: absolute; top: 4px; right: 4px; width: 6px; height: 6px; border-radius: 50%; background: var(--accent); } +.lr .slide-note { margin-top: 16px; } +.lr .slide-note label { display: block; font-size: 12px; color: var(--faint); margin-bottom: 6px; } +.lr .slide-note .note-ta { min-height: 72px; } + +/* ---------- code files ---------- */ +.lr .files { display: grid; grid-template-columns: 220px minmax(0, 1fr); border: 1px solid var(--line); border-radius: 10px; overflow: hidden; margin: 0 0 1.4em; font-family: var(--sans); } +.lr .tree { background: var(--side); border-right: 1px solid var(--line); padding: 8px 0; font-size: 13px; max-height: 520px; overflow: auto; } +.lr .tree .dir { padding: 5px 12px; color: var(--muted); display: flex; gap: 7px; align-items: center; white-space: nowrap; } +.lr .tree button { display: flex; gap: 7px; align-items: center; width: 100%; border: 0; background: none; text-align: left; padding: 5px 12px; color: var(--muted); font-size: 13px; white-space: nowrap; } +.lr .tree button:hover { color: var(--text); } +.lr .tree button[aria-current="true"] { background: var(--raised); color: var(--head); } +.lr .tree svg { flex: none; color: var(--faint); } +.lr .viewer { min-width: 0; display: flex; flex-direction: column; background: var(--ground); } +.lr .tabs { display: flex; align-items: center; border-bottom: 1px solid var(--line); padding: 0 6px 0 0; overflow-x: auto; min-height: 40px; } +.lr .tabs [role="tab"] { border: 0; background: none; padding: 10px 14px; font-size: 12.5px; color: var(--faint); border-bottom: 2px solid transparent; margin-bottom: -1px; white-space: nowrap; font-family: var(--mono); } +.lr .tabs [role="tab"][aria-selected="true"] { color: var(--head); border-bottom-color: var(--accent); } +.lr .tabs .stc-pill { margin-left: auto; } +.lr .path { font-size: 12px; color: var(--faint); padding: 8px 16px 0; overflow-wrap: anywhere; } +.lr .viewer pre.code { border: 0; border-radius: 0; background: none; margin: 0; counter-reset: ln; max-height: 520px; overflow: auto; } +.lr .ln { display: block; counter-increment: ln; } +.lr .ln::before { content: counter(ln); display: inline-block; width: 2.2em; margin-right: 1em; text-align: right; color: var(--faint); opacity: .6; } +.lr .cmd { position: relative; } +.lr .cmd pre.code { padding-right: 80px; white-space: pre-wrap; overflow-wrap: anywhere; } +.lr .cmd .stc-pill { position: absolute; top: 8px; right: 8px; } + +/* ---------- exercise ---------- */ +.lr .ex { display: grid; grid-template-columns: minmax(0, .9fr) minmax(0, 1.1fr); gap: 36px; align-items: start; } +.lr .ex .prose { font-size: 15px; } +.lr .edit { border: 1px solid var(--line); border-radius: 10px; overflow: hidden; position: sticky; top: calc(var(--top) + 20px); } +.lr .edit-bar { display: flex; align-items: center; gap: 6px; padding: 6px 6px 6px 14px; background: var(--side); border-bottom: 1px solid var(--line); } +.lr .edit-bar .f { font: 500 12.5px var(--mono); color: var(--muted); margin-right: auto; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.lr .editor { display: grid; grid-template-columns: 44px 1fr; background: var(--ground); height: 300px; } +.lr .gutter { margin: 0; padding: 12px 10px 12px 0; text-align: right; color: var(--faint); opacity: .6; font: 13.5px/1.6 var(--mono); overflow: hidden; user-select: none; } +.lr .editor textarea { border: 0; resize: none; outline: 0; background: none; padding: 12px 14px 12px 6px; font: 13.5px/1.6 var(--mono); color: var(--text); white-space: pre; overflow: auto; tab-size: 2; } +.lr .editor:focus-within { box-shadow: inset 0 0 0 1px var(--faint); } +.lr .results { border-top: 1px solid var(--line); background: var(--side); } +.lr .res-sum { display: flex; align-items: center; justify-content: space-between; gap: 10px; padding: 10px 14px; font-size: 13px; color: var(--muted); } +.lr .res-sum b { font-weight: 600; color: var(--head); } +.lr .res-list { list-style: none; margin: 0; padding: 0; max-height: 300px; overflow: auto; border-top: 1px solid var(--line); } +.lr .res-list li { padding: 8px 14px; border-bottom: 1px solid var(--line); font-size: 13px; } +.lr .res-list li:last-child { border-bottom: 0; } +.lr .res-row { display: grid; grid-template-columns: 40px 1fr; gap: 8px; } +.lr .res-row .s { font-weight: 600; font-size: 12px; } +.lr .pass .s { color: var(--ok); } +.lr .fail .s { color: var(--bad); } +.lr .res-row code { font-size: 12.5px; color: var(--text); } +.lr .diff { display: grid; grid-template-columns: 64px 1fr; gap: 2px 8px; margin: 6px 0 2px 48px; font-size: 12.5px; } +.lr .diff dt { color: var(--faint); } +.lr .diff dd { margin: 0; font-family: var(--mono); color: var(--text); white-space: pre-wrap; overflow-wrap: anywhere; } +.lr .local { margin: 20px 0 0; padding: 12px 0 0; border-top: 1px solid var(--line); font-size: 13px; color: var(--muted); } +.lr .local strong { color: var(--text); font-weight: 500; } +.lr .local code { display: block; margin-top: 6px; font-size: 12.5px; color: var(--text); white-space: pre-wrap; overflow-wrap: anywhere; } +.lr .local-panel { border: 1px solid var(--line); border-radius: 10px; padding: 16px; background: var(--side); } +.lr .local-panel h2 { margin: 0 0 6px; font: 600 15px/1.3 var(--sans); color: var(--head); } +.lr .local-panel p { margin: 0 0 12px; font-size: 13.5px; color: var(--muted); } +.lr .local-panel .cmd pre.code { background: var(--ground); } +.lr .local-panel textarea { min-height: 90px; font-family: var(--mono); font-size: 12.5px; } + +/* ---------- lock, empty, error ---------- */ +.lr .notice { border-top: 1px solid var(--line); border-bottom: 1px solid var(--line); padding: 18px 2px; margin-top: 28px; color: var(--muted); font-size: 14px; } +.lr .notice strong { color: var(--head); font-weight: 500; } +.lr .notice .stc-pill { margin-top: 12px; } + +/* ---------- library ---------- */ +.lr-lib { max-width: 1080px; margin: 0 auto; padding-bottom: 60px; } +.lr .lib-h { font: 600 24px/1.2 var(--sans); color: var(--head); margin: 0 0 4px; } +.lr .lib-sub { color: var(--faint); margin: 0 0 32px; font-size: 13.5px; } +.lr .feat { display: grid; grid-template-columns: auto minmax(0, 1fr); gap: 28px; align-items: end; padding: 0 0 32px; border-bottom: 1px solid var(--line); margin-bottom: 36px; } +.lr a:has(> .stc-cover), .lr a .stc-cover { text-decoration: none; } +.lr .feat .k { font-size: 12.5px; color: var(--faint); margin: 0 0 8px; } +.lr .feat h2 { margin: 0; font: 600 30px/1.15 var(--sans); color: var(--head); letter-spacing: -.01em; text-wrap: balance; } +.lr .feat h2 a { text-decoration: none; } +.lr .feat p.pitch { margin: 10px 0 0; font-size: 15px; color: var(--muted); max-width: 560px; } +.lr .feat .facts { display: flex; flex-wrap: wrap; gap: 6px 18px; margin: 14px 0 18px; font-size: 12.5px; color: var(--faint); } +.lr .feat .actions { display: flex; gap: 8px; align-items: center; } +.lr .continue { display: grid; grid-template-columns: auto 1fr auto; gap: 20px; align-items: center; padding: 18px 0; border-bottom: 1px solid var(--line); } +.lr .continue-list { border-top: 1px solid var(--line); margin-bottom: 44px; } +.lr .continue .t { font-size: 16px; font-weight: 600; color: var(--head); } +.lr .continue .s { font-size: 13.5px; color: var(--muted); margin: 2px 0 10px; } +.lr .continue .lr-prog { max-width: 360px; margin: 0; } +.lr .filters { display: flex; flex-wrap: wrap; gap: 10px 14px; align-items: center; margin: 0 0 30px; } +.lr .filters .lbl { font-size: 12px; color: var(--faint); margin-right: -6px; } +.lr .lib-row-h { display: flex; align-items: baseline; justify-content: space-between; gap: 12px; } +.lr .strip-wrap { margin-bottom: 40px; } +.lr .strip { display: flex; gap: 22px; overflow-x: auto; padding: 4px 2px 12px; scroll-snap-type: x proximity; } +.lr .tile { flex: none; width: 150px; text-decoration: none; scroll-snap-align: start; } +.lr .tile .stc-cover { --cw: 150px; } +.lr .tile .tt { display: block; margin-top: 10px; font-size: 13.5px; font-weight: 600; color: var(--text); line-height: 1.3; } +.lr .tile:hover .tt { color: var(--head); text-decoration: underline; text-underline-offset: 3px; text-decoration-color: var(--line2); } +.lr .tile .ts { display: block; font-size: 12px; color: var(--faint); margin-top: 2px; } +.lr .tile .stc-bar { margin-top: 8px; } +.lr .lib { border-top: 1px solid var(--line); } +.lr .lib-row, .lr .lib-cols { display: grid; grid-template-columns: minmax(0, 1fr) 90px 210px 180px; gap: 24px; align-items: center; } +.lr .lib-cols { padding: 0 0 8px; font-size: 12px; color: var(--faint); } +.lr .lib-row { padding: 16px 0; border-bottom: 1px solid var(--line); text-decoration: none; } +.lr .lib-row:hover .lt { color: var(--head); text-decoration: underline; text-underline-offset: 3px; text-decoration-color: var(--line2); } +.lr .lib-main { display: flex; gap: 16px; align-items: center; min-width: 0; } +.lr .lt { font-size: 15px; font-weight: 600; color: var(--text); } +.lr .ls { display: block; font-size: 13px; color: var(--faint); margin-top: 2px; } +.lr .lnum { font-size: 13.5px; color: var(--muted); } +.lr .tags { display: flex; flex-wrap: wrap; gap: 5px; } +.lr .tag { font-size: 11.5px; color: var(--muted); border: 1px solid var(--line2); border-radius: 10px; padding: 1px 8px; } +.lr .tag.bad { color: var(--bad); } +.lr .lib-row .lr-prog { margin: 0; } +.lr .hits { list-style: none; margin: 0 0 36px; padding: 0; border-top: 1px solid var(--line); } +.lr .hits li a { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: 12px; padding: 10px 2px; border-bottom: 1px solid var(--line); text-decoration: none; font-size: 13.5px; } +.lr .hits li a:hover .h1 { color: var(--head); text-decoration: underline; text-underline-offset: 3px; } +.lr .hits .h2 { font-size: 12px; color: var(--faint); } + +/* ---------- notes page, account ---------- */ +.lr .notes-group { margin-top: 30px; } +.lr .notes-group h2 { font: 600 15px/1.3 var(--sans); margin: 0 0 6px; } +.lr .notes-group h2 a { color: var(--head); text-decoration: none; } +.lr .notes-group h2 a:hover { text-decoration: underline; text-underline-offset: 3px; } +.lr .notes-group .note-list { margin-top: 8px; } +.lr .anchor { font-size: 12px; color: var(--muted); text-decoration: none; border: 1px solid var(--line2); border-radius: 10px; padding: 1px 8px; } +.lr .anchor:hover { color: var(--head); } + +.lr .form { display: grid; gap: 14px; max-width: 380px; } +.lr .form label { display: grid; gap: 5px; font-size: 12.5px; color: var(--muted); } +.lr .form input, .lr .field { height: 38px; padding: 0 12px; border: 1px solid var(--line2); border-radius: 10px; background: var(--side); color: var(--text); font-size: 14px; } +.lr .form input:focus, .lr .field:focus { outline: 0; border-color: var(--faint); } +.lr select.field { padding-right: 8px; } +.lr .form .err, .lr .err { color: var(--bad); font-size: 13px; margin: 0; } +.lr .ok-msg { color: var(--ok); font-size: 13px; margin: 0; } +.lr .or { display: flex; align-items: center; gap: 12px; color: var(--faint); font-size: 12px; margin: 26px 0 16px; max-width: 380px; } +.lr .or::before, .lr .or::after { content: ""; flex: 1; border-top: 1px solid var(--line); } +.lr .prov { display: flex; flex-wrap: wrap; gap: 8px; max-width: 380px; } +.lr .acct-dl { display: grid; grid-template-columns: 90px 1fr; gap: 8px 16px; margin: 22px 0; font-size: 14px; } +.lr .acct-dl dt { color: var(--faint); } +.lr .acct-dl dd { margin: 0; } +.lr .admin-sec { margin-top: 44px; } +.lr .admin-sec > h2 { font: 600 16px/1.3 var(--sans); color: var(--head); margin: 0 0 12px; } +.lr .table { width: 100%; border-collapse: collapse; font-size: 13.5px; } +.lr .table th { text-align: left; font-weight: 500; font-size: 12px; color: var(--faint); padding: 0 12px 8px 0; border-bottom: 1px solid var(--line); } +.lr .table td { padding: 10px 12px 10px 0; border-bottom: 1px solid var(--line); vertical-align: middle; } +.lr .table td .ls { margin: 0; } +.lr .checks { display: flex; flex-wrap: wrap; gap: 6px 16px; } +.lr .checks label { display: inline-flex; gap: 6px; align-items: center; font-size: 13px; color: var(--muted); } +.lr input[type="checkbox"] { accent-color: var(--accent); } +.lr .row-form { display: flex; gap: 8px; flex-wrap: wrap; align-items: center; margin: 12px 0 0; } + +.lr-toast { position: fixed; left: 50%; bottom: 24px; transform: translateX(-50%); z-index: 60; background: var(--raised); border: 1px solid var(--line2); border-radius: 18px; padding: 9px 16px; font-size: 13px; color: var(--text); } + +/* ---------- responsive ---------- */ +@media (max-width: 1100px) { + .lr .ex { grid-template-columns: minmax(0, 1fr); } + .lr .edit { position: static; } + .lr .split { grid-template-columns: 1fr; gap: 28px; } + .lr .lib-row, .lr .lib-cols { grid-template-columns: minmax(0, 1fr) 70px 150px; } + .lr .lib-row > .tags, .lr .lib-cols > :nth-child(3) { display: none; } +} +@media (max-width: 960px) { + .stc-top > a:first-child { flex: none; } + .lr-icon { display: inline-grid; } + .lr-side { position: fixed; top: var(--top); left: 0; bottom: 0; height: auto; z-index: 40; transform: translateX(-100%); transition: transform 160ms ease-out; } + .lr-shell.side-open .lr-side { transform: none; } + .lr-shell.side-open .lr-scrim { display: block; position: fixed; inset: var(--top) 0 0 0; z-index: 35; background: rgb(10 11 15 / .6); } + .lr-main { padding: 32px 24px 0; } +} +@media (max-width: 680px) { + .stc-top { gap: 8px; padding: 0 10px; } + .stc-top > a:first-child { display: none; } + .lr-search { display: none; } + .lr-switch-btn { max-width: 170px; } + .lr-switch-btn .t { font-size: 12.5px; } + .lr .stc-seg button { padding: 0 10px; } + .lr-main { padding: 26px 18px 0; } + .lr[data-look="reading"], .lr [data-look="reading"] { --ps: 18px; --h1: 31px; } + .lr .meta { gap: 12px 22px; } + .lr .files { grid-template-columns: 1fr; } + .lr .tree { border-right: 0; border-bottom: 1px solid var(--line); max-height: 200px; } + .lr .continue { grid-template-columns: auto 1fr; } + .lr .continue .stc-pill { grid-column: 1 / -1; justify-self: start; } + .lr .feat { grid-template-columns: 1fr; } + .lr .lib-row, .lr .lib-cols { grid-template-columns: minmax(0, 1fr) 92px; gap: 14px; } + .lr .lib-row > .lnum, .lr .lib-cols > :nth-child(2) { display: none; } + .lr-menu { width: min(330px, calc(100vw - 20px)); left: -48px; } + .lr .stc-dock { bottom: 12px; } + .lr .stc-dock .lbl-long { display: none; } + .lr .thumbs button { width: 84px; } +} +@media (prefers-reduced-motion: reduce) { + .lr *, .lr *::before, .lr *::after, .lr-side { transition: none !important; animation: none !important; scroll-behavior: auto !important; } +} +`; diff --git a/packages/feature-learner/test/account.test.tsx b/packages/feature-learner/test/account.test.tsx new file mode 100644 index 0000000..82273a5 --- /dev/null +++ b/packages/feature-learner/test/account.test.tsx @@ -0,0 +1,19 @@ +import { expect, test } from "bun:test"; +import { renderToString } from "react-dom/server"; +import { MemoryRouter } from "react-router-dom"; +import { SignInForms } from "../src/web/account/AccountPage"; +import AdminPage from "../src/web/account/AdminPage"; + +test("the sign-in page has email and password fields and a create-account switch", () => { + const out = renderToString(<MemoryRouter initialEntries={["/account?next=/c/sample"]}><SignInForms /></MemoryRouter>); + expect(out).toContain('type="email"'); + expect(out).toContain('type="password"'); + expect(out).toContain('autoComplete="current-password"'); + expect(out).toContain("Create account"); +}); + +test("the admin page renders its frame before the session is known", () => { + const out = renderToString(<MemoryRouter><AdminPage /></MemoryRouter>); + expect(out).toContain("Users, groups and library"); + expect(out).not.toContain("Users and roles"); // owner-only sections wait for the session +}); diff --git a/packages/feature-learner/test/assets.test.ts b/packages/feature-learner/test/assets.test.ts new file mode 100644 index 0000000..4f30eb1 --- /dev/null +++ b/packages/feature-learner/test/assets.test.ts @@ -0,0 +1,11 @@ +import { expect } from "bun:test"; +import { forEachDb } from "./helpers"; +import { makeApp } from "./app"; + +forEachDb("the pdf.js worker is served for client-side deck rendering", async (db) => { + const { call } = await makeApp(db); + const r = await call("GET", "/api/learner/pdf.worker.mjs"); + expect(r.status).toBe(200); + expect(r.headers.get("content-type")).toContain("javascript"); + expect((await r.text()).length).toBeGreaterThan(100_000); +}); diff --git a/packages/feature-learner/test/lesson-view.test.tsx b/packages/feature-learner/test/lesson-view.test.tsx new file mode 100644 index 0000000..60f8478 --- /dev/null +++ b/packages/feature-learner/test/lesson-view.test.tsx @@ -0,0 +1,108 @@ +import { expect, test } from "bun:test"; +import { renderToString } from "react-dom/server"; +import { MemoryRouter } from "react-router-dom"; +import { connect } from "@stc/db"; +import { LessonView } from "../src/web"; +import { LessonView as FromIndex } from "../src/index"; +import type { LessonPayload } from "../src/shared/types"; +import { localCommandFor } from "../src/web/lesson/Exercise"; +import { firstStamp } from "../src/web/lesson/notes"; +import { buildTree } from "../src/web/lesson/CodeFiles"; +import { runMigrations } from "../src/server/migrate"; +import { makeApp } from "./app"; + +async function sampleLessons(ids: string[]): Promise<Record<string, LessonPayload>> { + const db = await connect("sqlite://:memory:"); + await runMigrations(db); + const { call } = await makeApp(db); + const out: Record<string, LessonPayload> = {}; + for (const id of ids) out[id] = (await (await call("GET", `/api/courses/sample/lessons/${id}`)).json()) as LessonPayload; + await db.close(); + return out; +} + +const html = (el: React.ReactElement) => renderToString(<MemoryRouter>{el}</MemoryRouter>); + +test("LessonView is exported from the package index and the web entry", () => { + expect(FromIndex).toBe(LessonView); +}); + +test("LessonView renders every sample lesson type", async () => { + const l = await sampleLessons(["m1-intro", "m1-video", "m2-ex-add"]); + + const text = html(<LessonView lesson={l["m1-intro"]!} />); + expect(text).toContain('data-type="text"'); + expect(text).toContain('data-look="reading"'); + expect(text).toContain("<h1 class=\"title\">Introduction</h1>"); + expect(text).toContain("Welcome to the sample course"); + expect(text).toContain("Files for this section"); // module m1 has companion code + expect(text).toContain("/api/courses/sample/code/m1/zip"); + + const video = html(<LessonView lesson={l["m1-video"]!} signedIn={false} />); + expect(video).toContain('data-type="video"'); + expect(video).toMatch(/<video[^>]+src="\/api\/media\/sample\/media\/clip\.mp4"/); + expect(video).toContain('data-deck="d1"'); // companion deck + expect(video).toContain("Transcript"); + expect(video).toContain("Sign in"); // notes need an account + + const ex = html(<LessonView lesson={l["m2-ex-add"]!} look="reading" signedIn />); + expect(ex).toContain('data-look="work"'); // exercises always use Work + expect(ex).toContain("Run tests"); + expect(ex).toContain("return a - b;"); // the starter is in the editor + expect(ex).toContain('data-tier="1"'); + expect(ex).toContain('data-tiers="2"'); + expect(ex).toContain("node --test"); + expect(ex).not.toContain("return a + b"); // no solution for learners +}); + +test("preview mode renders without notes or saved-attempt UI", async () => { + const l = await sampleLessons(["m1-video"]); + const out = html(<LessonView lesson={l["m1-video"]!} preview />); + expect(out).not.toContain("Your notes"); + expect(out).not.toContain("Insert time"); +}); + +test("a locked lesson shows the lock notice and no content", () => { + const locked: LessonPayload = { + courseId: "c", courseTitle: "C", lesson: { id: "l2", title: "Later", type: "text", minutes: null }, + module: { id: "m2", title: "Two", index: 1 }, locked: true, markdown: null, baseFile: null, trackMarkdown: {}, tracks: [], + video: null, transcript: null, hints: [], decks: [], code: null, attachments: [], browser: null, localCommand: null, + prev: { id: "l1", title: "Earlier" }, next: null, lessonFiles: {}, + }; + const out = html(<LessonView lesson={locked} signedIn />); + expect(out).toContain("This module is locked."); + expect(out).toMatch(/Back to (<!-- -->)?Earlier/); +}); + +test("track variants: the chosen track's markdown and code are shown", () => { + const lesson: LessonPayload = { + courseId: "c", courseTitle: "C", lesson: { id: "t", title: "Tracks `x`", type: "text", minutes: 5 }, + module: { id: "m", title: "M", index: 0 }, locked: false, + markdown: "# T\n\n**JavaScript**\n\n```js\nconst a = 1\n```\n\n**Scala 3**\n\n```scala\nval a = 1\n```\n", + baseFile: "course/t.md", trackMarkdown: {}, tracks: [{ id: "js", label: "JavaScript" }, { id: "scala", label: "Scala 3" }], + video: null, transcript: null, hints: [], decks: [], code: null, attachments: [], browser: null, + localCommand: { js: "npm test", scala: "sbt test" }, prev: null, next: null, lessonFiles: {}, + }; + const js = html(<LessonView lesson={lesson} track="js" />); + expect(js).toContain("const a = 1"); + expect(js).not.toContain("val a = 1"); + expect(js).toContain("<code>x</code>"); // inline code in the title + expect(js).toContain('aria-label="Track"'); + expect(js).toContain("npm test"); + const scala = html(<LessonView lesson={lesson} track="scala" />); + expect(scala).toContain("val a = 1"); + expect(scala).toContain("sbt test"); +}); + +test("helpers: local command per track, note timestamps, code tree", () => { + expect(localCommandFor({ js: "a", scala: "b" }, "scala")).toBe("b"); + expect(localCommandFor({ js: "a" }, "fsharp")).toBe("a"); + expect(localCommandFor("x", null)).toBe("x"); + expect(firstStamp("see [1:05] and [0:10]")).toBe(65); + expect(firstStamp("[1:02:03] x")).toBe(3723); + expect(firstStamp("no stamp")).toBeNull(); + const t = buildTree([{ path: "a.txt", size: 1 }, { path: "src/b.ts", size: 2 }, { path: "src/x/c.ts", size: 3 }]); + expect(t.files.map((f) => f.name)).toEqual(["a.txt"]); + expect(t.dirs.get("src")!.files.map((f) => f.path)).toEqual(["src/b.ts"]); + expect(t.dirs.get("src")!.dirs.get("x")!.files[0]!.path).toBe("src/x/c.ts"); +}); diff --git a/packages/feature-learner/test/web-shell.test.tsx b/packages/feature-learner/test/web-shell.test.tsx new file mode 100644 index 0000000..b44945c --- /dev/null +++ b/packages/feature-learner/test/web-shell.test.tsx @@ -0,0 +1,99 @@ +import { expect, test } from "bun:test"; +import { renderToString } from "react-dom/server"; +import { MemoryRouter } from "react-router-dom"; +import { learnerWeb } from "../src/web"; +import { filterCourses, LibraryView } from "../src/web/library/LibraryPage"; +import { CourseFrame, moduleNumber } from "../src/web/shell/CourseFrame"; +import { nextLesson } from "../src/web/course/pages"; +import { safeNext } from "../src/web/account/AccountPage"; +import type { CourseCard, CoursePayload, LibraryPayload } from "../src/shared/types"; + +const card = (id: string, over: Partial<CourseCard> = {}): CourseCard => ({ + id, title: id.toUpperCase(), pitch: `${id} pitch`, authors: [], coverUrl: null, accent: null, formats: ["text"], lessonCount: 3, + minutes: 30, collections: [], tracks: [], source: "authored", addedAt: "2026-10-01T00:00:00Z", status: "ok", progress: 0, doneCount: 0, started: false, ...over, +}); + +const lib: LibraryPayload = { + signedIn: true, + featured: card("snake", { formats: ["text", "exercises"], minutes: 900 }), + continue: [{ courseId: "snake", lessonId: "m3", lessonTitle: "Pure game rules", moduleTitle: "Pure game rules", moduleIndex: 3, moduleDone: 1, moduleTotal: 5, progress: 0.2 }], + newest: ["docker", "snake"], + collections: [{ id: "devops", title: "DevOps", courseIds: ["docker"] }], + courses: [card("docker", { formats: ["video", "slides"], minutes: 300, started: true, progress: 0.5 }), card("snake", { formats: ["text", "exercises"], minutes: 900 })], +}; + +test("the learner web feature registers its routes", () => { + expect(learnerWeb.routes.map((r) => r.path)).toEqual(["/", "/c/:courseId", "/c/:courseId/notes", "/c/:courseId/:lessonId", "/account", "/account/admin"]); +}); + +test("every route module loads and has a default component", async () => { + for (const r of learnerWeb.routes) expect(typeof (await r.element()).default).toBe("function"); +}); + +test("library: featured course, continue row, browsable rows and the course list", () => { + const out = renderToString(<MemoryRouter><LibraryView lib={lib} /></MemoryRouter>); + expect(out).toContain("Featured"); + expect(out).toContain("SNAKE"); + expect(out).toContain("Next up: <!-- -->Pure game rules"); + expect(out).toContain("New in the library"); + expect(out).toContain("DevOps"); + expect(out).toContain("All courses"); + expect(out).toContain('href="/c/snake/m3"'); + expect(out).not.toMatch(/price|rating|bestseller/i); +}); + +test("library: search and filters switch to a results list", () => { + const out = renderToString(<MemoryRouter><LibraryView lib={lib} initialQuery="dock" /></MemoryRouter>); + expect(out).toContain("Courses matching “dock”"); + expect(out).not.toContain("New in the library"); +}); + +test("library filters by format, length and in-progress", () => { + const cs = lib.courses; + expect(filterCourses(cs, { format: "video", length: "any", inProgress: false }).map((c) => c.id)).toEqual(["docker"]); + expect(filterCourses(cs, { format: "all", length: "long", inProgress: false }).map((c) => c.id)).toEqual(["snake"]); + expect(filterCourses(cs, { format: "all", length: "medium", inProgress: false }).map((c) => c.id)).toEqual(["docker"]); + expect(filterCourses(cs, { format: "all", length: "any", inProgress: true }).map((c) => c.id)).toEqual(["docker"]); +}); + +const course: CoursePayload = { + id: "sd", title: "Systemic Design", pitch: null, authors: [], accent: "#86c7a6", coverUrl: null, + tracks: [{ id: "rust", label: "Rust" }, { id: "ts", label: "TypeScript" }], gating: "sequential", source: "authored", status: "ok", + modules: [ + { id: "m00", title: "Intro", index: 0, locked: false, hasCode: false, lessons: [{ id: "a", title: "A", type: "text", minutes: 5, hasDeck: false, browser: false }] }, + { id: "m01", title: "Next", index: 1, locked: true, hasCode: false, lessons: [{ id: "b", title: "Exercise 1.1 — `move`", type: "exercise", minutes: 15, hasDeck: false, browser: false }] }, + ], + progress: { a: { state: "done", positionSeconds: null, updatedAt: "" } }, canEdit: false, signedIn: true, diagnostics: [], +}; + +test("course frame: outline with progress, locks, current lesson and course accent", () => { + const out = renderToString( + <MemoryRouter initialEntries={["/c/sd/a"]}> + <CourseFrame course={course} currentLessonId="a" look="reading" lookMode="toggle"><p>body</p></CourseFrame> + </MemoryRouter>, + ); + expect(out).toContain("--accent:#86c7a6"); + expect(out).toContain('aria-current="page"'); + expect(out).toContain("lr-les done"); + expect(out).toContain("(locked)"); + expect(out).toContain('aria-label="Course progress"'); + expect(out).toContain("Your notes"); +}); + +test("module numbers count from 0 when the first module id ends in 0", () => { + expect(moduleNumber(course, 1)).toBe(1); + expect(moduleNumber({ modules: [{ id: "m1" }] }, 0)).toBe(1); +}); + +test("next lesson skips done lessons and stops at a locked module", () => { + expect(nextLesson(course)).toEqual(course.modules[0]!.lessons[0]!); + const open = { ...course, modules: course.modules.map((m) => ({ ...m, locked: false })) }; + expect(nextLesson(open)!.id).toBe("b"); +}); + +test("sign-in only returns to same-site paths", () => { + expect(safeNext("/c/x")).toBe("/c/x"); + expect(safeNext("//evil.example")).toBe("/"); + expect(safeNext("https://evil.example")).toBe("/"); + expect(safeNext(null)).toBe("/"); +}); diff --git a/packages/feature-learner/web/src/index.ts b/packages/feature-learner/web/src/index.ts new file mode 100644 index 0000000..4c833d1 --- /dev/null +++ b/packages/feature-learner/web/src/index.ts @@ -0,0 +1,4 @@ +// Browser entry "@stc/feature-learner/web". The repo maps "@stc/*" to "packages/*/src", so this subpath +// resolves here (tsc and Bun alike). It re-exports the web half only: the package index also carries the +// server (Better Auth, SSO), which cannot be bundled for the browser. +export * from "../../src/web"; From 061ae7a2d9d783d0f238a52c304df7eba03e533c Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 04:54:53 +0100 Subject: [PATCH 54/62] feat(learner): search results show lesson matches cleanly; contract notes for the browser entry and LessonView; plan marked done Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- docs/contract-requests/learner.md | 10 +++- docs/superpowers/plans/2026-10-30-learner.md | 49 ++++++++++++------- .../src/web/library/LibraryPage.tsx | 11 +++-- 3 files changed, 46 insertions(+), 24 deletions(-) diff --git a/docs/contract-requests/learner.md b/docs/contract-requests/learner.md index cf5c9cc..d6ce696 100644 --- a/docs/contract-requests/learner.md +++ b/docs/contract-requests/learner.md @@ -19,4 +19,12 @@ and let `startServer` take `Array<Feature | FeatureFactory>`. ## 3. Package entry for the browser -`@stc/feature-learner`'s index exports both `learnerFeature` (server) and `learnerWeb`/`LessonView` (browser). Bun's browser bundler tree-shakes the server half (checked), so one entry works. The web shell imports `learnerWeb` from `@stc/feature-learner`; the studio can import `LessonView` the same way. +**Problem.** The package index exports both the server (`learnerFeature`, which pulls in Better Auth and `@better-auth/sso`) and the browser half (`learnerWeb`, `LessonView`). A browser bundle of the index fails: `@better-auth/sso` imports `X509Certificate` from `node:crypto`, which has no browser polyfill. The root `tsconfig.json` maps `@stc/*` to `packages/*/src`, so package.json `exports` subpaths are not used either. + +**Workaround in `feat/learner` (applied).** `LessonView` and `learnerWeb` are exported from the index (for server-side and type use) and from a browser-only entry, `@stc/feature-learner/web`. That subpath resolves through the existing mapping to `packages/feature-learner/web/src/index.ts`, a one-line re-export of `src/web`. `apps/web/src/features.ts` imports `learnerWeb` from it, and the studio should import `LessonView` from it too. + +**Proposal.** Add `"@stc/*/web": ["packages/*/src/web"]` ahead of the general mapping in `tsconfig.json` (Bun honours tsconfig paths), and let every feature package expose `src/web/index.ts`. The shim directory can then be deleted. This also settles the importer's relative import in `features.ts`. + +## 4. LessonView for the studio preview + +`import { LessonView } from "@stc/feature-learner/web"`. Props: `lesson: LessonPayload` (the JSON from `GET /api/courses/:id/lessons/:lessonId`), plus optional `look`, `track`, `onTrackChange`, `preview`. With `preview` set, it makes no notes, attempt or progress requests; it still loads code listings and decks through the normal media routes. It injects its own stylesheet (scoped under `.lr`) and works with or without a router. diff --git a/docs/superpowers/plans/2026-10-30-learner.md b/docs/superpowers/plans/2026-10-30-learner.md index f5c82be..e6c1e8b 100644 --- a/docs/superpowers/plans/2026-10-30-learner.md +++ b/docs/superpowers/plans/2026-10-30-learner.md @@ -43,16 +43,16 @@ Inherited from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun only, plain - Tables: `auth_user`, `auth_session`, `auth_account`, `auth_verification`, `auth_sso_provider` (Better Auth, snake_case columns, ISO text dates, integer booleans), `learner_meta (key PK, value)`, `learner_group_claims (user_id, group_id)`. - test helper `forEachDb(name, fn)` = `@stc/db`'s helper plus the learner migrations. -- [ ] Step 1: failing test: after `runMigrations` the tables exist on both dialects and a second run applies nothing. -- [ ] Step 2: implement; Step 3: green on SQLite and Postgres; commit. +- [x] Step 1: failing test: after `runMigrations` the tables exist on both dialects and a second run applies nothing. +- [x] Step 2: implement; Step 3: green on SQLite and Postgres; commit. ### Task 2: Better Auth adapter over Bun.SQL **Files:** `src/server/auth/adapter.ts`, `test/adapter.test.ts` **Interfaces:** `bunSqlAdapter(db: Db)` → Better Auth `DBAdapterInstance`. Model `x` → table `auth_<snake(x)>`, field `camelCase` → `snake_case`. `supportsDates/Booleans/JSON: false`, no transactions (SQLite has one connection; a nested query in a transaction would deadlock). -- [ ] Failing tests (both dialects): create/findOne/findMany (eq, ne, in, not_in, contains, starts_with, ends_with, lt/gt, insensitive mode, OR grouping, sort, limit, offset), update, updateMany, delete, deleteMany, count, consumeOne, incrementOne. -- [ ] Implement, green, commit. +- [x] Failing tests (both dialects): create/findOne/findMany (eq, ne, in, not_in, contains, starts_with, ends_with, lt/gt, insensitive mode, OR grouping, sort, limit, offset), update, updateMany, delete, deleteMany, count, consumeOne, incrementOne. +- [x] Implement, green, commit. ### Task 3: AuthService, owner bootstrap, claim groups, feature wiring @@ -64,15 +64,15 @@ Inherited from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun only, plain - First user to sign up becomes `owner` (claimed via `INSERT` into `learner_meta` key `owner`, PK conflict loses); others get `learner`. - `syncClaimGroups(db, userId, names)` creates groups by name and replaces the user's claim-sourced memberships; manual memberships are untouched. - `learnerFeature(ctx)` returns `Feature & { auth }`; `startServer` accepts feature factories `(ctx) => Feature` and installs a feature's `auth` (contract request filed; two-line change in `main.ts`). -- [ ] Failing tests: sign-up → session cookie → `userFromRequest` returns roles `[owner]`; second user `[learner]`; concurrent first sign-ups give one owner; sign-out clears; claim sync keeps manual memberships. -- [ ] Implement, green, commit. +- [x] Failing tests: sign-up → session cookie → `userFromRequest` returns roles `[owner]`; second user `[learner]`; concurrent first sign-ups give one owner; sign-out clears; claim sync keeps manual memberships. +- [x] Implement, green, commit. ### Task 4: Admin API (roles, groups, course ACL, library curation) **Files:** `src/server/admin.ts`, `test/admin.test.ts` - Routes (owner only, 403 otherwise): `GET /api/admin/users`, `PUT /api/admin/users/:id/roles`, `GET|POST /api/admin/groups`, `DELETE /api/admin/groups/:id`, `PUT /api/admin/groups/:id/members`, `GET|PUT /api/admin/courses/:id/acl`, `GET|PUT /api/admin/library` (featured + collections). - The last owner cannot drop their own owner role. -- [ ] Tests first; implement; commit. +- [x] Tests first; implement; commit. ### Task 5: Library, outline and lesson API @@ -82,13 +82,13 @@ Inherited from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun only, plain - `GET /api/courses/:id` → outline, tracks, gating, `locked` per module, progress, diagnostics (errors only for owner/authors). - `GET /api/courses/:id/lessons/:lessonId` → `LessonPayload` (markdown, hints, decks, video, transcript, code, attachments, browser exercise, local command, prev/next) or `{ locked: true }`. - Gating: module *i* locked when any lesson of an earlier module is not done; owners and course authors bypass. -- [ ] Tests against the sample fixture plus temp courses; implement; commit. +- [x] Tests against the sample fixture plus temp courses; implement; commit. ### Task 6: Progress, notes, exercise attempts API **Files:** `src/server/userdata.ts`, `test/userdata.test.ts` - `PUT /api/courses/:id/progress/:lessonId` `{state, positionSeconds?}`; notes CRUD (`GET/POST /api/courses/:id/notes`, `PATCH/DELETE /api/notes/:id`); attempts `GET/PUT /api/courses/:id/lessons/:lessonId/attempt?track=`. A passing attempt marks the lesson done. Anonymous → 401. Hidden course → 404. -- [ ] Tests on both dialects (acceptance: notes persist on SQLite and Postgres); implement; commit. +- [x] Tests on both dialects (acceptance: notes persist on SQLite and Postgres); implement; commit. ### Task 7: Code files and zip download @@ -96,41 +96,52 @@ Inherited from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun only, plain - `GET /api/courses/:id/code/:moduleId` → `{ dir, command, files: {path,size}[] }` (skips `.git`, `node_modules`, symlinks escaping root; caps 2000 files). - `GET /api/courses/:id/code/:moduleId/zip` → `application/zip`, `Content-Disposition` via `contentDisposition("<dir name>.zip")`. - `zip(entries)`: store/deflate, CRC32, UTF-8 name flag. -- [ ] Tests: zip round-trip via `unzip -l`, non-ASCII names, symlink escape skipped; commit. +- [x] Tests: zip round-trip via `unzip -l`, non-ASCII names, symlink escape skipped; commit. ### Task 8: Browser exercise runner **Files:** `src/web/runner/{assert.ts,worker-source.ts,runner.ts}`, `src/server/transpile.ts`, `test/runner.test.ts` - `createBrowserRunner(language, { transpile })`: `ExerciseRunner`. Worker disables `fetch`, `XMLHttpRequest`, `WebSocket`, `importScripts` before importing code. Results carry expected/actual for failed asserts. -- [ ] Tests (Bun supports module workers and blob URLs): starter fails, solution passes, a syntax error is reported, `while(true)` times out, TS is transpiled. -- [ ] commit. +- [x] Tests (Bun supports module workers and blob URLs): starter fails, solution passes, a syntax error is reported, `while(true)` times out, TS is transpiled. +- [x] commit. ### Task 9: Markdown pipeline **Files:** `src/web/markdown.ts`, `test/markdown.test.ts` - `renderLessonHtml(md, { courseId, baseFile, tracks, lessonFiles })` → sanitized HTML: heading ids, code blocks tagged with their track (`data-t`), bold track labels tagged, `<details>` turned into folds; consecutive hint folds become a tier group (`data-tier`, `data-tiers`), relative images/links rewritten to `/api/media/...` or lesson routes. -- [ ] Tests with Pure Snake-style input; commit. +- [x] Tests with Pure Snake-style input; commit. ### Task 10: Web shell and library **Files:** `src/web/{routes.tsx,api.ts,styles.css,shell/*,library/*}` - Header extras (course switcher, search, Reading/Work, account) portalled into the Phase 0 header; course outline sidebar (mobile overlay, Escape closes); library: featured hero, continue row, New row, collection rows of covers, format/length/in-progress filters, search. -- [ ] Component smoke tests with `react-dom/server` render; screenshot check later; commit. +- [x] Component smoke tests with `react-dom/server` render; screenshot check later; commit. ### Task 11: Lesson views and `LessonView` **Files:** `src/web/lesson/*` - Text (track switcher, Reading/Work, folds, hint tiers), video (range-served `<video>`, resume, done near end, timestamp notes), deck viewer (pdfjs, thumbs, arrows, fullscreen, per-slide note), code files (tree, tabs, copy, zip), exercise page (editor, run, reset, per-test results, hints, local command; local-only: command + "Mark done" + optional pasted evidence), notes list, floating action bar. -- [ ] Render tests for `LessonView` with each lesson type; commit. +- [x] Render tests for `LessonView` with each lesson type; commit. ### Task 12: Account and admin pages **Files:** `src/web/account/*` - Sign in / create account (email+password), configured provider buttons, SSO by email, sign out; owner admin page for users/roles, groups, featured course and collections. -- [ ] Render tests; commit. +- [x] Render tests; commit. ### Task 13: Wiring, verification, report -- [ ] Append `learnerWeb` to `apps/web/src/features.ts`; `learnerFeature` in `main.ts`. -- [ ] `bun test`, `bun run typecheck`, Postgres run; dev server on 3461 with `STC_COURSE_DIRS=./fixtures/courses:./overlays`; headless Chrome screenshots of library, text, video, deck, code, exercise, account for all three courses; fix what looks wrong. -- [ ] Write `/home/user/STCourses/.superpowers/sdd/2026-10-30-stcourses/wp-learner-report.md`. +- [x] Append `learnerWeb` to `apps/web/src/features.ts`; `learnerFeature` in `main.ts`. +- [x] `bun test`, `bun run typecheck`, Postgres run; dev server on 3461 with `STC_COURSE_DIRS=./fixtures/courses:./overlays`; headless Chrome screenshots of library, text, video, deck, code, exercise, account for all three courses; fix what looks wrong. +- [x] Write `/home/user/STCourses/.superpowers/sdd/2026-10-30-stcourses/wp-learner-report.md`. + +--- + +## Execution notes (added when the plan was finished) + +- Tasks 1–9 were done by the first implementer (0b11715..99c0e49). Tasks 10–13 were done after a hand-over (89c51f0 onwards). +- Tasks 10–12 landed in one commit, because the route modules import each other (the routes list loads the lesson, library and account pages). +- Styles are a TS string (`src/web/styles.ts`) injected once, not a `.css` text import: Bun's dev server loses text imports on hot rebuilds. +- The browser entry is `@stc/feature-learner/web` (see `docs/contract-requests/learner.md` §3); the index cannot be bundled for the browser. +- Decks render client-side with pdf.js; the importer's `<dataDir>/cache/decks` is not used. +- The exercise editor is a textarea with a line gutter, as in the mockup (CodeMirror was optional). diff --git a/packages/feature-learner/src/web/library/LibraryPage.tsx b/packages/feature-learner/src/web/library/LibraryPage.tsx index 3b9131a..3667f11 100644 --- a/packages/feature-learner/src/web/library/LibraryPage.tsx +++ b/packages/feature-learner/src/web/library/LibraryPage.tsx @@ -5,6 +5,7 @@ import type { CourseCard, ContinueItem, Format, LibraryPayload, SearchHit } from import { api, courseHref, ensureStyles, fmtMinutes, lessonHref, refreshLibrary, useLibrary, useSearch } from "../api"; import { HeaderExtras } from "../shell/HeaderExtras"; import { accentStyle } from "../shell/CourseFrame"; +import { TitleText } from "../lesson/TitleText"; const FORMAT_LABEL: Record<Format, string> = { text: "Text", video: "Video", slides: "Slides", exercises: "Exercises" }; type LengthFilter = "any" | "short" | "medium" | "long"; @@ -104,16 +105,18 @@ export function LibraryView({ lib, initialQuery }: { lib: LibraryPayload; initia </> ) : ( <> - <h2 className="sec-h">{q ? `Courses matching “${q}”` : "Courses"}</h2> - {q && hits === null ? <p className="lr-nohit">Searching…</p> : results.length ? <CourseTable courses={results} signedIn={lib.signedIn} /> : <p className="lr-nohit">No courses match.</p>} + {results.length || !lessonHits.length ? <h2 className="sec-h">{q ? `Courses matching “${q}”` : "Courses"}</h2> : null} + {q && hits === null ? <p className="lr-nohit">Searching…</p> + : results.length ? <CourseTable courses={results} signedIn={lib.signedIn} /> + : !lessonHits.length ? <p className="lr-nohit">Nothing matches. Try fewer words, or clear the filters.</p> : null} {lessonHits.length ? ( <> - <h2 className="sec-h" style={{ marginTop: 36 }}>Lessons</h2> + <h2 className="sec-h" style={{ marginTop: results.length ? 36 : 0 }}>{q ? `Lessons matching “${q}”` : "Lessons"}</h2> <ul className="hits"> {lessonHits.flatMap((h) => h.lessons.map((l) => ( <li key={`${h.courseId}/${l.id}`}> <Link to={lessonHref(h.courseId, l.id)}> - <span><span className="h1">{l.title}</span><span className="h2" style={{ display: "block" }}>{h.courseTitle} · {l.moduleTitle}</span></span> + <span><span className="h1"><TitleText text={l.title} /></span><span className="h2" style={{ display: "block" }}>{h.courseTitle} · {l.moduleTitle}</span></span> </Link> </li> )))} From c7b9e1f98eb24b4043f59bf7d20d8b8e2ac60c55 Mon Sep 17 00:00:00 2001 From: Lucas Winther <lucwin@ece.au.dk> Date: Fri, 30 Oct 2026 05:20:43 +0100 Subject: [PATCH 55/62] fix(learner): review round 1: sanitize-html, sandboxed runner frame, one visibility rule, no Tab capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit C1: lesson HTML goes through a sanitize-html allowlist on the fully rendered output (no on* attributes, no style, no svg/math/iframe/object/embed/form; URLs limited to http(s), mailto, relative and anchors). Raw tags outside the allowlist are escaped as text. The SPA document is served with a CSP (script-src 'self', no inline script) through spaWithCsp in main.ts; contract request §5 asks core to own it. C2: exercises run in <iframe sandbox="allow-scripts"> (opaque origin, no cookies) loaded from /api/learner/runner-frame, whose CSP allows only its own script (by hash) and blob: workers, and connect-src 'none'. postMessage only, both sides check event.source; the timeout removes the frame. The worker stubs now also cover Worker, SharedWorker and caches. I1: learner visibility calls core canView with per-course authorship only, like /api/media (contract request §6). I2: Tab and Shift+Tab move focus; Enter keeps the previous line's indentation. Tests: XSS unit cases, headless-Chrome checks for the exploit lesson and for fetch, import, Cache.add, nested Worker, img, EventSource, WebSocket, XHR and sendBeacon from learner and test code (plus a negative control), visibility on both dialects, SPA and runner-frame CSP headers. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- apps/server/src/main.ts | 7 +- bun.lock | 38 ++++ docs/contract-requests/learner.md | 14 ++ packages/feature-learner/package.json | 6 +- packages/feature-learner/src/index.ts | 1 + packages/feature-learner/src/server/access.ts | 8 +- .../feature-learner/src/server/feature.ts | 2 + .../feature-learner/src/server/runnerFrame.ts | 67 +++++++ packages/feature-learner/src/server/spa.ts | 80 ++++++++ .../src/web/lesson/Exercise.tsx | 21 +- packages/feature-learner/src/web/markdown.ts | 76 +++++-- .../feature-learner/src/web/runner/runner.ts | 67 ++++++- .../src/web/runner/worker-source.ts | 5 +- packages/feature-learner/src/web/styles.ts | 4 +- .../test/browser-isolation.test.ts | 185 ++++++++++++++++++ packages/feature-learner/test/chrome.ts | 68 +++++++ packages/feature-learner/test/e2e/harness.tsx | 39 ++++ .../feature-learner/test/lesson-view.test.tsx | 9 + .../feature-learner/test/markdown.test.ts | 9 +- packages/feature-learner/test/spa.test.ts | 65 ++++++ .../feature-learner/test/visibility.test.ts | 27 +++ packages/feature-learner/test/xss.test.ts | 84 ++++++++ 22 files changed, 840 insertions(+), 42 deletions(-) create mode 100644 packages/feature-learner/src/server/runnerFrame.ts create mode 100644 packages/feature-learner/src/server/spa.ts create mode 100644 packages/feature-learner/test/browser-isolation.test.ts create mode 100644 packages/feature-learner/test/chrome.ts create mode 100644 packages/feature-learner/test/e2e/harness.tsx create mode 100644 packages/feature-learner/test/spa.test.ts create mode 100644 packages/feature-learner/test/visibility.test.ts create mode 100644 packages/feature-learner/test/xss.test.ts diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index ca26cd7..9187eae 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -2,7 +2,7 @@ import type { AppConfig, AuthService, Feature, JobHandler, JobQueue, ServerConte import { createApp, createJobQueue, loadConfig } from "@stc/core"; import { createCourseRepository } from "@stc/course-format"; import { connect, migrate } from "@stc/db"; -import { learnerFeature } from "@stc/feature-learner"; +import { learnerFeature, spaWithCsp } from "@stc/feature-learner"; import indexHtml from "../../web/index.html"; // Used until a feature supplies an `auth` service (the learner package does). @@ -31,16 +31,19 @@ export async function startServer( ctx.jobs = jobs; const app = createApp(ctx, built); jobs.start(); + // The SPA document carries a CSP header (served through spaWithCsp; contract request learner §5). + const spa = spaWithCsp(indexHtml, { development: process.env.NODE_ENV !== "production" }); const server = Bun.serve({ port: config.port, // Bun routes win over fetch; "/api/*" falls through to the composed app (404 for unknown API paths). - routes: { ...app.routes, "/api/*": (req) => app.fetch(req), "/*": indexHtml }, + routes: { ...app.routes, "/api/*": (req) => app.fetch(req), ...spa.routes }, fetch: (req) => app.fetch(req), }); return { server, async stop() { await server.stop(true); + spa.stop(); await jobs.stop(); await db.close(); }, diff --git a/bun.lock b/bun.lock index a0be2cc..9b52941 100644 --- a/bun.lock +++ b/bun.lock @@ -62,6 +62,10 @@ "better-auth": "^1.7.14", "marked": "^18.1.2", "pdfjs-dist": "^6.4.299", + "sanitize-html": "^2", + }, + "devDependencies": { + "@types/sanitize-html": "^2.16.2", }, }, "packages/feature-studio": { @@ -164,6 +168,8 @@ "@types/react-dom": ["@types/react-dom@19.3.0", "", { "peerDependencies": { "@types/react": "^19.3.0" } }, "sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q=="], + "@types/sanitize-html": ["@types/sanitize-html@2.16.2", "", { "dependencies": { "htmlparser2": "^12.0" } }, "sha512-kuAbgDCFv418flrp3z4o/4ufLA/5ovmXRH0L419iZCPGO7w00znjeDI5Wy/KZWNIBpDQc7hucCK+J5almB6FyA=="], + "@xmldom/is-dom-node": ["@xmldom/is-dom-node@1.0.1", "", {}, "sha512-CJDxIgE5I0FH+ttq/Fxy6nRpxP70+e2O048EPe85J2use3XKdatVM7dDVvFNjQudd9B49NPoZ+8PG49zj4Er8Q=="], "@xmldom/xmldom": ["@xmldom/xmldom@0.9.12", "", {}, "sha512-5AXjrcMClTryPe9LgZrygpB1lj7s0S9E0+W+AHaVKAVyHanafK86iPSvG5xHVSp/jC+VH1UXu0TAEmY279xH7A=="], @@ -178,28 +184,56 @@ "csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="], + "dayjs": ["dayjs@1.11.23", "", {}, "sha512-QDTCU0M0MxR3hQfnlDJfwekQiaanm1ubOD231u73WBckQ/fsamwRLiE2GBz6D3a/xF1NgfiDLJjXBa1hYOYTtQ=="], + "defu": ["defu@6.1.7", "", {}, "sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ=="], + "dom-serializer": ["dom-serializer@3.1.1", "", { "dependencies": { "domelementtype": "^3.0.0", "domhandler": "^6.0.0", "entities": "^8.0.0" } }, "sha512-4MEa38/QexBob6gFNwu+EGdWvhJ1OKuNwdYY3Y3NyeWDQfnGeDYQUDfIRzWu5B5gsv03so2Uxd28YC6zrsx3Lw=="], + + "domelementtype": ["domelementtype@3.0.0", "", {}, "sha512-umCQid3jKbDmVjx8jGaW7uUykm4DEUeyV21hPxNMo2nV955DhUThwqyOIDtreepP31hl84X7G5U9ZfsWvIB3Pg=="], + + "domhandler": ["domhandler@6.0.1", "", { "dependencies": { "domelementtype": "^3.0.0" } }, "sha512-gYzvtM72ZtxQO0T048kd6HWSbbGCNOUwcnfQ01cqIJ4X2IYKFFHZ5mKvrQETcFXxsRObZulDaKmy//R7TPtsBg=="], + + "domutils": ["domutils@4.0.2", "", { "dependencies": { "dom-serializer": "^3.0.0", "domelementtype": "^3.0.0", "domhandler": "^6.0.0" } }, "sha512-qI4JLRKnSzqFqr7hAlS5xQDusBCjKSEG4t4+7aNrIQMHBcsC2TGEhuyABJdYkgSewL57PNLYEiibY2iPKhKpaA=="], + + "entities": ["entities@8.1.0", "", {}, "sha512-kxL7msIffSuh9aaFAMD7rxAIuTRMAHMeBtgHW2yUdWw732ZNh4MehkF2gdjvtdmikkaIP9bFDDJOPlsvm7avrA=="], + "escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="], + "escape-string-regexp": ["escape-string-regexp@4.0.0", "", {}, "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="], + "fast-xml-builder": ["fast-xml-builder@1.3.1", "", { "dependencies": { "path-expression-matcher": "^1.6.2", "xml-naming": "^0.3.0" } }, "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug=="], "fast-xml-parser": ["fast-xml-parser@5.11.2", "", { "dependencies": { "@nodable/entities": "^3.0.1", "fast-xml-builder": "^1.2.0", "is-unsafe": "^2.0.0", "path-expression-matcher": "^1.6.2", "strnum": "^2.4.2", "xml-naming": "^0.3.0" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-R9iDuNrQYeQut46cn2r2wHKn4HYzVDvDm5J1wW+koZewykv0yuO2HChTYeZtrULyHIDM9cj9TUXloCsueCQUog=="], + "htmlparser2": ["htmlparser2@12.0.0", "", { "dependencies": { "domelementtype": "^3.0.0", "domhandler": "^6.0.0", "domutils": "^4.0.2", "entities": "^8.0.0" } }, "sha512-Tz7u1i95/g2x2jz81+x0FBVhBhY5aRTvD3tXXdFaljuNdzDLJ8UGNRrTcj2cgQvAg3iW/h77Fz15nLW0L0CrZw=="], + + "is-plain-object": ["is-plain-object@5.1.0", "", {}, "sha512-bUi/yjmtKYcRVUtWRGr0UA6xEFh2I6zWUwMrUXB3s7bmYCaZ8a+0ZsTRkrawh/mzlSD1Y0Ph8bp/U+TvBpWDNw=="], + "is-unsafe": ["is-unsafe@2.0.2", "", {}, "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ=="], "jose": ["jose@6.2.14", "", {}, "sha512-3JasLuki1NRLQvY4wCoC37KZk4qxoBx2k2aFqLxT84IezqW0bkJKnNtxnZj/Fu1xsQivoc/lytxaRHJoGLjJ1A=="], "kysely": ["kysely@0.29.7", "", {}, "sha512-fmIq+7kv4WbFm2tcfznRzikqB4cuNpgZ40PSU2fAJ2/OO5XhojqnTPrmgrCjYSRJVwBln03XcRXJGnw9H099IQ=="], + "launder": ["launder@1.7.2", "", { "dependencies": { "dayjs": "^1.11.23" } }, "sha512-DLg3HPnHUfBi5/MxMLmJD12dmlMpFEC2HgMW6vkZ/9JR0RU00kXoRGvDxAvFmdO0o610OA77i4FgNTLucmhDVg=="], + "marked": ["marked@18.1.2", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-DfOnvLAQ6WYeWxWO1u4RyFOeqyHup6dks5PPqnoFiooU7ipRvidO/2j8plhAHIiZg574E9ZlzSjUiPMLAzFkgg=="], + "nanoid": ["nanoid@3.3.20", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ=="], + "nanostores": ["nanostores@1.5.5", "", {}, "sha512-FixtE239Gl6Gz0ZGCK0WzvNwObNCIwVT+L7zQcf1QgcA6AzBxblzqe9OT88qiucNkhsjSOqiogSaF5jVbJiqSA=="], + "parse-srcset": ["parse-srcset@1.0.2", "", {}, "sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q=="], + "path-expression-matcher": ["path-expression-matcher@1.6.2", "", {}, "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ=="], "pdfjs-dist": ["pdfjs-dist@6.4.299", "", { "optionalDependencies": { "@napi-rs/canvas": "^1.0.10" } }, "sha512-AVl138zALtfaAPvADulE0PZThbYzCBS79nL4pOSL/6Sm/4AH5A21BD9VHt97OlCuzJuCpmeZtAtkinisF4Vb1g=="], + "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], + + "postcss": ["postcss@8.5.32", "", { "dependencies": { "nanoid": "^3.3.20", "picocolors": "^1.1.1", "source-map-js": "^1.2.2" } }, "sha512-LJlKCCAUFKOuZxipUdYw1NzIITEsPcgyOG9LkvRJvuKj/bRzbuRY0ARXvL0pWGpsZvn8jku+LMep2eU0seHp8g=="], + "react": ["react@19.3.1", "", {}, "sha512-QNhWdt7nrgVWWQG58QuGsAR3jv3Hbe4mb4vz7w2csnHavciZJgda1CPNaldF6eVxv90ud3jUXpKlHL69gO38yw=="], "react-dom": ["react-dom@19.3.1", "", { "dependencies": { "scheduler": "^0.28.0" }, "peerDependencies": { "react": "^19.3.1" } }, "sha512-Y0bARz53lhEC1NMCASs0/j+faB/zqt3R8hpvg1e8+61ZVJ4UL0UdC4cBbXfZMcIuhrfgeILTLe+ZeBCavQPtKQ=="], @@ -212,10 +246,14 @@ "samlify": ["samlify@2.15.1", "", { "dependencies": { "@authenio/xml-encryption": "^6.0.2", "@xmldom/xmldom": "^0.9.12", "xml": "^1.0.1", "xml-crypto": "^6.4.0", "xml-escape": "^1.1.0", "xpath": "^0.0.35" } }, "sha512-peZitMO2cAnf0pY45H0yw0Drqmby7U91yRovH3AH8TdwYCNRnmidVORXIjcK+21cBk3Mjl8EVz8Nvuw/v20rAw=="], + "sanitize-html": ["sanitize-html@2.18.1", "", { "dependencies": { "escape-string-regexp": "^4.0.0", "htmlparser2": "^12.0.0", "is-plain-object": "^5.1.0", "launder": "^1.7.2", "parse-srcset": "^1.0.2", "postcss": "^8.5.27" } }, "sha512-2X5lC8fYla8YaT8g73Nj+HcLWXan5dLyCWLgVvaOmfw7JI78IRsyNtIDytTHdn7hBQbIkCah7Zbmd5jabzeDig=="], + "scheduler": ["scheduler@0.28.0", "", {}, "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw=="], "set-cookie-parser": ["set-cookie-parser@3.2.1", "", {}, "sha512-Npd1Pt+4zb3BrSZDPd2qFwRpS6b1EUEasRrO8xS0zMJi0fApXXK5YuzP0minyJnxwnFHoVoJwA55q0A0cu63gQ=="], + "source-map-js": ["source-map-js@1.2.2", "", {}, "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw=="], + "strnum": ["strnum@2.5.1", "", { "dependencies": { "anynum": "^1.0.1" } }, "sha512-UM4nTLWzQiDBseVxUGxN0wJzIk0UU89Qi7pKXz9qISo+lhviJikTuikBsK/oFRlpvYHvSdz0TVoLyQ+sEpYwxA=="], "tldts": ["tldts@7.4.19", "", { "dependencies": { "tldts-core": "^7.4.19" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-48dH6aSpQnpbzDUc4vIQZ1VjB8tkK+e8c1yx05/SZK+rkssgjEOlxDd/yU8/a4vGYQ9vRgJr9dwMq9TJ0b3X2Q=="], diff --git a/docs/contract-requests/learner.md b/docs/contract-requests/learner.md index d6ce696..b751e07 100644 --- a/docs/contract-requests/learner.md +++ b/docs/contract-requests/learner.md @@ -28,3 +28,17 @@ and let `startServer` take `Array<Feature | FeatureFactory>`. ## 4. LessonView for the studio preview `import { LessonView } from "@stc/feature-learner/web"`. Props: `lesson: LessonPayload` (the JSON from `GET /api/courses/:id/lessons/:lessonId`), plus optional `look`, `track`, `onTrackChange`, `preview`. With `preview` set, it makes no notes, attempt or progress requests; it still loads code listings and decks through the normal media routes. It injects its own stylesheet (scoped under `.lr`) and works with or without a router. + +## 5. Content-Security-Policy on the SPA, set by core (review C1) + +**Problem.** Lesson HTML is author content. The sanitizer (sanitize-html allowlist) is the first defence, and a CSP without inline script is the second. Bun.serve cannot attach headers to an HTML import route. + +**Workaround in `feat/learner` (applied).** `spaWithCsp(indexHtml, { development })` in `@stc/feature-learner` serves the bundle from an internal server bound to 127.0.0.1 and proxies it. HTML responses get `Content-Security-Policy` (`script-src 'self'`, no `'unsafe-inline'`; `connect-src 'self'`; `object-src 'none'`; `frame-ancestors 'self'`; see `spaCsp()`), plus `X-Content-Type-Options: nosniff` and `Referrer-Policy: same-origin`. `main.ts` uses it for `"/*"`. HMR is off on the internal server, so Bun injects no inline script. If it ever does in development, only the hashes of those exact scripts are allowed. + +**Proposal.** Core owns the SPA route and sets this CSP globally (and for every feature's HTML), replacing the proxy. Other features must then avoid inline scripts as well. The studio preview through `LessonView` already complies. + +## 6. `canView` and the global `author` role (review I1) + +**Ruling applied.** Learner routes now call core `canView` with per-course authorship only (`course_authors`), exactly like core's `/api/media`. A user with only the global `author` role gets a 404 for a restricted course they are not listed on, on every route. + +**Proposal for Phase 2.** Extend core `canView` so the global `author` role can view every course (the studio and AI need the same rule), for example `canView(user, course, acl, isCourseAuthor)` returning true when `user.roles.includes("author")`. The learner then needs no change: it already delegates to `canView`. diff --git a/packages/feature-learner/package.json b/packages/feature-learner/package.json index 807eab9..bf47b54 100644 --- a/packages/feature-learner/package.json +++ b/packages/feature-learner/package.json @@ -8,10 +8,14 @@ "@better-auth/sso": "^1.7.14", "better-auth": "^1.7.14", "marked": "^18.1.2", - "pdfjs-dist": "^6.4.299" + "pdfjs-dist": "^6.4.299", + "sanitize-html": "^2" }, "exports": { ".": "./src/index.ts", "./web": "./web/src/index.ts" + }, + "devDependencies": { + "@types/sanitize-html": "^2.16.2" } } diff --git a/packages/feature-learner/src/index.ts b/packages/feature-learner/src/index.ts index c01208d..dc67d6b 100644 --- a/packages/feature-learner/src/index.ts +++ b/packages/feature-learner/src/index.ts @@ -1,6 +1,7 @@ // Server export { migrationsDirs, runMigrations } from "./server/migrate"; export { learnerFeature } from "./server/feature"; +export { spaWithCsp, spaCsp } from "./server/spa"; // Browser (also available without the server half as "@stc/feature-learner/web") export { learnerWeb, LessonView, type LessonViewProps, ensureLearnerStyles } from "./web"; // Shared JSON shapes diff --git a/packages/feature-learner/src/server/access.ts b/packages/feature-learner/src/server/access.ts index 9ad4ea2..64fbbec 100644 --- a/packages/feature-learner/src/server/access.ts +++ b/packages/feature-learner/src/server/access.ts @@ -28,10 +28,14 @@ export function canEdit(user: AuthUser | null, courseId: string, idx: AccessInde return !!user && (user.roles.includes("owner") || user.roles.includes("author") || idx.authorOf.has(courseId)); } -/** Learners see courses with a manifest that `canView` allows; owners also see broken ones. */ +/** + * Learners see courses with a manifest that core `canView` allows; owners also see broken ones. The author input + * is per-course authorship only, the same as core's media route, so every route applies one rule. (The global + * `author` role grants viewing only once Phase 2 extends `canView`; contract request learner §6.) + */ export function visible(user: AuthUser | null, c: LoadedCourse, idx: AccessIndex): boolean { if (!c.manifest) return isOwner(user); - return canView(user, c, idx.acl.get(c.id) ?? [], canEdit(user, c.id, idx)); + return canView(user, c, idx.acl.get(c.id) ?? [], idx.authorOf.has(c.id)); } /** The course if this user may read it, else null (callers answer 404, never 403, so hidden courses stay hidden). */ diff --git a/packages/feature-learner/src/server/feature.ts b/packages/feature-learner/src/server/feature.ts index 0d2c512..ebfe694 100644 --- a/packages/feature-learner/src/server/feature.ts +++ b/packages/feature-learner/src/server/feature.ts @@ -9,6 +9,7 @@ import { userDataRoutes } from "./userdata"; import { codeRoutes } from "./code"; import { transpileRoutes } from "./transpile"; import { assetRoutes } from "./assets"; +import { runnerFrameRoutes } from "./runnerFrame"; /** * The learner feature: auth, library, reader, notes, progress and exercises. @@ -34,6 +35,7 @@ export function learnerFeature(ctx: ServerContext): Feature & { auth: AuthServic ...codeRoutes(ctx), ...transpileRoutes(), ...assetRoutes(), + ...runnerFrameRoutes(), }; // Every handler waits for the learner migrations (a no-op once Phase 2 runs them in migrate()). for (const methods of Object.values(routes)) { diff --git a/packages/feature-learner/src/server/runnerFrame.ts b/packages/feature-learner/src/server/runnerFrame.ts new file mode 100644 index 0000000..a95ef1e --- /dev/null +++ b/packages/feature-learner/src/server/runnerFrame.ts @@ -0,0 +1,67 @@ +import type { Feature } from "@stc/contracts"; +import { WORKER_SOURCE } from "../web/runner/worker-source"; + +/** + * The exercise runner frame. The page loads it in `<iframe sandbox="allow-scripts">` (never allow-same-origin), + * so it has an opaque origin: no cookies, no storage, no access to the page. Its own CSP allows only this one + * inline script (by hash) and blob: scripts and workers, and no connections at all (`connect-src 'none'`), which + * also covers dynamic import of remote URLs, Cache.add, nested workers, EventSource and WebSocket. The worker + * inherits that policy. The page talks to the frame only with postMessage, and both sides check `event.source`. + */ +export const RUNNER_FRAME_PATH = "/api/learner/runner-frame"; + +export const FRAME_SCRIPT = `(() => { + "use strict"; + const SRC = ${JSON.stringify(WORKER_SOURCE)}; + let worker = null; + const send = (msg) => parent.postMessage(msg, "*"); + addEventListener("message", (ev) => { + if (ev.source !== parent) return; + const m = ev.data; + if (!m || m.type !== "run" || worker) return; + const url = URL.createObjectURL(new Blob([SRC], { type: "text/javascript" })); + // A classic worker: module workers from blob: URLs fail to start in an opaque-origin frame in Chrome. + // The worker still loads the learner and test modules with import() of blob: URLs. + worker = new Worker(url); + worker.onmessage = (e) => send({ type: "worker", data: e.data }); + worker.onerror = (e) => { e.preventDefault(); send({ type: "crash", message: e.message || "The exercise worker crashed" }); }; + worker.postMessage({ code: String(m.code), tests: String(m.tests) }); + }); + send({ type: "ready" }); +})();`; + +const hash = new Bun.CryptoHasher("sha256").update(FRAME_SCRIPT).digest("base64"); + +export const RUNNER_FRAME_CSP = [ + "default-src 'none'", + `script-src 'sha256-${hash}' blob:`, + "worker-src blob:", + "connect-src 'none'", + "img-src 'none'", + "style-src 'none'", + "font-src 'none'", + "media-src 'none'", + "object-src 'none'", + "base-uri 'none'", + "form-action 'none'", + "frame-ancestors 'self'", + "sandbox allow-scripts", +].join("; "); + +const HTML = `<!doctype html><html><head><meta charset="utf-8"><title>Exercise runner`; + +export function runnerFrameRoutes(): Feature["routes"] { + return { + [RUNNER_FRAME_PATH]: { + GET: async () => new Response(HTML, { + headers: { + "Content-Type": "text/html; charset=utf-8", + "Content-Security-Policy": RUNNER_FRAME_CSP, + "X-Content-Type-Options": "nosniff", + "Referrer-Policy": "no-referrer", + "Cache-Control": "no-store", + }, + }), + }, + }; +} diff --git a/packages/feature-learner/src/server/spa.ts b/packages/feature-learner/src/server/spa.ts new file mode 100644 index 0000000..41342d5 --- /dev/null +++ b/packages/feature-learner/src/server/spa.ts @@ -0,0 +1,80 @@ +import type { HTMLBundle } from "bun"; + +/** + * Content-Security-Policy for the SPA document. No inline script: `script-src 'self'` only (plus, in development, + * the hashes of the inline helper scripts Bun's dev server injects). Style attributes are allowed because React + * components set them; styles cannot run script. The exercise runner frame is a same-origin URL (`frame-src + * 'self'`) with its own, stricter policy, see `runnerFrame.ts`. + */ +export function spaCsp(inlineScriptHashes: string[] = []): string { + const hashes = inlineScriptHashes.map((h) => ` '${h}'`).join(""); + return [ + "default-src 'self'", + `script-src 'self'${hashes}`, + "style-src 'self' 'unsafe-inline'", + "img-src 'self' data: blob: https:", + "media-src 'self' blob:", + "font-src 'self' data:", + "connect-src 'self'", + "worker-src 'self' blob:", + "frame-src 'self'", + "object-src 'none'", + "base-uri 'self'", + "form-action 'self'", + "frame-ancestors 'self'", + ].join("; "); +} + +/** `sha256-…` CSP sources for every inline `
      `; + app = Bun.serve({ + hostname: "localhost", port: 0, + fetch(req) { + const u = new URL(req.url); + // The harness page gets the same CSP as the SPA document. + if (u.pathname === "/harness.html") return new Response(page, { headers: { "Content-Type": "text/html", "Content-Security-Policy": spaCsp(), "Set-Cookie": "stc_probe=secret-session; Path=/; SameSite=Lax" } }); + // The same page without any CSP: the runner must isolate learner code on its own. + if (u.pathname === "/harness-nocsp.html") return new Response(page, { headers: { "Content-Type": "text/html" } }); + // Same-origin probe: a request here from learner code would carry the session cookie. + if (u.pathname.startsWith("/cookie-probe")) { + hits.push({ path: `app:${u.pathname}`, cookie: req.headers.get("cookie") }); + return new Response("export default 1;", { headers: { "Content-Type": "text/javascript" } }); + } + if (u.pathname === "/harness.js") return new Response(js, { headers: { "Content-Type": "text/javascript" } }); + return learner.fetch(req); + }, + }); + ORIGIN = `http://localhost:${app.port}`; + chrome = await Chrome.launch(); + await chrome.goto(`${ORIGIN}/harness.html`, 2000); + expect(await chrome.eval("window.stcReady === true")).toBe(true); + }, 30_000); + + afterAll(async () => { + await chrome?.close(); + app?.stop(true); + probe?.stop(true); + await db?.close(); + }); + + test("control: the probe sees requests from this browser, with the page's cookie", async () => { + // The SPA CSP (connect-src 'self') blocks fetches from the page itself, so the control is a top-level + // navigation. It proves that a leak would be recorded, and would carry the cookie. + await chrome.goto(`${P}/control`, 500); + await chrome.goto(`${ORIGIN}/harness.html`, 1500); + expect(await chrome.eval("window.stcReady === true")).toBe(true); + const c = hits.find((h) => h.path === "/control"); + expect(c).toBeDefined(); + expect(c!.cookie ?? "").toContain("stc_probe=secret-session"); + }); + + test("the SPA CSP itself blocks a page-level fetch to another origin", async () => { + const before = hits.length; + const r = await chrome.eval(`fetch(${JSON.stringify(P + "/page-fetch")}).then(() => "open", (e) => "blocked")`); + expect(r).toBe("blocked"); + expect(hits.slice(before)).toEqual([]); + }); + + test("the runner still works inside the sandboxed frame", async () => { + const dir = join(FIXTURES, "sample/exercises/add"); + const tests = readFileSync(join(dir, "tests.js"), "utf8"), starter = readFileSync(join(dir, "starter.js"), "utf8"); + const fail = await chrome.eval<{ passed: boolean; expected?: string; actual?: string }[]>(`stcRun(${JSON.stringify(starter)}, ${JSON.stringify(tests)}, 5000)`); + expect(fail.map((r) => r.passed)).toEqual([false, false]); + expect(fail[0]).toMatchObject({ expected: "5", actual: "-1" }); + const pass = await chrome.eval<{ passed: boolean }[]>(`stcRun(${JSON.stringify(starter.replace("a - b", "a + b"))}, ${JSON.stringify(tests)}, 5000)`); + expect(pass.map((r) => r.passed)).toEqual([true, true]); + expect(await chrome.eval("document.querySelectorAll('iframe').length")).toBe(0); + }, 20_000); + + test("the timeout destroys the frame and its worker", async () => { + const tests = `export const tests = [{ name: "loop", run: (m) => m.spin() }];`; + const r = await chrome.eval<{ passed: boolean; message?: string }[]>(`stcRun("export function spin() { while (true) {} }", ${JSON.stringify(tests)}, 800)`); + expect(r[0]!.passed).toBe(false); + expect(r[0]!.message).toContain("Timed out"); + expect(await chrome.eval("document.querySelectorAll('iframe').length")).toBe(0); + }, 20_000); + + const attack = (who: string) => ` + const P = ${JSON.stringify(P)} + "/${who}"; + const tryIt = (f) => { try { const r = f(); if (r && typeof r.then === "function") r.then(() => {}, () => {}); } catch (_) {} }; + tryIt(() => fetch(P + "-fetch", { credentials: "include" })); + tryIt(() => import(P + "-import.js")); + tryIt(() => import(URL.createObjectURL(new Blob(['import "' + P + '-static-import.js";'], { type: "text/javascript" })))); + tryIt(() => caches.open("x").then((c) => c.add(P + "-cache"))); + tryIt(() => new Worker(URL.createObjectURL(new Blob(['fetch("' + P + '-nested", { credentials: "include" })'], { type: "text/javascript" })))); + tryIt(() => { const i = new Image(); i.src = P + "-img"; }); + tryIt(() => new EventSource(P + "-es", { withCredentials: true })); + tryIt(() => new WebSocket(P.replace("http", "ws") + "-ws")); + tryIt(() => { const x = new XMLHttpRequest(); x.open("GET", P + "-xhr"); x.send(); }); + tryIt(() => importScripts(P + "-importscripts.js")); + tryIt(() => navigator.sendBeacon && navigator.sendBeacon(P + "-beacon", "x")); + tryIt(() => import(${JSON.stringify(ORIGIN)} + "/cookie-probe-${who}.js")); + tryIt(() => fetch(${JSON.stringify(ORIGIN)} + "/cookie-probe-${who}-fetch", { credentials: "include" })); + `; + const code = () => `${attack("code")}\nexport const add = (a, b) => a + b;`; + const tests = () => `${attack("tests")}\nexport const tests = [{ name: "wait", run: async () => { ${attack("run")} await new Promise((r) => setTimeout(r, 1500)); } }];`; + const leaksFrom = async (fn: "stcRun" | "stcRunPlain") => { + const before = hits.length; + await chrome.eval(`${fn}(${JSON.stringify(code())}, ${JSON.stringify(tests())}, 4000)`); + await Bun.sleep(1500); + return hits.slice(before); + }; + + let controlEnd = 0; + test("negative control: without the sandbox, a same-origin worker on a page without CSP leaks", async () => { + await chrome.goto(`${ORIGIN}/harness-nocsp.html`, 1500); + const leaked = await leaksFrom("stcRunPlain"); + // Dynamic import gets through the stubs, to other origins and, with the session cookie, to the app itself. + expect(leaked.map((h) => h.path)).toContain("/code-import.js"); + expect(leaked.some((h) => h.path === "app:/cookie-probe-code.js" && (h.cookie ?? "").includes("stc_probe"))).toBe(true); + controlEnd = hits.length; + }, 30_000); + + test("C2: learner and test code reach nothing from the sandboxed runner, even on a page without CSP", async () => { + await chrome.goto(`${ORIGIN}/harness-nocsp.html`, 1500); + expect(await leaksFrom("stcRun")).toEqual([]); + }, 30_000); + + test("C2: learner and test code reach nothing from the sandboxed runner on the SPA (with its CSP)", async () => { + await chrome.goto(`${ORIGIN}/harness.html`, 1500); + expect(await leaksFrom("stcRun")).toEqual([]); + // Only the control navigation (and the favicon request Chrome makes for it) ever carried the cookie. + // (The negative control's own leaks are excluded; they ran the old design on purpose.) + expect(hits.slice(controlEnd).filter((h) => (h.cookie ?? "").includes("stc_probe"))).toEqual([]); + }, 30_000); + + test("C1: the reviewer's exploit lesson does not run script in LessonView (page without CSP)", async () => { + // No CSP here, so the sanitizer alone has to stop the script. + await chrome.goto(`${ORIGIN}/harness-nocsp.html`, 1500); + const beacon = `fetch('${P}/xss')`; + const md = [ + "
      \nThink first\n", + "
      \n", + "", + "
      Hint 1 — nudge\n\nx\n\n
      ", + "[a](javascript:window.__xss=5) [b](JaVaScRiPt:window.__xss=6) c", + "", + ].join("\n\n"); + const before = hits.length; + const html = await chrome.eval(`stcRenderLesson(${JSON.stringify(md)}, ${JSON.stringify(" { for (const a of document.querySelectorAll("#root a, #root summary")) { try { a.dispatchEvent(new MouseEvent("click", { bubbles: true, cancelable: true })); } catch (_) {} } })()`); + await Bun.sleep(800); + expect(await chrome.eval("window.__xss ?? null")).toBeNull(); + expect(chrome.dialogs).toEqual([]); + expect(hits.slice(before).filter((h) => h.path === "/xss")).toEqual([]); + expect(html).toContain("Next paragraph"); + expect(html).not.toMatch(/<[a-z][^>]*\son[a-z]+=/i); + }, 30_000); +}); diff --git a/packages/feature-learner/test/chrome.ts b/packages/feature-learner/test/chrome.ts new file mode 100644 index 0000000..f9dd787 --- /dev/null +++ b/packages/feature-learner/test/chrome.ts @@ -0,0 +1,68 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +/** Path of a usable Chrome, or null (the browser tests are skipped without one). */ +export const chromePath = Bun.which("google-chrome") ?? Bun.which("chromium") ?? Bun.which("chromium-browser"); + +/** A minimal Chrome DevTools Protocol driver for headless end-to-end checks. */ +export class Chrome { + private ws!: WebSocket; + private id = 0; + private pending = new Map void>(); + readonly console: string[] = []; + readonly dialogs: string[] = []; + private proc!: Bun.Subprocess; + private dir!: string; + + static async launch(): Promise { + const c = new Chrome(); + c.dir = mkdtempSync(join(tmpdir(), "stc-chrome-")); + const port = 9400 + Math.floor(Math.random() * 500); + c.proc = Bun.spawn([chromePath!, "--headless=new", "--no-sandbox", "--disable-gpu", "--no-first-run", `--remote-debugging-port=${port}`, `--user-data-dir=${c.dir}`, "about:blank"], { stdout: "ignore", stderr: "ignore" }); + let url: string | null = null; + for (let i = 0; i < 100 && !url; i++) { + await Bun.sleep(100); + try { + const list = (await (await fetch(`http://127.0.0.1:${port}/json`)).json()) as { type: string; webSocketDebuggerUrl: string }[]; + url = list.find((t) => t.type === "page")?.webSocketDebuggerUrl ?? null; + } catch { /* not up yet */ } + } + if (!url) throw new Error("chrome did not start"); + c.ws = new WebSocket(url); + await new Promise((r) => c.ws.addEventListener("open", r, { once: true })); + c.ws.addEventListener("message", (ev) => { + const m = JSON.parse(String(ev.data)); + if (m.id && c.pending.has(m.id)) { c.pending.get(m.id)!(m); c.pending.delete(m.id); } + if (m.method === "Runtime.consoleAPICalled") c.console.push(`${m.params.type}: ${m.params.args.map((a: any) => a.value ?? a.description).join(" ")}`); + if (m.method === "Runtime.exceptionThrown") c.console.push(`exception: ${m.params.exceptionDetails.exception?.description ?? m.params.exceptionDetails.text}`); + if (m.method === "Page.javascriptDialogOpening") { c.dialogs.push(m.params.message); void c.send("Page.handleJavaScriptDialog", { accept: true }); } + }); + await c.send("Page.enable"); + await c.send("Runtime.enable"); + return c; + } + + send(method: string, params: object = {}): Promise<{ result?: any; error?: any }> { + return new Promise((r) => { const i = ++this.id; this.pending.set(i, r); this.ws.send(JSON.stringify({ id: i, method, params })); }); + } + + async goto(url: string, settleMs = 1500) { + await this.send("Page.navigate", { url }); + await Bun.sleep(settleMs); + } + + /** Evaluates `js` in the page (awaiting promises) and returns the JSON value. */ + async eval(js: string): Promise { + const r = await this.send("Runtime.evaluate", { expression: js, awaitPromise: true, returnByValue: true }); + if (r.result?.exceptionDetails) throw new Error(r.result.exceptionDetails.exception?.description ?? "evaluation failed"); + return r.result?.result?.value as T; + } + + async close() { + try { this.ws.close(); } catch { /* closed */ } + this.proc.kill(); + await this.proc.exited; + rmSync(this.dir, { recursive: true, force: true }); + } +} diff --git a/packages/feature-learner/test/e2e/harness.tsx b/packages/feature-learner/test/e2e/harness.tsx new file mode 100644 index 0000000..d44ff19 --- /dev/null +++ b/packages/feature-learner/test/e2e/harness.tsx @@ -0,0 +1,39 @@ +// Browser harness for the headless-Chrome tests: the real runner and the real LessonView, bundled for the browser. +import { createRoot } from "react-dom/client"; +import { createBrowserRunner } from "../../src/web/runner/runner"; +import { WORKER_SOURCE } from "../../src/web/runner/worker-source"; +import { LessonView } from "../../src/web/lesson/LessonView"; +import type { LessonPayload } from "../../src/shared/types"; + +declare global { + interface Window { + stcRun(code: string, tests: string, timeoutMs: number): Promise; + /** Negative control only: the old design, a same-origin blob worker in the page. */ + stcRunPlain(code: string, tests: string, timeoutMs: number): Promise; + stcRenderLesson(markdown: string, hintBody: string): Promise; + stcReady: boolean; + } +} + +window.stcRun = (code, tests, timeoutMs) => createBrowserRunner("js").run(code, tests, timeoutMs); +window.stcRunPlain = (code, tests, timeoutMs) => createBrowserRunner("js", { + createWorker: () => new Worker(URL.createObjectURL(new Blob([WORKER_SOURCE], { type: "text/javascript" })), { type: "module" }), +}).run(code, tests, timeoutMs); + +window.stcRenderLesson = async (markdown, hintBody) => { + const lesson: LessonPayload = { + courseId: "c", courseTitle: "C", lesson: { id: "x", title: "Exploit", type: "exercise", minutes: null }, + module: { id: "m", title: "M", index: 0 }, locked: false, markdown, baseFile: "course/x.md", trackMarkdown: {}, tracks: [], + video: null, transcript: null, hints: [{ label: "nudge", body: hintBody }], decks: [], code: null, attachments: [], browser: null, + localCommand: null, prev: null, next: null, lessonFiles: {}, + }; + const el = document.getElementById("root")!; + createRoot(el).render(); + await new Promise((r) => setTimeout(r, 300)); + // Open every fold so nested content is live in the DOM too. + for (const d of document.querySelectorAll("details")) d.open = true; + await new Promise((r) => setTimeout(r, 500)); + return el.innerHTML; +}; + +window.stcReady = true; diff --git a/packages/feature-learner/test/lesson-view.test.tsx b/packages/feature-learner/test/lesson-view.test.tsx index 60f8478..fbd89f6 100644 --- a/packages/feature-learner/test/lesson-view.test.tsx +++ b/packages/feature-learner/test/lesson-view.test.tsx @@ -106,3 +106,12 @@ test("helpers: local command per track, note timestamps, code tree", () => { expect(t.dirs.get("src")!.files.map((f) => f.path)).toEqual(["src/b.ts"]); expect(t.dirs.get("src")!.dirs.get("x")!.files[0]!.path).toBe("src/x/c.ts"); }); + +test("the exercise editor: no Tab capture, Enter keeps the indentation", async () => { + const { newlineWithIndent } = await import("../src/web/lesson/Exercise"); + expect(newlineWithIndent("function f() {\n return 1", 26)).toBe("\n "); + expect(newlineWithIndent("x", 1)).toBe("\n"); + expect(newlineWithIndent("\t\tx", 3)).toBe("\n\t\t"); + const src = await Bun.file(new URL("../src/web/lesson/Exercise.tsx", import.meta.url)).text(); + expect(src).not.toMatch(/e\.key === "Tab"/); +}); diff --git a/packages/feature-learner/test/markdown.test.ts b/packages/feature-learner/test/markdown.test.ts index 604b444..a3b3630 100644 --- a/packages/feature-learner/test/markdown.test.ts +++ b/packages/feature-learner/test/markdown.test.ts @@ -100,11 +100,12 @@ test("systemic-design hint numbering forms one group", () => { }); test("raw HTML is reduced to safe tags", () => { - expect(sanitizeHtml('x
      ')).toBe("<script>alert(1)</script>x<img src=x onerror=alert(1)>
      "); + expect(sanitizeHtml('x
      ')).toBe('<script>alert(1)</script>x
      '); const html = renderLessonHtml('
      hi
      \n\n[x](javascript:alert(1)) s', opts); - expect(html).not.toContain("hi
      "); + expect(html).not.toContain("onclick"); expect(html).not.toContain("javascript:"); - expect(html).not.toContain(' { @@ -114,7 +115,7 @@ test("links and images resolve inside the course", () => { expect(resolveHref("https://example.com", opts, "link")).toEqual({ url: "https://example.com", external: true }); expect(resolveHref("img/a.png", opts, "image")).toEqual({ url: "/api/media/pure-snake/course/img/a.png", external: false }); const html = renderLessonHtml("![board](img/board.png) and [ext](https://x.org)", opts); - expect(html).toContain('board'); + expect(html).toContain('board'); expect(html).toContain('target="_blank" rel="noopener noreferrer"'); }); diff --git a/packages/feature-learner/test/spa.test.ts b/packages/feature-learner/test/spa.test.ts new file mode 100644 index 0000000..7e78e56 --- /dev/null +++ b/packages/feature-learner/test/spa.test.ts @@ -0,0 +1,65 @@ +import { expect, test } from "bun:test"; +import { inlineScriptHashes, spaCsp, spaWithCsp } from "../src/server/spa"; +import { RUNNER_FRAME_CSP, RUNNER_FRAME_PATH } from "../src/server/runnerFrame"; +import { connect } from "@stc/db"; +import { runMigrations } from "../src/server/migrate"; +import { makeApp } from "./app"; + +const scriptSrc = (csp: string) => csp.split(";").map((d) => d.trim()).find((d) => d.startsWith("script-src"))!; + +test("the SPA CSP allows scripts from 'self' only, never inline", () => { + const s = scriptSrc(spaCsp()); + expect(s).toBe("script-src 'self'"); + expect(spaCsp()).toContain("object-src 'none'"); + expect(spaCsp()).toContain("frame-ancestors 'self'"); + expect(inlineScriptHashes('')).toEqual([ + `sha256-${new Bun.CryptoHasher("sha256").update("x()").digest("base64")}`, + ]); +}); + +// A stand-in for the bundled index.html (bundling inside bun test is unreliable); the real bundle is checked +// against the dev server (see wp-learner-report.md, fix round 1). +const DOC = '
      '; +const upstream = (req: Request) => new URL(req.url).pathname.endsWith(".js") + ? new Response("console.log(1)", { headers: { "Content-Type": "text/javascript" } }) + : new Response(DOC, { headers: { "Content-Type": "text/html;charset=utf-8" } }); + +for (const development of [false, true]) { + test(`the SPA document is served with the CSP header (development: ${development})`, async () => { + const spa = spaWithCsp(upstream, { development }); + const server = Bun.serve({ hostname: "127.0.0.1", port: 0, routes: { "/api/x": () => new Response("api"), ...spa.routes } }); + try { + const r = await fetch(`http://127.0.0.1:${server.port}/c/sample/m1-intro`); + expect(r.status).toBe(200); + const csp = r.headers.get("content-security-policy")!; + expect(await r.text()).toBe(DOC); + const s = scriptSrc(csp); + expect(s).not.toContain("unsafe-inline"); + expect(s).not.toContain("unsafe-eval"); + // Production never allows inline script; development allows only the exact injected helper, by hash. + expect(s).toBe(development ? `script-src 'self' '${inlineScriptHashes(DOC)[0]}'` : "script-src 'self'"); + const asset = await fetch(`http://127.0.0.1:${server.port}/_bun/client/app.js`); + expect(asset.status).toBe(200); + expect(asset.headers.get("content-security-policy")).toBeNull(); + expect(await (await fetch(`http://127.0.0.1:${server.port}/api/x`)).text()).toBe("api"); + } finally { + server.stop(true); + spa.stop(); + } + }); +} + +test("the runner frame is served sandboxed, with no connections and only its own script", async () => { + const db = await connect("sqlite://:memory:"); + await runMigrations(db); + const { call } = await makeApp(db); + const r = await call("GET", RUNNER_FRAME_PATH); + expect(r.status).toBe(200); + const csp = r.headers.get("content-security-policy")!; + expect(csp).toBe(RUNNER_FRAME_CSP); + expect(csp).toContain("connect-src 'none'"); + expect(csp).toContain("sandbox allow-scripts"); + expect(csp).not.toContain("allow-same-origin"); + expect(scriptSrc(csp)).toMatch(/^script-src 'sha256-[A-Za-z0-9+/=]+' blob:$/); + await db.close(); +}); diff --git a/packages/feature-learner/test/visibility.test.ts b/packages/feature-learner/test/visibility.test.ts new file mode 100644 index 0000000..1d9ec5f --- /dev/null +++ b/packages/feature-learner/test/visibility.test.ts @@ -0,0 +1,27 @@ +import { expect } from "bun:test"; +import { forEachDb } from "./helpers"; +import { FIXTURES, makeApp } from "./app"; +import { extraCourses } from "./courses"; + +forEachDb("one visibility rule: the global author role alone does not open a restricted course (I1)", async (db) => { + const dir = await extraCourses(); + const { call, signUp } = await makeApp(db, [FIXTURES, dir]); + const owner = await signUp("owner@x.test"); + const author = await signUp("author@x.test"); + const coAuthor = await signUp("co@x.test"); + const users = (await (await call("GET", "/api/admin/users", { cookie: owner })).json()) as { id: string; email: string }[]; + const id = (e: string) => users.find((u) => u.email === e)!.id; + expect((await call("PUT", `/api/admin/users/${id("author@x.test")}/roles`, { cookie: owner, body: { roles: ["author"] } })).status).toBe(200); + expect((await call("PUT", "/api/admin/courses/secret/access", { cookie: owner, body: { authors: [id("co@x.test")] } })).status).toBe(200); + + const paths = ["/api/courses/secret", "/api/courses/secret/lessons/s1", "/api/media/secret/a.md"]; + for (const p of paths) { + expect([p, (await call("GET", p, { cookie: author })).status]).toEqual([p, 404]); + expect([p, (await call("GET", p, { cookie: coAuthor })).status]).toEqual([p, 200]); + } + const lib = async (c: string) => ((await (await call("GET", "/api/library", { cookie: c })).json()) as { courses: { id: string }[] }).courses.map((x) => x.id); + expect(await lib(author)).not.toContain("secret"); + expect(await lib(coAuthor)).toContain("secret"); + const search = (await (await call("GET", "/api/library/search?q=hidden", { cookie: author })).json()) as unknown[]; + expect(search).toEqual([]); +}); diff --git a/packages/feature-learner/test/xss.test.ts b/packages/feature-learner/test/xss.test.ts new file mode 100644 index 0000000..eaa5727 --- /dev/null +++ b/packages/feature-learner/test/xss.test.ts @@ -0,0 +1,84 @@ +import { expect, test } from "bun:test"; +import { renderHintsHtml, renderLessonHtml, sanitizeHtml } from "../src/web/markdown"; + +const opts = { courseId: "c", baseFile: "course/l.md", tracks: [], lessonFiles: {} }; + +const decode = (s: string) => s.replace(/&#x([0-9a-f]+);?/gi, (_, h) => String.fromCodePoint(parseInt(h, 16))) + .replace(/&#(\d+);?/g, (_, d) => String.fromCodePoint(Number(d))).replace(/:/gi, ":").replace(/&tab;/gi, "\t").replace(/&newline;/gi, "\n").replace(/&/g, "&"); + +/** Fails when real tags in `html` carry event handlers, style, a dangerous URL, or are script-capable elements. */ +function expectSafe(html: string) { + const tags = html.match(/<[a-zA-Z][^>]*>/g) ?? []; + for (const t of tags) { + expect(t).not.toMatch(/\son[a-z]+\s*=/i); + expect(t).not.toMatch(/\sstyle\s*=/i); + expect(t).not.toMatch(/^<(script|style|svg|math|iframe|object|embed|form|input|button|textarea|select|link|meta|base|frame|frameset)\b/i); + for (const m of t.matchAll(/\s(?:href|src|action|formaction|xlink:href)\s*=\s*"([^"]*)"/gi)) { + const url = decode(m[1]!).replace(/[\u0000- \u007f-\u009f]/g, "").toLowerCase(); + expect(url).not.toMatch(/^(javascript|vbscript|data|file):/); + } + } + // An unterminated tag must never swallow following markup into its attributes. + expect(html).not.toMatch(/]*

      or

      ", () => { + for (const wrapper of ["details", "div"]) { + const md = `<${wrapper}>\nThink first\n { + for (const md of [ + "bold", + "
      a\n\n", + "ok'); +}); + +test("
      keeps the fold but drops the handler", () => { + const html = renderLessonHtml('
      Hint 1 — nudge\n\ntext\n\n
      ', opts); + expectSafe(html); + expect(html).toContain('
      ", body: " { + const html = renderLessonHtml("A value of type IO or Stream.", opts); + expect(html).toContain("IO<A>"); + expect(html).toContain("Stream<Action>"); +}); From 213f2247b428e8534d08c3fb86e95d0d858bd678 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 14:27:56 +0100 Subject: [PATCH 56/62] fix(learner): SPA proxy forwards no credentials and never goes through HTTP_PROXY spaWithCsp proxies GET and HEAD only (405 otherwise, no body buffering) and forwards only accept, if-none-match, if-modified-since and range. The internal server listens on a Unix socket in a private 0700 temp dir. Bun 1.3.14 ignores proxy: false for this hop, and over a socket with a proxy set it sends an absolute-form target that breaks asset routing, so localhost, 127.0.0.1 and ::1 are appended to NO_PROXY/no_proxy (existing entries kept). accept-encoding is no longer forced; fetch decompresses and the encoding headers are dropped. Test: in a subprocess with HTTP(S)_PROXY and ALL_PROXY set to a dead proxy, the SPA and an asset load with the CSP, POST gets 405, and the upstream sees no Cookie, Authorization or X-Forwarded-For. Co-Authored-By: Claude Opus 5.5 --- packages/feature-learner/src/server/spa.ts | 38 ++++++--- .../test/e2e/spa-proxy-check.ts | 77 +++++++++++++++++++ packages/feature-learner/test/spa.test.ts | 22 ++++++ 3 files changed, 125 insertions(+), 12 deletions(-) create mode 100644 packages/feature-learner/test/e2e/spa-proxy-check.ts diff --git a/packages/feature-learner/src/server/spa.ts b/packages/feature-learner/src/server/spa.ts index 41342d5..9b774ac 100644 --- a/packages/feature-learner/src/server/spa.ts +++ b/packages/feature-learner/src/server/spa.ts @@ -1,3 +1,6 @@ +import { chmodSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import type { HTMLBundle } from "bun"; /** @@ -36,30 +39,41 @@ export function inlineScriptHashes(html: string): string[] { /** * Serves the SPA (`index.html` bundle) with a CSP header. Bun.serve cannot add headers to an HTML import route, - * so the bundle is served by an internal server bound to 127.0.0.1 on a random port, and these routes proxy to - * it, adding the header to HTML responses. Phase 2 can replace this once core sets the CSP globally + * so the bundle is served by an internal server on a private Unix socket, and these routes proxy to it (GET and + * HEAD only, a few cache/range headers only, never cookies), adding the header to HTML responses. Phase 2 can replace this once core sets the CSP globally * (docs/contract-requests/learner.md §5). */ export function spaWithCsp(indexHtml: HTMLBundle | ((req: Request) => Response | Promise), opts: { development: boolean }) { + // Bun 1.3.14 sends this hop through HTTP_PROXY even with `proxy: false`, and over a Unix socket it then uses an + // absolute-form request target that breaks asset routing. Loopback never belongs behind a proxy, so the loopback + // names are appended to NO_PROXY (existing entries are kept). The socket lives in a private (0700) temp dir. + for (const k of ["NO_PROXY", "no_proxy"]) { + const have = (process.env[k] ?? "").split(",").map((x) => x.trim()).filter(Boolean); + process.env[k] = [...new Set([...have, "localhost", "127.0.0.1", "::1"])].join(","); + } + const dir = mkdtempSync(join(tmpdir(), "stc-spa-")); + chmodSync(dir, 0o700); + const socket = join(dir, "spa.sock"); const inner = Bun.serve({ - hostname: "127.0.0.1", - port: 0, + unix: socket, routes: { "/*": indexHtml }, // HMR needs a websocket on the public server; it is off, and the bundle still rebuilds per request in dev. development: opts.development ? { hmr: false, console: false } : false, fetch: () => new Response("Not found", { status: 404 }), }); - const base = `http://127.0.0.1:${inner.port}`; + + /** Only these request headers reach the internal server: never Cookie, Authorization or X-Forwarded-*. */ + const FORWARD = ["accept", "if-none-match", "if-modified-since", "range"]; async function handle(req: Request): Promise { + // The SPA and its assets are read-only; nothing else is proxied (and no request body is buffered). + if (req.method !== "GET" && req.method !== "HEAD") return new Response("Method not allowed", { status: 405, headers: { Allow: "GET, HEAD" } }); const u = new URL(req.url); - const headers = new Headers(req.headers); - headers.delete("host"); - headers.set("accept-encoding", "identity"); - const r = await fetch(base + u.pathname + u.search, { - method: req.method, headers, body: req.method === "GET" || req.method === "HEAD" ? undefined : await req.arrayBuffer(), redirect: "manual", - }); + const headers = new Headers(); + for (const h of FORWARD) { const v = req.headers.get(h); if (v !== null) headers.set(h, v); } + const r = await fetch(`http://localhost${u.pathname}${u.search}`, { method: req.method, headers, redirect: "manual", unix: socket, proxy: false }); const out = new Headers(r.headers); + // fetch already decompressed the body, so the upstream encoding and length no longer apply. out.delete("content-encoding"); out.delete("content-length"); out.delete("transfer-encoding"); @@ -75,6 +89,6 @@ export function spaWithCsp(indexHtml: HTMLBundle | ((req: Request) => Response | return { routes: { "/*": handle } as Record Promise>, - stop: () => inner.stop(true), + stop: () => { inner.stop(true); rmSync(dir, { recursive: true, force: true }); }, }; } diff --git a/packages/feature-learner/test/e2e/spa-proxy-check.ts b/packages/feature-learner/test/e2e/spa-proxy-check.ts new file mode 100644 index 0000000..8e91deb --- /dev/null +++ b/packages/feature-learner/test/e2e/spa-proxy-check.ts @@ -0,0 +1,77 @@ +// Run by spa.test.ts in a subprocess with HTTP_PROXY/ALL_PROXY pointing at an unreachable proxy. +import { spaWithCsp } from "../../src/server/spa"; + +/** + * The test client speaks raw HTTP/1.1 over Bun.connect: fetch and Bun's node:http both honour HTTP_PROXY, and + * only the code under test should meet the proxy. + */ +async function get(url: string, method = "GET", headers: Record = {}): Promise<{ status: number; headers: Record; body: string }> { + const u = new URL(url); + let raw = ""; + await new Promise((resolve, reject) => { + Bun.connect({ + hostname: u.hostname, port: Number(u.port), + socket: { + open(sock) { + const body = method === "POST" ? "x" : ""; + const lines = [`${method} ${u.pathname}${u.search} HTTP/1.1`, `Host: ${u.host}`, "Connection: close", ...Object.entries(headers).map(([k, v]) => `${k}: ${v}`)]; + if (body) lines.push(`Content-Length: ${body.length}`); + sock.write(`${lines.join("\r\n")}\r\n\r\n${body}`); + }, + data(sock, d) { + raw += new TextDecoder().decode(d); + // Bun keeps the connection open despite "Connection: close"; stop once the whole response is here. + const end = raw.indexOf("\r\n\r\n"); + if (end < 0) return; + const len = /\r\ncontent-length:\s*(\d+)/i.exec(raw.slice(0, end)); + const done = len ? raw.length - end - 4 >= Number(len[1]) : /\r\n0\r\n\r\n$/.test(raw); + if (done) { sock.end(); resolve(); } + }, + close() { resolve(); }, + error(_s, e) { reject(e); }, + }, + }).catch(reject); + }); + const [head, ...rest] = raw.split("\r\n\r\n"); + const [statusLine, ...hl] = head!.split("\r\n"); + const hs = Object.fromEntries(hl.map((l) => [l.slice(0, l.indexOf(":")).toLowerCase(), l.slice(l.indexOf(":") + 1).trim()])); + let body = rest.join("\r\n\r\n"); + if (hs["transfer-encoding"] === "chunked") { + let out = "", i = 0; + for (;;) { const nl = body.indexOf("\r\n", i); const n = parseInt(body.slice(i, nl), 16); if (!n) break; out += body.slice(nl + 2, nl + 2 + n); i = nl + 2 + n + 2; } + body = out; + } + return { status: Number(statusLine!.split(" ")[1]), headers: hs, body }; +} + +// Control, before spaWithCsp touches NO_PROXY: a plain loopback fetch in this process goes to the env proxy. +const plain = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => new Response("direct") }); +const controlViaEnvProxy = await fetch(`http://127.0.0.1:${plain.port}/`, { signal: AbortSignal.timeout(2000) }).then((r) => `status ${r.status}`, (e) => `error ${(e as Error).name}`); +plain.stop(true); + +const seen: Record[] = []; +const DOC = '
      '; +const spa = spaWithCsp((req) => { + if (new URL(req.url).pathname === "/app.js") return new Response("console.log(1)", { headers: { "Content-Type": "text/javascript" } }); + seen.push({ cookie: req.headers.get("cookie"), authorization: req.headers.get("authorization"), xff: req.headers.get("x-forwarded-for"), accept: req.headers.get("accept") }); + return new Response(DOC, { headers: { "Content-Type": "text/html;charset=utf-8" } }); +}, { development: false }); +const server = Bun.serve({ hostname: "127.0.0.1", port: 0, routes: spa.routes }); +const url = `http://127.0.0.1:${server.port}/c/sample`; +const out: Record = {}; +try { + out.controlViaEnvProxy = controlViaEnvProxy; + const r = await get(url, "GET", { cookie: "better-auth.session_token=secret", authorization: "Bearer secret", "x-forwarded-for": "1.2.3.4", accept: "text/html" }); + out.status = r.status; + out.csp = r.headers["content-security-policy"]; + out.body = r.body; + out.post = (await get(url, "POST")).status; + const asset = await get(`http://127.0.0.1:${server.port}/app.js`); + out.asset = `${asset.status} ${asset.headers["content-type"]} ${asset.body}`; + out.noProxy = process.env.NO_PROXY; + out.seen = seen; +} finally { + server.stop(true); + spa.stop(); +} +console.log(JSON.stringify(out)); diff --git a/packages/feature-learner/test/spa.test.ts b/packages/feature-learner/test/spa.test.ts index 7e78e56..2e9bd2a 100644 --- a/packages/feature-learner/test/spa.test.ts +++ b/packages/feature-learner/test/spa.test.ts @@ -63,3 +63,25 @@ test("the runner frame is served sandboxed, with no connections and only its own expect(scriptSrc(csp)).toMatch(/^script-src 'sha256-[A-Za-z0-9+/=]+' blob:$/); await db.close(); }); + +test("the SPA proxy ignores HTTP_PROXY/ALL_PROXY and forwards no credentials", () => { + const dead = "http://127.0.0.1:9"; // nothing listens on the discard port + const p = Bun.spawnSync([process.execPath, new URL("./e2e/spa-proxy-check.ts", import.meta.url).pathname], { + env: { ...process.env, HTTP_PROXY: dead, http_proxy: dead, HTTPS_PROXY: dead, https_proxy: dead, ALL_PROXY: dead, all_proxy: dead, NO_PROXY: "corp.example", no_proxy: "" }, + stdout: "pipe", stderr: "pipe", + }); + expect(p.exitCode).toBe(0); + const out = JSON.parse(p.stdout.toString().trim().split("\n").pop()!); + // The environment proxy really is in effect for plain fetches in that process... + expect(out.controlViaEnvProxy).not.toBe("status 200"); + // ...but the SPA still loads through spaWithCsp, with its CSP. + expect(out.status).toBe(200); + expect(out.body).toContain('
      '); + expect(out.csp).toContain("script-src 'self'"); + expect(out.post).toBe(405); + // Assets come through too (with a proxy in the env, Bun's socket fetch would otherwise hit the HTML fallback). + expect(out.asset).toBe("200 text/javascript console.log(1)"); + expect(out.noProxy).toBe("corp.example,localhost,127.0.0.1,::1"); + // The upstream saw only the allowlisted headers: no Cookie, Authorization or X-Forwarded-*. + expect(out.seen).toEqual([{ cookie: null, authorization: null, xff: null, accept: "text/html" }]); +}, 30_000); From 0da35ec6f849d2a245eb88229018a384906e72a5 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 14:38:04 +0100 Subject: [PATCH 57/62] fix(learner): remove the SPA proxy; CSP as a meta tag in index.html (ruling R16) The proxy, its internal server, Unix socket and NO_PROXY changes are gone; main.ts serves the HTML import directly again. apps/web/index.html carries the SPA policy as ahead of every script (script-src 'self' plus the hash of Bun's dev-server helper, no unsafe-inline/eval; no frame-ancestors, which meta cannot carry). Dev HMR connects under it. The runner frame keeps its header CSP. Tests: the served index.html (prod and dev, in a subprocess) has the meta policy and every inline script is hash-covered; the XSS browser test fails with the sanitizer disabled and passes with it, and the meta CSP alone blocks unsanitized handlers (with a no-CSP control); no source assigns proxy env vars, and the feature leaves them untouched at runtime. Contract request 5 asks core for frame-ancestors/X-Frame-Options. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/main.ts | 7 +- apps/web/index.html | 4 + docs/contract-requests/learner.md | 21 +++- packages/feature-learner/src/index.ts | 1 - packages/feature-learner/src/server/spa.ts | 94 ------------------ .../test/browser-isolation.test.ts | 85 +++++++++++----- packages/feature-learner/test/csp.test.ts | 96 +++++++++++++++++++ packages/feature-learner/test/e2e/harness.tsx | 13 +++ .../feature-learner/test/e2e/serve-index.ts | 12 +++ .../test/e2e/spa-proxy-check.ts | 77 --------------- packages/feature-learner/test/spa.test.ts | 87 ----------------- 11 files changed, 204 insertions(+), 293 deletions(-) delete mode 100644 packages/feature-learner/src/server/spa.ts create mode 100644 packages/feature-learner/test/csp.test.ts create mode 100644 packages/feature-learner/test/e2e/serve-index.ts delete mode 100644 packages/feature-learner/test/e2e/spa-proxy-check.ts delete mode 100644 packages/feature-learner/test/spa.test.ts diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index 9187eae..ca26cd7 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -2,7 +2,7 @@ import type { AppConfig, AuthService, Feature, JobHandler, JobQueue, ServerConte import { createApp, createJobQueue, loadConfig } from "@stc/core"; import { createCourseRepository } from "@stc/course-format"; import { connect, migrate } from "@stc/db"; -import { learnerFeature, spaWithCsp } from "@stc/feature-learner"; +import { learnerFeature } from "@stc/feature-learner"; import indexHtml from "../../web/index.html"; // Used until a feature supplies an `auth` service (the learner package does). @@ -31,19 +31,16 @@ export async function startServer( ctx.jobs = jobs; const app = createApp(ctx, built); jobs.start(); - // The SPA document carries a CSP header (served through spaWithCsp; contract request learner §5). - const spa = spaWithCsp(indexHtml, { development: process.env.NODE_ENV !== "production" }); const server = Bun.serve({ port: config.port, // Bun routes win over fetch; "/api/*" falls through to the composed app (404 for unknown API paths). - routes: { ...app.routes, "/api/*": (req) => app.fetch(req), ...spa.routes }, + routes: { ...app.routes, "/api/*": (req) => app.fetch(req), "/*": indexHtml }, fetch: (req) => app.fetch(req), }); return { server, async stop() { await server.stop(true); - spa.stop(); await jobs.stop(); await db.close(); }, diff --git a/apps/web/index.html b/apps/web/index.html index 2d79c9a..cd1895c 100644 --- a/apps/web/index.html +++ b/apps/web/index.html @@ -3,6 +3,10 @@ + + STCourses diff --git a/docs/contract-requests/learner.md b/docs/contract-requests/learner.md index b751e07..6121a3d 100644 --- a/docs/contract-requests/learner.md +++ b/docs/contract-requests/learner.md @@ -29,13 +29,26 @@ and let `startServer` take `Array`. `import { LessonView } from "@stc/feature-learner/web"`. Props: `lesson: LessonPayload` (the JSON from `GET /api/courses/:id/lessons/:lessonId`), plus optional `look`, `track`, `onTrackChange`, `preview`. With `preview` set, it makes no notes, attempt or progress requests; it still loads code listings and decks through the normal media routes. It injects its own stylesheet (scoped under `.lr`) and works with or without a router. -## 5. Content-Security-Policy on the SPA, set by core (review C1) +## 5. Content-Security-Policy on the SPA (review C1, ruling R16) -**Problem.** Lesson HTML is author content. The sanitizer (sanitize-html allowlist) is the first defence, and a CSP without inline script is the second. Bun.serve cannot attach headers to an HTML import route. +**Applied in `feat/learner`.** The SPA's CSP is a `` tag in `apps/web/index.html`, ahead of every script. `Bun.serve` serves the HTML import directly, as in Phase 0; there is no proxy and no change to proxy environment variables. The policy: -**Workaround in `feat/learner` (applied).** `spaWithCsp(indexHtml, { development })` in `@stc/feature-learner` serves the bundle from an internal server bound to 127.0.0.1 and proxies it. HTML responses get `Content-Security-Policy` (`script-src 'self'`, no `'unsafe-inline'`; `connect-src 'self'`; `object-src 'none'`; `frame-ancestors 'self'`; see `spaCsp()`), plus `X-Content-Type-Options: nosniff` and `Referrer-Policy: same-origin`. `main.ts` uses it for `"/*"`. HMR is off on the internal server, so Bun injects no inline script. If it ever does in development, only the hashes of those exact scripts are allowed. +``` +default-src 'self'; script-src 'self' 'sha256-'; style-src 'self' 'unsafe-inline'; +img-src 'self' data: blob: https:; media-src 'self' blob:; font-src 'self' data:; connect-src 'self'; +worker-src 'self' blob:; frame-src 'self'; object-src 'none'; base-uri 'self'; form-action 'self' +``` -**Proposal.** Core owns the SPA route and sets this CSP globally (and for every feature's HTML), replacing the proxy. Other features must then avoid inline scripts as well. The studio preview through `LessonView` already complies. +- **No inline script.** + - The one hash is the inline helper Bun's dev server injects (the `/_bun/unref` beacon). Production serving does not inject it. + - If a Bun upgrade changes the helper, the only effect is a dev-only CSP console error; `test/csp.test.ts` (dev mode) will then fail and name it. +- **Dev HMR works:** the HMR websocket `ws:///_bun/hmr` is same-origin, and Chrome matches it with `connect-src 'self'` (checked: handshake 101, no violations). +- **The runner frame** (`/api/learner/runner-frame`) keeps its own header-based CSP, including `sandbox allow-scripts` and `frame-ancestors 'self'`. + +**Proposal for core (Phase 2).** +- A meta tag cannot carry `frame-ancestors`, `sandbox` or `report-uri`. Core should additionally send `Content-Security-Policy: frame-ancestors 'self'` (or `X-Frame-Options: SAMEORIGIN`), plus `X-Content-Type-Options: nosniff`, on HTML responses. That is possible if Bun ever allows headers on HTML imports, or by serving a production static build (`bun build apps/web/index.html`) through a normal route that sets headers. +- Ideally core then sends the whole policy as a header and the meta tag stays as defence in depth. +- Every feature's web code must keep to this policy: no inline scripts or event handlers, and no `eval`. The learner and studio preview, through `LessonView`, comply. ## 6. `canView` and the global `author` role (review I1) diff --git a/packages/feature-learner/src/index.ts b/packages/feature-learner/src/index.ts index dc67d6b..c01208d 100644 --- a/packages/feature-learner/src/index.ts +++ b/packages/feature-learner/src/index.ts @@ -1,7 +1,6 @@ // Server export { migrationsDirs, runMigrations } from "./server/migrate"; export { learnerFeature } from "./server/feature"; -export { spaWithCsp, spaCsp } from "./server/spa"; // Browser (also available without the server half as "@stc/feature-learner/web") export { learnerWeb, LessonView, type LessonViewProps, ensureLearnerStyles } from "./web"; // Shared JSON shapes diff --git a/packages/feature-learner/src/server/spa.ts b/packages/feature-learner/src/server/spa.ts deleted file mode 100644 index 9b774ac..0000000 --- a/packages/feature-learner/src/server/spa.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { chmodSync, mkdtempSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import type { HTMLBundle } from "bun"; - -/** - * Content-Security-Policy for the SPA document. No inline script: `script-src 'self'` only (plus, in development, - * the hashes of the inline helper scripts Bun's dev server injects). Style attributes are allowed because React - * components set them; styles cannot run script. The exercise runner frame is a same-origin URL (`frame-src - * 'self'`) with its own, stricter policy, see `runnerFrame.ts`. - */ -export function spaCsp(inlineScriptHashes: string[] = []): string { - const hashes = inlineScriptHashes.map((h) => ` '${h}'`).join(""); - return [ - "default-src 'self'", - `script-src 'self'${hashes}`, - "style-src 'self' 'unsafe-inline'", - "img-src 'self' data: blob: https:", - "media-src 'self' blob:", - "font-src 'self' data:", - "connect-src 'self'", - "worker-src 'self' blob:", - "frame-src 'self'", - "object-src 'none'", - "base-uri 'self'", - "form-action 'self'", - "frame-ancestors 'self'", - ].join("; "); -} - -/** `sha256-…` CSP sources for every inline `
      `; + // The SPA's own CSP meta tag, taken from apps/web/index.html (the policy the learner app runs under). + const meta = /]*>/i.exec(readFileSync(join(import.meta.dir, "../../../apps/web/index.html"), "utf8"))![0]; + const page = (csp: boolean) => `${csp ? meta : ""}
      `; app = Bun.serve({ hostname: "localhost", port: 0, fetch(req) { const u = new URL(req.url); // The harness page gets the same CSP as the SPA document. - if (u.pathname === "/harness.html") return new Response(page, { headers: { "Content-Type": "text/html", "Content-Security-Policy": spaCsp(), "Set-Cookie": "stc_probe=secret-session; Path=/; SameSite=Lax" } }); + if (u.pathname === "/harness.html") return new Response(page(true), { headers: { "Content-Type": "text/html", "Set-Cookie": "stc_probe=secret-session; Path=/; SameSite=Lax" } }); // The same page without any CSP: the runner must isolate learner code on its own. - if (u.pathname === "/harness-nocsp.html") return new Response(page, { headers: { "Content-Type": "text/html" } }); + if (u.pathname === "/harness-nocsp.html") return new Response(page(false), { headers: { "Content-Type": "text/html" } }); // Same-origin probe: a request here from learner code would carry the session cookie. if (u.pathname.startsWith("/cookie-probe")) { hits.push({ path: `app:${u.pathname}`, cookie: req.headers.get("cookie") }); @@ -77,7 +89,7 @@ run("browser isolation (headless Chrome)", () => { // The SPA CSP (connect-src 'self') blocks fetches from the page itself, so the control is a top-level // navigation. It proves that a leak would be recorded, and would carry the cookie. await chrome.goto(`${P}/control`, 500); - await chrome.goto(`${ORIGIN}/harness.html`, 1500); + await openHarness("/harness.html"); expect(await chrome.eval("window.stcReady === true")).toBe(true); const c = hits.find((h) => h.path === "/control"); expect(c).toBeDefined(); @@ -138,7 +150,7 @@ run("browser isolation (headless Chrome)", () => { let controlEnd = 0; test("negative control: without the sandbox, a same-origin worker on a page without CSP leaks", async () => { - await chrome.goto(`${ORIGIN}/harness-nocsp.html`, 1500); + await openHarness("/harness-nocsp.html"); const leaked = await leaksFrom("stcRunPlain"); // Dynamic import gets through the stubs, to other origins and, with the session cookie, to the app itself. expect(leaked.map((h) => h.path)).toContain("/code-import.js"); @@ -147,39 +159,62 @@ run("browser isolation (headless Chrome)", () => { }, 30_000); test("C2: learner and test code reach nothing from the sandboxed runner, even on a page without CSP", async () => { - await chrome.goto(`${ORIGIN}/harness-nocsp.html`, 1500); + await openHarness("/harness-nocsp.html"); expect(await leaksFrom("stcRun")).toEqual([]); }, 30_000); test("C2: learner and test code reach nothing from the sandboxed runner on the SPA (with its CSP)", async () => { - await chrome.goto(`${ORIGIN}/harness.html`, 1500); + await openHarness("/harness.html"); expect(await leaksFrom("stcRun")).toEqual([]); // Only the control navigation (and the favicon request Chrome makes for it) ever carried the cookie. // (The negative control's own leaks are excluded; they ran the old design on purpose.) expect(hits.slice(controlEnd).filter((h) => (h.cookie ?? "").includes("stc_probe"))).toEqual([]); }, 30_000); - test("C1: the reviewer's exploit lesson does not run script in LessonView (page without CSP)", async () => { - // No CSP here, so the sanitizer alone has to stop the script. - await chrome.goto(`${ORIGIN}/harness-nocsp.html`, 1500); - const beacon = `fetch('${P}/xss')`; - const md = [ - "
      \nThink first\n", - "
      \n", - "", - "
      Hint 1 — nudge\n\nx\n\n
      ", - "[a](javascript:window.__xss=5) [b](JaVaScRiPt:window.__xss=6)       c", - "", - ].join("\n\n"); + const exploitMd = () => [ + "
      \nThink first\n", + "
      \n", + "", + "
      Hint 1 — nudge\n\nx\n\n
      ", + "[a](javascript:window.__xss=5) [b](JaVaScRiPt:window.__xss=6) c", + "", + ].join("\n\n"); + + async function renderExploit(pagePath: string) { + await openHarness(pagePath); const before = hits.length; - const html = await chrome.eval(`stcRenderLesson(${JSON.stringify(md)}, ${JSON.stringify("(`stcRenderLesson(${JSON.stringify(exploitMd())}, ${JSON.stringify(" { for (const a of document.querySelectorAll("#root a, #root summary")) { try { a.dispatchEvent(new MouseEvent("click", { bubbles: true, cancelable: true })); } catch (_) {} } })()`); await Bun.sleep(800); - expect(await chrome.eval("window.__xss ?? null")).toBeNull(); + return { html, xss: await chrome.eval("window.__xss ?? null"), leaked: hits.slice(before).filter((h) => h.path === "/xss") }; + } + + test("C1, first layer: the exploit lesson runs no script on a page without CSP (sanitizer alone)", async () => { + const r = await renderExploit("/harness-nocsp.html"); + expect(r.xss).toBeNull(); expect(chrome.dialogs).toEqual([]); - expect(hits.slice(before).filter((h) => h.path === "/xss")).toEqual([]); - expect(html).toContain("Next paragraph"); - expect(html).not.toMatch(/<[a-z][^>]*\son[a-z]+=/i); + expect(r.leaked).toEqual([]); + expect(r.html).toContain("Next paragraph"); + expect(r.html).not.toMatch(/<[a-z][^>]*\son[a-z]+=/i); + }, 30_000); + + test("C1, both layers: the exploit lesson runs no script under the SPA's meta CSP", async () => { + const r = await renderExploit("/harness.html"); + expect(r.xss).toBeNull(); + expect(r.leaked).toEqual([]); + expect(r.html).toContain("Next paragraph"); + }, 30_000); + + const RAW = `a`; + + test("C1, second layer: the meta CSP alone blocks unsanitized handlers (control: they run without it)", async () => { + await openHarness("/harness-nocsp.html"); + await chrome.eval(`stcRawHtml(${JSON.stringify(RAW)})`); + expect(await chrome.eval("window.__xss ?? null")).not.toBeNull(); // the raw payload does run without a CSP + await openHarness("/harness.html"); + const violations = await chrome.eval(`stcRawHtml(${JSON.stringify(RAW)})`); + expect(await chrome.eval("window.__xss ?? null")).toBeNull(); + expect(violations).toBeGreaterThan(0); }, 30_000); }); diff --git a/packages/feature-learner/test/csp.test.ts b/packages/feature-learner/test/csp.test.ts new file mode 100644 index 0000000..e5bd7c9 --- /dev/null +++ b/packages/feature-learner/test/csp.test.ts @@ -0,0 +1,96 @@ +import { expect, test } from "bun:test"; +import { Glob } from "bun"; +import { join } from "node:path"; +import { connect } from "@stc/db"; +import { RUNNER_FRAME_CSP, RUNNER_FRAME_PATH } from "../src/server/runnerFrame"; +import { runMigrations } from "../src/server/migrate"; +import { makeApp } from "./app"; +import { forEachDb } from "./helpers"; + +const ROOT = join(import.meta.dir, "../../.."); +const INDEX = join(ROOT, "apps/web/index.html"); + +/** The CSP meta tag's policy from an HTML document, as a directive map. */ +export function metaCsp(html: string): Map | null { + const m = //i.exec(html); + if (!m) return null; + return new Map(m[1]!.split(";").map((d) => d.trim()).filter(Boolean).map((d) => { + const [name, ...values] = d.split(/\s+/); + return [name!, values] as const; + })); +} + +function expectSpaPolicy(p: Map | null) { + expect(p).not.toBeNull(); + const script = p!.get("script-src")!; + expect(script[0]).toBe("'self'"); + // No inline script: only 'self' plus exact hashes (Bun's dev-server helper), never unsafe-inline/eval. + for (const s of script.slice(1)) expect(s).toMatch(/^'sha256-[A-Za-z0-9+/=]+'$/); + expect(p!.get("default-src")).toEqual(["'self'"]); + expect(p!.get("connect-src")).toEqual(["'self'"]); + expect(p!.get("object-src")).toEqual(["'none'"]); + expect(p!.get("base-uri")).toEqual(["'self'"]); + expect(p!.get("frame-src")).toEqual(["'self'"]); + // Directives a meta tag cannot carry are not present (they would be ignored with a console warning). + for (const d of ["frame-ancestors", "sandbox", "report-uri"]) expect(p!.has(d)).toBe(false); +} + +test("apps/web/index.html carries the SPA CSP as a meta tag", async () => { + const html = await Bun.file(INDEX).text(); + expectSpaPolicy(metaCsp(html)); + // The meta tag must come before any script, so it governs all of them. + expect(html.indexOf('http-equiv="Content-Security-Policy"')).toBeLessThan(html.indexOf(" { + const p = Bun.spawnSync([process.execPath, join(import.meta.dir, "e2e/serve-index.ts"), mode], { stdout: "pipe", stderr: "pipe", cwd: ROOT }); + expect(p.exitCode).toBe(0); + const out = JSON.parse(p.stdout.toString().trim().split("\n").pop()!) as { status: number; type: string; html: string }; + expect(out.status).toBe(200); + expect(out.type).toContain("text/html"); + const policy = metaCsp(out.html); + expectSpaPolicy(policy); + // Every inline script Bun injects is covered by a hash in the policy (dev: the /_bun/unref helper). + const allowed = new Set(policy!.get("script-src")!.slice(1)); + for (const m of out.html.matchAll(/]*\bsrc\s*=)[^>]*>([\s\S]*?)<\/script>/gi)) { + expect(allowed.has(`'sha256-${new Bun.CryptoHasher("sha256").update(m[1]!).digest("base64")}'`)).toBe(true); + } + }, 60_000); +} + +test("the runner frame is served sandboxed, with no connections and only its own script", async () => { + const db = await connect("sqlite://:memory:"); + await runMigrations(db); + const { call } = await makeApp(db); + const r = await call("GET", RUNNER_FRAME_PATH); + expect(r.status).toBe(200); + const csp = r.headers.get("content-security-policy")!; + expect(csp).toBe(RUNNER_FRAME_CSP); + expect(csp).toContain("connect-src 'none'"); + expect(csp).toContain("sandbox allow-scripts"); + expect(csp).not.toContain("allow-same-origin"); + expect(csp.split(";").map((d) => d.trim()).find((d) => d.startsWith("script-src"))).toMatch(/^script-src 'sha256-[A-Za-z0-9+/=]+' blob:$/); + await db.close(); +}); + +const PROXY_VARS = ["HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy", "NO_PROXY", "no_proxy"]; + +test("no source file changes proxy environment variables", async () => { + const offenders: string[] = []; + const assign = new RegExp(`process\\.env(\\.(${PROXY_VARS.join("|")})|\\[[^\\]]+\\])\\s*(=(?!=)|\\?\\?=|\\|\\|=)|delete\\s+process\\.env|Object\\.assign\\(\\s*process\\.env`); + for (const dir of ["packages", "apps"]) { + for await (const f of new Glob("*/src/**/*.{ts,tsx}").scan({ cwd: join(ROOT, dir) })) { + const src = await Bun.file(join(ROOT, dir, f)).text(); + if (assign.test(src)) offenders.push(`${dir}/${f}`); + } + } + expect(offenders).toEqual([]); +}); + +forEachDb("starting the learner feature and serving requests leaves proxy variables untouched", async (db) => { + const before = Object.fromEntries(PROXY_VARS.map((k) => [k, process.env[k]])); + const { call } = await makeApp(db); + for (const p of ["/api/library", "/api/courses/sample", RUNNER_FRAME_PATH, "/api/learner/pdf.worker.mjs"]) expect((await call("GET", p)).status).toBe(200); + expect(Object.fromEntries(PROXY_VARS.map((k) => [k, process.env[k]]))).toEqual(before); +}); diff --git a/packages/feature-learner/test/e2e/harness.tsx b/packages/feature-learner/test/e2e/harness.tsx index d44ff19..ef6f0fa 100644 --- a/packages/feature-learner/test/e2e/harness.tsx +++ b/packages/feature-learner/test/e2e/harness.tsx @@ -11,6 +11,8 @@ declare global { /** Negative control only: the old design, a same-origin blob worker in the page. */ stcRunPlain(code: string, tests: string, timeoutMs: number): Promise; stcRenderLesson(markdown: string, hintBody: string): Promise; + /** Sets unsanitized HTML with innerHTML (bypasses the sanitizer) to test the CSP layer on its own. */ + stcRawHtml(html: string): Promise; stcReady: boolean; } } @@ -36,4 +38,15 @@ window.stcRenderLesson = async (markdown, hintBody) => { return el.innerHTML; }; +window.stcRawHtml = async (html) => { + let violations = 0; + document.addEventListener("securitypolicyviolation", () => { violations++; }); + const el = document.createElement("div"); + el.innerHTML = html; + document.body.append(el); + for (const a of el.querySelectorAll("a, summary, img")) { try { a.dispatchEvent(new MouseEvent("click", { bubbles: true, cancelable: true })); } catch (_) { /* ignore */ } } + await new Promise((r) => setTimeout(r, 600)); + return violations; +}; + window.stcReady = true; diff --git a/packages/feature-learner/test/e2e/serve-index.ts b/packages/feature-learner/test/e2e/serve-index.ts new file mode 100644 index 0000000..70ea2a8 --- /dev/null +++ b/packages/feature-learner/test/e2e/serve-index.ts @@ -0,0 +1,12 @@ +// Run by csp.test.ts in a subprocess (bundling an HTML import inside a full bun test run is unreliable): +// serves apps/web/index.html the way main.ts does, as a plain Bun HTML import, and prints the document. +import indexHtml from "../../../../apps/web/index.html"; + +const development = process.argv[2] === "dev"; +const server = Bun.serve({ hostname: "127.0.0.1", port: 0, routes: { "/*": indexHtml }, development }); +try { + const r = await fetch(`http://127.0.0.1:${server.port}/c/sample/m1-intro`); + console.log(JSON.stringify({ status: r.status, type: r.headers.get("content-type"), html: await r.text() })); +} finally { + server.stop(true); +} diff --git a/packages/feature-learner/test/e2e/spa-proxy-check.ts b/packages/feature-learner/test/e2e/spa-proxy-check.ts deleted file mode 100644 index 8e91deb..0000000 --- a/packages/feature-learner/test/e2e/spa-proxy-check.ts +++ /dev/null @@ -1,77 +0,0 @@ -// Run by spa.test.ts in a subprocess with HTTP_PROXY/ALL_PROXY pointing at an unreachable proxy. -import { spaWithCsp } from "../../src/server/spa"; - -/** - * The test client speaks raw HTTP/1.1 over Bun.connect: fetch and Bun's node:http both honour HTTP_PROXY, and - * only the code under test should meet the proxy. - */ -async function get(url: string, method = "GET", headers: Record = {}): Promise<{ status: number; headers: Record; body: string }> { - const u = new URL(url); - let raw = ""; - await new Promise((resolve, reject) => { - Bun.connect({ - hostname: u.hostname, port: Number(u.port), - socket: { - open(sock) { - const body = method === "POST" ? "x" : ""; - const lines = [`${method} ${u.pathname}${u.search} HTTP/1.1`, `Host: ${u.host}`, "Connection: close", ...Object.entries(headers).map(([k, v]) => `${k}: ${v}`)]; - if (body) lines.push(`Content-Length: ${body.length}`); - sock.write(`${lines.join("\r\n")}\r\n\r\n${body}`); - }, - data(sock, d) { - raw += new TextDecoder().decode(d); - // Bun keeps the connection open despite "Connection: close"; stop once the whole response is here. - const end = raw.indexOf("\r\n\r\n"); - if (end < 0) return; - const len = /\r\ncontent-length:\s*(\d+)/i.exec(raw.slice(0, end)); - const done = len ? raw.length - end - 4 >= Number(len[1]) : /\r\n0\r\n\r\n$/.test(raw); - if (done) { sock.end(); resolve(); } - }, - close() { resolve(); }, - error(_s, e) { reject(e); }, - }, - }).catch(reject); - }); - const [head, ...rest] = raw.split("\r\n\r\n"); - const [statusLine, ...hl] = head!.split("\r\n"); - const hs = Object.fromEntries(hl.map((l) => [l.slice(0, l.indexOf(":")).toLowerCase(), l.slice(l.indexOf(":") + 1).trim()])); - let body = rest.join("\r\n\r\n"); - if (hs["transfer-encoding"] === "chunked") { - let out = "", i = 0; - for (;;) { const nl = body.indexOf("\r\n", i); const n = parseInt(body.slice(i, nl), 16); if (!n) break; out += body.slice(nl + 2, nl + 2 + n); i = nl + 2 + n + 2; } - body = out; - } - return { status: Number(statusLine!.split(" ")[1]), headers: hs, body }; -} - -// Control, before spaWithCsp touches NO_PROXY: a plain loopback fetch in this process goes to the env proxy. -const plain = Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: () => new Response("direct") }); -const controlViaEnvProxy = await fetch(`http://127.0.0.1:${plain.port}/`, { signal: AbortSignal.timeout(2000) }).then((r) => `status ${r.status}`, (e) => `error ${(e as Error).name}`); -plain.stop(true); - -const seen: Record[] = []; -const DOC = '
      '; -const spa = spaWithCsp((req) => { - if (new URL(req.url).pathname === "/app.js") return new Response("console.log(1)", { headers: { "Content-Type": "text/javascript" } }); - seen.push({ cookie: req.headers.get("cookie"), authorization: req.headers.get("authorization"), xff: req.headers.get("x-forwarded-for"), accept: req.headers.get("accept") }); - return new Response(DOC, { headers: { "Content-Type": "text/html;charset=utf-8" } }); -}, { development: false }); -const server = Bun.serve({ hostname: "127.0.0.1", port: 0, routes: spa.routes }); -const url = `http://127.0.0.1:${server.port}/c/sample`; -const out: Record = {}; -try { - out.controlViaEnvProxy = controlViaEnvProxy; - const r = await get(url, "GET", { cookie: "better-auth.session_token=secret", authorization: "Bearer secret", "x-forwarded-for": "1.2.3.4", accept: "text/html" }); - out.status = r.status; - out.csp = r.headers["content-security-policy"]; - out.body = r.body; - out.post = (await get(url, "POST")).status; - const asset = await get(`http://127.0.0.1:${server.port}/app.js`); - out.asset = `${asset.status} ${asset.headers["content-type"]} ${asset.body}`; - out.noProxy = process.env.NO_PROXY; - out.seen = seen; -} finally { - server.stop(true); - spa.stop(); -} -console.log(JSON.stringify(out)); diff --git a/packages/feature-learner/test/spa.test.ts b/packages/feature-learner/test/spa.test.ts deleted file mode 100644 index 2e9bd2a..0000000 --- a/packages/feature-learner/test/spa.test.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { expect, test } from "bun:test"; -import { inlineScriptHashes, spaCsp, spaWithCsp } from "../src/server/spa"; -import { RUNNER_FRAME_CSP, RUNNER_FRAME_PATH } from "../src/server/runnerFrame"; -import { connect } from "@stc/db"; -import { runMigrations } from "../src/server/migrate"; -import { makeApp } from "./app"; - -const scriptSrc = (csp: string) => csp.split(";").map((d) => d.trim()).find((d) => d.startsWith("script-src"))!; - -test("the SPA CSP allows scripts from 'self' only, never inline", () => { - const s = scriptSrc(spaCsp()); - expect(s).toBe("script-src 'self'"); - expect(spaCsp()).toContain("object-src 'none'"); - expect(spaCsp()).toContain("frame-ancestors 'self'"); - expect(inlineScriptHashes('')).toEqual([ - `sha256-${new Bun.CryptoHasher("sha256").update("x()").digest("base64")}`, - ]); -}); - -// A stand-in for the bundled index.html (bundling inside bun test is unreliable); the real bundle is checked -// against the dev server (see wp-learner-report.md, fix round 1). -const DOC = '
      '; -const upstream = (req: Request) => new URL(req.url).pathname.endsWith(".js") - ? new Response("console.log(1)", { headers: { "Content-Type": "text/javascript" } }) - : new Response(DOC, { headers: { "Content-Type": "text/html;charset=utf-8" } }); - -for (const development of [false, true]) { - test(`the SPA document is served with the CSP header (development: ${development})`, async () => { - const spa = spaWithCsp(upstream, { development }); - const server = Bun.serve({ hostname: "127.0.0.1", port: 0, routes: { "/api/x": () => new Response("api"), ...spa.routes } }); - try { - const r = await fetch(`http://127.0.0.1:${server.port}/c/sample/m1-intro`); - expect(r.status).toBe(200); - const csp = r.headers.get("content-security-policy")!; - expect(await r.text()).toBe(DOC); - const s = scriptSrc(csp); - expect(s).not.toContain("unsafe-inline"); - expect(s).not.toContain("unsafe-eval"); - // Production never allows inline script; development allows only the exact injected helper, by hash. - expect(s).toBe(development ? `script-src 'self' '${inlineScriptHashes(DOC)[0]}'` : "script-src 'self'"); - const asset = await fetch(`http://127.0.0.1:${server.port}/_bun/client/app.js`); - expect(asset.status).toBe(200); - expect(asset.headers.get("content-security-policy")).toBeNull(); - expect(await (await fetch(`http://127.0.0.1:${server.port}/api/x`)).text()).toBe("api"); - } finally { - server.stop(true); - spa.stop(); - } - }); -} - -test("the runner frame is served sandboxed, with no connections and only its own script", async () => { - const db = await connect("sqlite://:memory:"); - await runMigrations(db); - const { call } = await makeApp(db); - const r = await call("GET", RUNNER_FRAME_PATH); - expect(r.status).toBe(200); - const csp = r.headers.get("content-security-policy")!; - expect(csp).toBe(RUNNER_FRAME_CSP); - expect(csp).toContain("connect-src 'none'"); - expect(csp).toContain("sandbox allow-scripts"); - expect(csp).not.toContain("allow-same-origin"); - expect(scriptSrc(csp)).toMatch(/^script-src 'sha256-[A-Za-z0-9+/=]+' blob:$/); - await db.close(); -}); - -test("the SPA proxy ignores HTTP_PROXY/ALL_PROXY and forwards no credentials", () => { - const dead = "http://127.0.0.1:9"; // nothing listens on the discard port - const p = Bun.spawnSync([process.execPath, new URL("./e2e/spa-proxy-check.ts", import.meta.url).pathname], { - env: { ...process.env, HTTP_PROXY: dead, http_proxy: dead, HTTPS_PROXY: dead, https_proxy: dead, ALL_PROXY: dead, all_proxy: dead, NO_PROXY: "corp.example", no_proxy: "" }, - stdout: "pipe", stderr: "pipe", - }); - expect(p.exitCode).toBe(0); - const out = JSON.parse(p.stdout.toString().trim().split("\n").pop()!); - // The environment proxy really is in effect for plain fetches in that process... - expect(out.controlViaEnvProxy).not.toBe("status 200"); - // ...but the SPA still loads through spaWithCsp, with its CSP. - expect(out.status).toBe(200); - expect(out.body).toContain('
      '); - expect(out.csp).toContain("script-src 'self'"); - expect(out.post).toBe(405); - // Assets come through too (with a proxy in the env, Bun's socket fetch would otherwise hit the HTML fallback). - expect(out.asset).toBe("200 text/javascript console.log(1)"); - expect(out.noProxy).toBe("corp.example,localhost,127.0.0.1,::1"); - // The upstream saw only the allowlisted headers: no Cookie, Authorization or X-Forwarded-*. - expect(out.seen).toEqual([{ cookie: null, authorization: null, xff: null, accept: "text/html" }]); -}, 30_000); From 777ae2cc3791e56da85a1aea150a089a674145ca Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 15:02:46 +0100 Subject: [PATCH 58/62] feat: resolve Phase 1 contract requests and compose every feature Contracts and core: - Feature gains auth and stop; FeatureFactory; core composeServer builds the context, runs the factories, then the job queue from the union of their jobs (R8/R14). startServer uses it; main.ts composes learner, import, studio, AI. - AppConfig/loadConfig own the layout and settings: courseDirs include /courses and /worktrees, studioReposDir, cacheDir, importRoots, uploadMaxMb, aiAllowedBaseUrls, transcribeUrl, python. resolveConfig derives the same for tests; cachePaths shares the importer's cache layout. - canView: the global author role views every course (R14); the learner, studio and AI packages all call it. - migrate(db, packages): core first, then packages in name order. - Media: nosniff on every response, a sandbox CSP on svg/html/js; a Range on an empty file is 416. - courseUrl/lessonUrl and the lesson route patterns live in @stc/contracts. Packages: - import, studio and AI read AppConfig instead of process.env; the importer refuses ids used by studio/repos. - learner migrations run through core migrate; library covers fall back to the importer's cached candidate behind a canView-checked route. - studio preview renders through the learner LessonView (anchors honoured, draft URLs); AI drafts go to the studio draft branch for studio courses (as the requesting user) and to /drafts otherwise, and the generator page says which. Web: tsconfig paths for @stc//web browser entries; features.ts imports them by package name; nav.roles hides links by role. Tests: compose, config, canView, media headers, migrations order, covers, preview, nav, SPA bundle without server code, and an end-to-end flow on SQLite and Postgres. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/features.ts | 44 +++++++ apps/server/src/main.ts | 72 ++++------- apps/server/test/e2e.test.ts | 117 ++++++++++++++++++ apps/server/test/pg.ts | 13 ++ apps/server/test/server.test.ts | 6 +- apps/web/src/bundle.test.ts | 25 ++++ apps/web/src/features.ts | 9 +- apps/web/src/main.tsx | 24 +++- apps/web/src/nav.test.ts | 21 ++++ apps/web/src/nav.ts | 15 +++ packages/contracts/src/index.ts | 1 + packages/contracts/src/routes.ts | 14 +++ packages/contracts/src/server.ts | 28 ++++- packages/contracts/src/web.ts | 4 +- packages/contracts/test/types.test.ts | 3 + packages/core/src/app.ts | 7 +- packages/core/src/compose.ts | 75 +++++++++++ packages/core/src/config.ts | 75 ++++++++++- packages/core/src/index.ts | 5 +- packages/core/src/media.ts | 21 +++- packages/core/test/app.test.ts | 8 ++ packages/core/test/compose.test.ts | 51 ++++++++ packages/core/test/config.test.ts | 47 ++++++- packages/core/test/media.test.ts | 33 +++++ packages/course-format/package.json | 4 + packages/db/src/index.ts | 2 +- packages/db/src/migrate.ts | 35 ++++-- packages/db/test/migrate.test.ts | 19 +++ packages/feature-ai/src/feature.ts | 12 +- packages/feature-ai/src/web/GeneratePage.tsx | 6 +- packages/feature-ai/src/web/index.ts | 2 +- packages/feature-ai/src/web/urls.ts | 13 +- packages/feature-ai/test/feature.test.ts | 26 ++-- packages/feature-ai/test/web.test.tsx | 12 +- packages/feature-import/src/context.ts | 16 ++- packages/feature-import/src/index.ts | 17 ++- packages/feature-import/src/jobs/common.ts | 2 +- .../feature-import/src/jobs/transcribe.ts | 2 +- packages/feature-import/src/roots.ts | 12 +- packages/feature-import/src/routes.ts | 4 +- packages/feature-import/src/web.ts | 2 +- packages/feature-import/test/harness.ts | 9 +- .../feature-import/test/packt-video.test.ts | 10 ++ packages/feature-import/test/routes.test.ts | 11 +- packages/feature-learner/package.json | 2 +- packages/feature-learner/src/index.ts | 1 + packages/feature-learner/src/server/access.ts | 4 +- .../feature-learner/src/server/catalog.ts | 5 +- packages/feature-learner/src/server/covers.ts | 47 +++++++ .../feature-learner/src/server/feature.ts | 16 +-- .../feature-learner/src/server/identity.ts | 9 ++ .../feature-learner/src/server/migrate.ts | 33 +---- packages/feature-learner/src/web/api.ts | 3 +- packages/feature-learner/src/web/index.ts | 7 +- .../src/web/lesson/LessonView.tsx | 14 ++- packages/feature-learner/src/web/markdown.ts | 4 +- packages/feature-learner/test/app.ts | 6 +- packages/feature-learner/test/covers.test.ts | 40 ++++++ .../feature-learner/test/visibility.test.ts | 12 +- packages/feature-learner/web/src/index.ts | 4 - .../feature-studio/src/server/draft-writer.ts | 8 +- packages/feature-studio/src/server/feature.ts | 31 +++-- packages/feature-studio/src/server/paths.ts | 14 ++- packages/feature-studio/src/server/store.ts | 12 +- .../src/ui/editors/LessonEditor.tsx | 2 +- .../feature-studio/src/ui/learnerPreview.tsx | 28 +++++ packages/feature-studio/src/ui/preview.tsx | 18 ++- packages/feature-studio/src/web.ts | 2 +- packages/feature-studio/test/app-helpers.ts | 8 +- packages/feature-studio/test/preview.test.tsx | 28 +++++ packages/feature-studio/test/security.test.ts | 7 +- tsconfig.json | 11 +- 72 files changed, 1066 insertions(+), 244 deletions(-) create mode 100644 apps/server/src/features.ts create mode 100644 apps/server/test/e2e.test.ts create mode 100644 apps/server/test/pg.ts create mode 100644 apps/web/src/bundle.test.ts create mode 100644 apps/web/src/nav.test.ts create mode 100644 apps/web/src/nav.ts create mode 100644 packages/contracts/src/routes.ts create mode 100644 packages/core/src/compose.ts create mode 100644 packages/core/test/compose.test.ts create mode 100644 packages/feature-learner/src/server/covers.ts create mode 100644 packages/feature-learner/src/server/identity.ts create mode 100644 packages/feature-learner/test/covers.test.ts delete mode 100644 packages/feature-learner/web/src/index.ts create mode 100644 packages/feature-studio/src/ui/learnerPreview.tsx create mode 100644 packages/feature-studio/test/preview.test.tsx diff --git a/apps/server/src/features.ts b/apps/server/src/features.ts new file mode 100644 index 0000000..10c7028 --- /dev/null +++ b/apps/server/src/features.ts @@ -0,0 +1,44 @@ +import type { FeatureFactory, ServerContext } from "@stc/contracts"; +import type { PackageMigrations } from "@stc/db"; +import { aiFeature, createFsDraftSink, migrationsDirs as aiMigrations, type DraftSink } from "@stc/feature-ai"; +import { importFeature, migrationsDirs as importMigrations } from "@stc/feature-import"; +import { learnerFeature, migrationsDirs as learnerMigrations, userIdentity } from "@stc/feature-learner"; +import { createDraftWriter, migrationsDirs as studioMigrations, studioFeature, type StudioFeature } from "@stc/feature-studio"; + +/** Every package's migration folders, keyed by package name (core runs first, then these in name order). */ +export const packageMigrations: PackageMigrations = { + "feature-ai": aiMigrations, + "feature-import": importMigrations, + "feature-learner": learnerMigrations, + "feature-studio": studioMigrations, +}; + +/** + * Where the AI generator writes drafts: studio-managed courses get a commit on the requesting author's + * `draft/` branch, authored as that user's name and email; overlay and imported courses (which the studio + * does not manage) fall back to files under `/drafts///`. The generator page says which. + */ +export function aiDraftSink(ctx: ServerContext, studio: Pick): DraftSink { + const writer = createDraftWriter(ctx, { identity: (userId) => userIdentity(ctx.db, userId) }); + const files = createFsDraftSink(ctx.config.dataDir); + return { + writeDraftFiles: (courseId, authorId, list, message) => + (studio.store.isManaged(courseId) ? writer : files).writeDraftFiles(courseId, authorId, list, message), + }; +} + +export interface FeatureSetOptions { + /** Studio published-worktree polling (default on). */ + studioWatch?: boolean; +} + +/** The product: learner (auth, library, reader), importer, studio and AI, in that order. */ +export function productFeatures(opts: FeatureSetOptions = {}): FeatureFactory[] { + let studio: StudioFeature | null = null; + return [ + learnerFeature, + (ctx) => importFeature(ctx), + (ctx) => (studio = studioFeature(ctx, { watch: opts.studioWatch ?? true })), + (ctx) => aiFeature(ctx, { draftSink: aiDraftSink(ctx, studio!) }), + ]; +} diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index 201fc97..6a46694 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -1,62 +1,42 @@ -import type { AppConfig, AuthService, Feature, JobHandler, JobQueue, ServerContext } from "@stc/contracts"; -import { createApp, createJobQueue, loadConfig } from "@stc/core"; -import { createCourseRepository } from "@stc/course-format"; -import { connect, migrate } from "@stc/db"; -import { importFeature } from "@stc/feature-import"; -import { learnerFeature } from "@stc/feature-learner"; -import { resolve } from "node:path"; +import type { AppConfig, Feature, FeatureFactory } from "@stc/contracts"; +import { composeServer, loadConfig, type ComposeOptions } from "@stc/core"; import indexHtml from "../../web/index.html"; +import { packageMigrations, productFeatures } from "./features"; -// Used until a feature supplies an `auth` service (the learner package does). -const anonymousAuth: AuthService = { userFromRequest: async () => null }; +export interface StartOptions extends ComposeOptions {} -/** A feature, or a factory that needs the server context. A feature may carry the `AuthService`. */ -export type FeatureEntry = (Feature & { auth?: AuthService }) | ((ctx: ServerContext) => Feature & { auth?: AuthService }); - -export async function startServer( - config: AppConfig, - features: FeatureEntry[], - _webFeatures?: unknown, -): Promise<{ server: Bun.Server; stop(): Promise }> { - const db = await connect(config.databaseUrl); - await migrate(db); - const courses = await createCourseRepository(config.courseDirs); - const handlers: Record = {}; - // Factories get the context before the job queue exists; ctx.jobs is set right after they run. - const ctx: ServerContext = { config, db, courses, jobs: null as unknown as JobQueue, auth: anonymousAuth }; - const built = features.map((f) => (typeof f === "function" ? f(ctx) : f)); - for (const f of built) { - Object.assign(handlers, f.jobs); - if (f.auth) ctx.auth = f.auth; +/** + * Composes the features (core `composeServer`) and serves them with the SPA. Bun routes win over `fetch`; + * "/api/*" falls through to the composed app, so unknown API paths are 404 rather than the SPA. + */ +export async function startServer(config: AppConfig, features: (Feature | FeatureFactory)[], opts: StartOptions = {}) { + const composed = await composeServer(config, features, opts); + const { app } = composed; + let server: Bun.Server; + try { + server = Bun.serve({ + port: config.port, + routes: { ...app.routes, "/api/*": (req) => app.fetch(req), "/*": indexHtml }, + fetch: (req) => app.fetch(req), + // Production (NODE_ENV=production, as in the Docker image) bundles the SPA once, minified, without HMR. + development: process.env.NODE_ENV !== "production", + }); + } catch (e) { + await composed.stop(); + throw e; } - const jobs = createJobQueue(db, handlers); - ctx.jobs = jobs; - const app = createApp(ctx, built); - jobs.start(); - const server = Bun.serve({ - port: config.port, - // Bun routes win over fetch; "/api/*" falls through to the composed app (404 for unknown API paths). - routes: { ...app.routes, "/api/*": (req) => app.fetch(req), "/*": indexHtml }, - fetch: (req) => app.fetch(req), - }); return { server, + ctx: composed.ctx, async stop() { await server.stop(true); - await jobs.stop(); - await db.close(); + await composed.stop(); }, }; } if (import.meta.main) { const config = loadConfig(); - // Contract request (import.md): core config should include /courses itself. - const generated = resolve(config.dataDir, "courses"); - if (!config.courseDirs.some((c) => resolve(c) === generated)) config.courseDirs.push(generated); - const { server } = await startServer(config, [ - learnerFeature, - (ctx) => importFeature(ctx), - ]); + const { server } = await startServer(config, productFeatures(), { migrations: packageMigrations }); console.log(`STCourses listening on ${server.url}`); } diff --git a/apps/server/test/e2e.test.ts b/apps/server/test/e2e.test.ts new file mode 100644 index 0000000..2952b6a --- /dev/null +++ b/apps/server/test/e2e.test.ts @@ -0,0 +1,117 @@ +// End-to-end (plan Task I3): the real server with every feature, against the fixtures, on SQLite and Postgres. +import { describe, expect, test } from "bun:test"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { loadConfig } from "@stc/core"; +import { studioFeature } from "@stc/feature-studio"; +import { createBrowserRunner } from "../../../packages/feature-learner/src/web/runner/runner"; +import { aiDraftSink, packageMigrations, productFeatures } from "../src/features"; +import { startServer } from "../src/main"; +import { PG, pgSchema } from "./pg"; + +const FIXTURES = resolve(import.meta.dir, "../../../fixtures/courses"); +const ORIGIN = "http://localhost:3000"; + +const cookieFrom = (r: Response) => r.headers.getSetCookie().map((c) => c.split(";")[0]).filter((c) => !c.endsWith("=")).join("; "); + +async function run(databaseUrl: string) { + const dataDir = mkdtempSync(join(tmpdir(), "stc-e2e-")); + const config = loadConfig({ + STC_PORT: "0", STC_DATABASE_URL: databaseUrl, STC_SECRET: "e2e-secret-0123456789abcdef0123456789", + STC_DATA_DIR: dataDir, STC_COURSE_DIRS: FIXTURES, STC_PUBLIC_URL: ORIGIN, + }); + const s = await startServer(config, productFeatures({ studioWatch: false }), { migrations: packageMigrations }); + const base = s.server.url.toString().replace(/\/$/, ""); + const call = async (method: string, path: string, opts: { cookie?: string; body?: unknown } = {}) => { + const headers: Record = { origin: ORIGIN }; + if (opts.cookie) headers.cookie = opts.cookie; + if (opts.body !== undefined) headers["content-type"] = "application/json"; + return fetch(base + path, { method, headers, body: opts.body === undefined ? undefined : JSON.stringify(opts.body) }); + }; + const signUp = async (email: string, name: string) => { + const r = await call("POST", "/api/auth/sign-up/email", { body: { email, name, password: "correct horse battery" } }); + expect(r.status).toBe(200); + return cookieFrom(r); + }; + try { + // 1. the server is up with every feature and the fixture course + expect(await (await call("GET", "/api/health")).json()).toEqual({ ok: true, courses: 1, db: databaseUrl.startsWith("postgres") ? "postgres" : "sqlite" }); + expect((await call("GET", "/c/sample")).headers.get("content-type")).toContain("text/html"); + + // 2. sign up: the first account is the owner + const owner = await signUp("ada@example.org", "Ada Lovelace"); + const me = (await (await call("GET", "/api/me", { cookie: owner })).json()) as { user: { id: string; roles: string[] } }; + expect(me.user.roles).toContain("owner"); + + // 3. mark a lesson done + expect((await call("PUT", "/api/courses/sample/progress/m1-intro", { cookie: owner, body: { state: "done" } })).status).toBe(200); + + // 4. a slide note on the video lesson's deck + const note = await call("POST", "/api/courses/sample/notes", { cookie: owner, body: { lessonId: "m1-video", anchorKind: "slide", anchorValue: 2, deckId: "d1", body: "Slide two matters" } }); + expect(note.status).toBe(201); + + // 5. run the sample exercise with the learner's runner, then save the passing attempt + const lesson = (await (await call("GET", "/api/courses/sample/lessons/m2-ex-add", { cookie: owner })).json()) as { browser: { starter: string; tests: string } }; + const runner = createBrowserRunner("js"); + const failing = await runner.run(lesson.browser.starter, lesson.browser.tests, 5000); + expect(failing.some((t) => !t.passed)).toBe(true); + const code = lesson.browser.starter.replace("a - b", "a + b"); + const results = await runner.run(code, lesson.browser.tests, 5000); + expect(results.length).toBeGreaterThan(0); + expect(results.every((t) => t.passed)).toBe(true); + expect((await call("PUT", "/api/courses/sample/lessons/m2-ex-add/attempt", { cookie: owner, body: { code, passed: true, result: results, track: "js" } })).status).toBe(200); + + // 6. read progress and notes back + const progress = (await (await call("GET", "/api/courses/sample/progress", { cookie: owner })).json()) as Record; + expect(progress["m1-intro"]?.state).toBe("done"); + expect(progress["m2-ex-add"]?.state).toBe("done"); + const notes = (await (await call("GET", "/api/courses/sample/notes", { cookie: owner })).json()) as { body: string; anchorKind: string; deckId: string }[]; + expect(notes.map((n) => [n.body, n.anchorKind, n.deckId])).toEqual([["Slide two matters", "slide", "d1"]]); + + // 7. a second account is a plain learner: own data only, and the owner/author features refuse it + const lea = await signUp("lea@example.org", "Lea"); + expect(await (await call("GET", "/api/courses/sample/progress", { cookie: lea })).json()).toEqual({}); + expect((await call("GET", "/api/import/info", { cookie: lea })).status).toBe(403); + expect((await call("GET", "/api/studio/courses", { cookie: lea })).status).toBe(403); + expect((await call("GET", "/api/ai/admin/config", { cookie: lea })).status).toBe(403); + expect((await call("GET", "/api/import/info", { cookie: owner })).status).toBe(200); + expect((await call("GET", "/api/studio/courses", { cookie: owner })).status).toBe(200); + expect((await call("GET", "/api/ai/status", { cookie: owner })).status).toBe(200); + + // 8. media: Range, nosniff + const clip = await call("GET", "/api/media/sample/media/clip.mp4", { cookie: lea }); + expect(clip.headers.get("x-content-type-options")).toBe("nosniff"); + + // 9. the AI draft sink: studio-managed course -> the author's draft branch, as that user; other courses -> files + const studio = studioFeature(s.ctx, { watch: false }); + try { + await studio.store.createCourse({ id: "gen-target", title: "Generated" }, { name: "Ada Lovelace", email: "ada@example.org" }); + const sink = aiDraftSink(s.ctx, studio); + const onStudio = await sink.writeDraftFiles("gen-target", me.user.id, [{ path: "lessons/q.md", content: "# Q\n" }], "AI draft"); + expect(onStudio.branch).toMatch(/^draft\//); + const repo = join(config.studioReposDir, "gen-target.git"); + const author = (await Bun.$`git --git-dir=${repo} log -1 ${"--format=%an <%ae>"} ${onStudio.branch!}`.text()).trim(); + expect(author).toBe("Ada Lovelace "); + const onFiles = await sink.writeDraftFiles("sample", me.user.id, [{ path: "q.md", content: "# Q\n" }], "AI draft"); + expect(onFiles.branch).toBeUndefined(); + expect(onFiles.location!.startsWith(resolve(dataDir, "drafts", "sample"))).toBe(true); + expect(readFileSync(join(onFiles.location!, "q.md"), "utf8")).toBe("# Q\n"); + } finally { + studio.stop(); + } + } finally { + await s.stop(); + } +} + +describe("end to end", () => { + test("sign up, mark done, slide note, exercise, progress [sqlite]", () => run("sqlite://:memory:"), 60_000); + const pg = PG; + if (pg) { + test("sign up, mark done, slide note, exercise, progress [postgres]", async () => { + const { url, drop } = await pgSchema(pg); + try { await run(url); } finally { await drop(); } + }, 60_000); + } +}); diff --git a/apps/server/test/pg.ts b/apps/server/test/pg.ts new file mode 100644 index 0000000..355a8a9 --- /dev/null +++ b/apps/server/test/pg.ts @@ -0,0 +1,13 @@ +import { connect } from "@stc/db"; + +export const PG = process.env.STC_TEST_PG_URL; + +/** A fresh Postgres schema (search_path in the URL), dropped by `drop()`. Same isolation as packages/db's forEachDb. */ +export async function pgSchema(base: string): Promise<{ url: string; drop(): Promise }> { + const schema = `t_${Bun.randomUUIDv7().replaceAll("-", "")}`; + const admin = await connect(base); + await admin.sql.unsafe(`CREATE SCHEMA ${schema}`); + const u = new URL(base); + u.searchParams.set("options", `-c search_path=${schema}`); + return { url: u.toString(), drop: async () => { await admin.sql.unsafe(`DROP SCHEMA ${schema} CASCADE`); await admin.close(); } }; +} diff --git a/apps/server/test/server.test.ts b/apps/server/test/server.test.ts index 8f17abb..722fd2b 100644 --- a/apps/server/test/server.test.ts +++ b/apps/server/test/server.test.ts @@ -1,5 +1,7 @@ import { afterAll, beforeAll, expect, test } from "bun:test"; -import { resolve } from "node:path"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; import { loadConfig } from "@stc/core"; import { startServer } from "../src/main"; @@ -7,7 +9,7 @@ let s: Awaited>; let base: string; beforeAll(async () => { const config = loadConfig({ - STC_PORT: "0", STC_DATABASE_URL: "sqlite://:memory:", STC_SECRET: "t", + STC_PORT: "0", STC_DATABASE_URL: "sqlite://:memory:", STC_SECRET: "t", STC_DATA_DIR: mkdtempSync(join(tmpdir(), "stc-server-")), STC_COURSE_DIRS: resolve(import.meta.dir, "../../../fixtures/courses"), }); s = await startServer(config, []); diff --git a/apps/web/src/bundle.test.ts b/apps/web/src/bundle.test.ts new file mode 100644 index 0000000..dc6efe6 --- /dev/null +++ b/apps/web/src/bundle.test.ts @@ -0,0 +1,25 @@ +import { expect, test } from "bun:test"; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +// A subprocess, because HTML bundling inside a full `bun test` run is unreliable (see the learner's csp.test). +test("the SPA bundles for the browser with no server code in it", async () => { + const out = mkdtempSync(join(tmpdir(), "stc-spa-")); + try { + const root = resolve(import.meta.dir, "../../.."); + const p = Bun.spawn(["bun", "build", "apps/web/index.html", "--outdir", out, "--target", "browser"], { cwd: root, stdout: "pipe", stderr: "pipe" }); + const code = await p.exited; + expect(code === 0 ? "" : await new Response(p.stderr).text()).toBe(""); + const js = readdirSync(out).filter((f) => f.endsWith(".js")).map((f) => readFileSync(join(out, f), "utf8")).join("\n"); + expect(js.length).toBeGreaterThan(1000); + // Server-only modules of the feature packages and their dependencies. + for (const marker of ["better-auth", "X509Certificate", "createStudioStore", "importFeature(", "aiFeature(", "createJobQueue", "bun:sqlite"]) { + expect([marker, js.includes(marker)]).toEqual([marker, false]); + } + // Every feature's browser half is present. + for (const page of ["/admin/import", "/studio/:courseId", "/admin/ai", "/c/:courseId/:lessonId"]) expect(js).toContain(page); + } finally { + rmSync(out, { recursive: true, force: true }); + } +}, 120_000); diff --git a/apps/web/src/features.ts b/apps/web/src/features.ts index 946564e..076a7bd 100644 --- a/apps/web/src/features.ts +++ b/apps/web/src/features.ts @@ -1,8 +1,9 @@ import type { WebFeature } from "@stc/contracts"; +// Browser-only entries ("@stc//web", mapped in tsconfig paths and package.json exports). Never import a +// package's server index here: it pulls Better Auth, git and node:fs into the bundle. import { learnerWeb } from "@stc/feature-learner/web"; -import { importWeb } from "../../../packages/feature-import/src/web"; -import { studioWeb } from "../../../packages/feature-studio/src/web"; -import { aiWeb } from "../../../packages/feature-ai/src/web"; +import { importWeb } from "@stc/feature-import/web"; +import { studioWeb } from "@stc/feature-studio/web"; +import { aiWeb } from "@stc/feature-ai/web"; -// Phase 1 packages append one import line each and push into this list. export const webFeatures: WebFeature[] = [learnerWeb, importWeb, studioWeb, aiWeb]; diff --git a/apps/web/src/main.tsx b/apps/web/src/main.tsx index a9afe98..a5a8c9f 100644 --- a/apps/web/src/main.tsx +++ b/apps/web/src/main.tsx @@ -1,15 +1,33 @@ import { createRoot } from "react-dom/client"; -import { lazy, Suspense } from "react"; -import { createBrowserRouter, Link, Outlet, RouterProvider, type RouteObject } from "react-router-dom"; +import { lazy, Suspense, useEffect, useState } from "react"; +import { createBrowserRouter, Link, Outlet, RouterProvider, useLocation, type RouteObject } from "react-router-dom"; +import type { AuthUser } from "@stc/contracts"; import { AppShell } from "@stc/ui"; import { webFeatures } from "./features"; +import { visibleNav } from "./nav"; function Home() { return

      No features installed yet

      ; } +/** The signed-in user (GET /api/me), refreshed on navigation so sign-in and sign-out update the nav. */ +function useSessionUser(): AuthUser | null { + const { pathname } = useLocation(); + const [user, setUser] = useState(null); + useEffect(() => { + let live = true; + fetch("/api/me", { credentials: "same-origin" }) + .then((r) => (r.ok ? r.json() : { user: null })) + .then((b: { user: AuthUser | null }) => { if (live) setUser(b.user ?? null); }) + .catch(() => { if (live) setUser(null); }); + return () => { live = false; }; + }, [pathname]); + return user; +} + function Layout() { - const nav = webFeatures.flatMap((f) => f.routes.filter((r) => r.nav?.area === "header")); + // Links the user cannot use are hidden; the pages and APIs still enforce access themselves. + const nav = visibleNav(webFeatures, useSessionUser()); const header = ( <> STCourses diff --git a/apps/web/src/nav.test.ts b/apps/web/src/nav.test.ts new file mode 100644 index 0000000..3e3a19f --- /dev/null +++ b/apps/web/src/nav.test.ts @@ -0,0 +1,21 @@ +import { expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { webFeatures } from "./features"; +import { visibleNav } from "./nav"; + +const labels = (roles: ("owner" | "author" | "learner")[] | null) => visibleNav(webFeatures, roles ? { roles } : null).map((r) => r.nav!.label); + +test("nav items are limited by role", () => { + expect(labels(null)).toEqual([]); + expect(labels(["learner"])).toEqual([]); + expect(labels(["author"])).toEqual(["Studio"]); + expect(labels(["owner"])).toEqual(["Studio", "Import", "AI"]); +}); + +test("features.ts imports only browser entries, by package name", () => { + const src = readFileSync(join(import.meta.dir, "features.ts"), "utf8"); + const specs = [...src.matchAll(/from\s+"([^"]+)"/g)].map((m) => m[1]!); + for (const s of specs) expect(s === "@stc/contracts" || /^@stc\/feature-[a-z]+\/web$/.test(s)).toBe(true); + expect(specs.filter((s) => s.endsWith("/web")).length).toBe(4); +}); diff --git a/apps/web/src/nav.ts b/apps/web/src/nav.ts new file mode 100644 index 0000000..3364a07 --- /dev/null +++ b/apps/web/src/nav.ts @@ -0,0 +1,15 @@ +import type { AuthUser, Role, WebFeature, WebRoute } from "@stc/contracts"; + +/** Admin-area links default to the owner; header links default to everyone. `nav.roles` overrides both. */ +const DEFAULT_ROLES: Record<"header" | "admin", Role[] | undefined> = { header: undefined, admin: ["owner"] }; + +/** The nav links this user may use, header links first, then admin links. */ +export function visibleNav(features: WebFeature[], user: Pick | null): WebRoute[] { + const all = features.flatMap((f) => f.routes.filter((r) => r.nav)); + const allowed = (r: WebRoute) => { + const roles = r.nav!.roles ?? DEFAULT_ROLES[r.nav!.area]; + if (!roles) return true; + return !!user && roles.some((x) => user.roles.includes(x)); + }; + return [...all.filter((r) => r.nav!.area === "header" && allowed(r)), ...all.filter((r) => r.nav!.area === "admin" && allowed(r))]; +} diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 859b47a..9ccab89 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -3,3 +3,4 @@ export * from "./exercise"; export * from "./ai"; export * from "./import"; export * from "./web"; +export * from "./routes"; diff --git a/packages/contracts/src/routes.ts b/packages/contracts/src/routes.ts new file mode 100644 index 0000000..e99b199 --- /dev/null +++ b/packages/contracts/src/routes.ts @@ -0,0 +1,14 @@ +// The learner's public page routes. The learner package registers these patterns; every other package (AI +// citations, studio links) builds links with the helpers, so a link always matches a real route. +// Pure functions with no dependencies, the only runtime code in @stc/contracts. + +export const COURSE_ROUTE = "/c/:courseId"; +export const LESSON_ROUTE = "/c/:courseId/:lessonId"; + +export function courseUrl(courseId: string): string { + return `/c/${encodeURIComponent(courseId)}`; +} + +export function lessonUrl(courseId: string, lessonId: string): string { + return `${courseUrl(courseId)}/${encodeURIComponent(lessonId)}`; +} diff --git a/packages/contracts/src/server.ts b/packages/contracts/src/server.ts index 1ac169e..cf8377c 100644 --- a/packages/contracts/src/server.ts +++ b/packages/contracts/src/server.ts @@ -7,7 +7,12 @@ export interface Feature { name: string; // "learner" | "import" | "studio" | "ai" | "core" routes: Record>>; // paths start with /api/ jobs?: Record; // job kind -> handler + auth?: AuthService; // replaces the anonymous AuthService (the learner package supplies it) + stop?(): void | Promise; // called when the server stops (timers, watchers) } +/** Features are built after the ServerContext exists. The job queue is built from the union of their `jobs`, so a + * factory must not use `ctx.jobs` synchronously (only while a request or job runs). */ +export type FeatureFactory = (ctx: ServerContext) => Feature; export interface JobContext { reportProgress(fraction: number, message?: string): Promise; signal: AbortSignal } export type JobHandler = (payload: unknown, jc: JobContext) => Promise; export type JobStatus = "queued" | "running" | "done" | "failed"; @@ -20,8 +25,27 @@ export interface ServerContext { jobs: JobQueue; auth: AuthService; } export interface AppConfig { - port: number; databaseUrl: string; dataDir: string; // dataDir holds cache/, media/, worktrees/ - courseDirs: string[]; // folders scanned for stcourse.yaml (course repos and overlays/) + port: number; databaseUrl: string; dataDir: string; // dataDir holds cache/, courses/, studio/, worktrees/ + /** Folders scanned for stcourse.yaml: STC_COURSE_DIRS, plus importedCoursesDir and studioWorktreesDir (deduplicated). */ + courseDirs: string[]; secret: string; // STC_SECRET, used to encrypt BYOK keys publicUrl: string; + /** `/courses`: the importer's plain course folders. */ + importedCoursesDir: string; + /** `/studio/repos`: studio bare repos (`.git`), never scanned. */ + studioReposDir: string; + /** `/worktrees`: studio published worktrees (`/published`); drafts sit one level deeper and are never scanned. */ + studioWorktreesDir: string; + /** `/cache`: derived files (decks/, covers/, transcripts/). */ + cacheDir: string; + /** STC_IMPORT_ROOTS (":"-separated), default `/import-inbox`: the only folders the importer reads from. */ + importRoots: string[]; + /** STC_UPLOAD_MAX_MB, default 100. */ + uploadMaxMb: number; + /** STC_AI_ALLOWED_BASE_URLS (comma-separated origins) that learners' own AI keys may use besides the official endpoints. */ + aiAllowedBaseUrls: string[]; + /** STC_TRANSCRIBE_URL: Whisper-compatible endpoint; null disables transcription. */ + transcribeUrl: string | null; + /** STC_PYTHON, default python3: Python with PyMuPDF, used for PDF page images and text. */ + python: string; } diff --git a/packages/contracts/src/web.ts b/packages/contracts/src/web.ts index 2b7717c..2f95a0c 100644 --- a/packages/contracts/src/web.ts +++ b/packages/contracts/src/web.ts @@ -1,4 +1,6 @@ import type React from "react"; +import type { Role } from "./server"; -export interface WebRoute { path: string; element: () => Promise<{ default: React.ComponentType }>; nav?: { label: string; area: "header" | "admin" } } +/** `nav.roles`: the shell shows the link only to signed-in users holding one of these roles (absent: everyone). */ +export interface WebRoute { path: string; element: () => Promise<{ default: React.ComponentType }>; nav?: { label: string; area: "header" | "admin"; roles?: Role[] } } export interface WebFeature { name: string; routes: WebRoute[] } diff --git a/packages/contracts/test/types.test.ts b/packages/contracts/test/types.test.ts index ebf1370..e5cfbf0 100644 --- a/packages/contracts/test/types.test.ts +++ b/packages/contracts/test/types.test.ts @@ -25,6 +25,9 @@ test("contract interfaces accept well-formed literals", async () => { const config: AppConfig = { port: 3000, databaseUrl: "sqlite://./data/stcourses.db", dataDir: "./data", courseDirs: ["./overlays"], secret: "s", publicUrl: "http://localhost:3000", + importedCoursesDir: "./data/courses", studioReposDir: "./data/studio/repos", studioWorktreesDir: "./data/worktrees", + cacheDir: "./data/cache", importRoots: ["./data/import-inbox"], uploadMaxMb: 100, aiAllowedBaseUrls: [], + transcribeUrl: null, python: "python3", }; const ctx: Pick = { config, jobs: queue, auth }; diff --git a/packages/core/src/app.ts b/packages/core/src/app.ts index fc7554f..931328c 100644 --- a/packages/core/src/app.ts +++ b/packages/core/src/app.ts @@ -6,12 +6,17 @@ import { serveMedia } from "./media"; type Method = "GET" | "POST" | "PUT" | "PATCH" | "DELETE"; export type Routes = Record Response | Promise>>>; +/** + * The one visibility rule (ruling R14). Every package calls this; none keeps a local variant. + * public: everyone; signed-in: any signed-in user; restricted: the owner, the global `author` role, a per-course + * author (`isAuthor`, a course_authors row), or an ACL grant to the user or one of their groups. + */ export function canView(user: AuthUser | null, course: LoadedCourse, acl: { principal_type: string; principal_id: string }[], isAuthor = false): boolean { const vis = course.manifest?.visibility ?? "restricted"; if (vis === "public") return true; if (!user) return false; if (vis === "signed-in") return true; - if (user.roles.includes("owner") || isAuthor) return true; + if (user.roles.includes("owner") || user.roles.includes("author") || isAuthor) return true; return acl.some((a) => (a.principal_type === "user" && a.principal_id === user.id) || (a.principal_type === "group" && user.groups.includes(a.principal_id))); } diff --git a/packages/core/src/compose.ts b/packages/core/src/compose.ts new file mode 100644 index 0000000..4b68181 --- /dev/null +++ b/packages/core/src/compose.ts @@ -0,0 +1,75 @@ +import type { AppConfig, AuthService, Feature, FeatureFactory, JobHandler, JobQueue, ServerContext } from "@stc/contracts"; +import { createCourseRepository } from "@stc/course-format"; +import { connect, migrate, type PackageMigrations } from "@stc/db"; +import { createApp } from "./app"; +import { createJobQueue } from "./jobs"; + +/** Used until a feature supplies an `auth` service (the learner package does). */ +const anonymousAuth: AuthService = { userFromRequest: async () => null }; + +export interface ComposeOptions { + /** Each package's migration folders, keyed by package name; run after core's, in name order. */ + migrations?: PackageMigrations; + /** Start polling the job queue (default true). */ + startJobs?: boolean; +} + +export interface ComposedServer { + ctx: ServerContext; + features: Feature[]; + app: ReturnType; + stop(): Promise; +} + +/** + * Builds the server context and the features (rulings R8/R14). Order: + * 1. connect and migrate (core, then packages); + * 2. load courses from `config.courseDirs`; + * 3. run each factory with the context. `ctx.jobs` already exists as a handle, but it forwards to the real queue, + * which is built in step 4 from the union of the features' `jobs`, so factories may keep it but must not + * enqueue while they run; + * 4. a feature carrying `auth` replaces the anonymous AuthService (at most one may). + */ +export async function composeServer(config: AppConfig, entries: (Feature | FeatureFactory)[], opts: ComposeOptions = {}): Promise { + const db = await connect(config.databaseUrl); + try { + await migrate(db, opts.migrations ?? {}); + const courses = await createCourseRepository(config.courseDirs); + let queue: (JobQueue & { start(): void; stop(): Promise }) | null = null; + const notYet = (): never => { throw new Error("the job queue is not ready while features are being built"); }; + const jobs: JobQueue = { + enqueue: (kind, payload) => (queue ?? notYet()).enqueue(kind, payload), + get: (id) => (queue ?? notYet()).get(id), + list: (kind) => (queue ?? notYet()).list(kind), + }; + const ctx: ServerContext = { config, db, courses, jobs, auth: anonymousAuth }; + const features = entries.map((f) => (typeof f === "function" ? f(ctx) : f)); + const handlers: Record = {}; + let authFrom: string | null = null; + for (const f of features) { + for (const [kind, h] of Object.entries(f.jobs ?? {})) { + if (handlers[kind]) throw new Error(`Duplicate job kind "${kind}" (feature "${f.name}")`); + handlers[kind] = h; + } + if (f.auth) { + if (authFrom) throw new Error(`Both "${authFrom}" and "${f.name}" supply an AuthService`); + authFrom = f.name; + ctx.auth = f.auth; + } + } + queue = createJobQueue(db, handlers); + const app = createApp(ctx, features); + if (opts.startJobs !== false) queue.start(); + return { + ctx, features, app, + async stop() { + for (const f of features) { try { await f.stop?.(); } catch (e) { console.error(`[stc] stopping ${f.name}`, e); } } + await queue!.stop(); + await db.close(); + }, + }; + } catch (e) { + await db.close().catch(() => {}); + throw e; + } +} diff --git a/packages/core/src/config.ts b/packages/core/src/config.ts index 0359442..aebf8b9 100644 --- a/packages/core/src/config.ts +++ b/packages/core/src/config.ts @@ -1,7 +1,48 @@ +import { resolve } from "node:path"; import type { AppConfig } from "@stc/contracts"; const DEV_SECRET = "stc-dev-secret-change-me"; +/** The settings a deployment chooses; everything else in AppConfig is derived by `resolveConfig`. */ +export type ConfigInput = Pick & Partial>; + +// Path lists are split on ":" only (paths may contain spaces); URL lists are trimmed. +const paths = (raw: string | undefined) => (raw ?? "").split(":").filter(Boolean); +const urls = (raw: string | undefined) => (raw ?? "").split(",").map((s) => s.trim()).filter(Boolean); + +/** + * Fills in the derived parts of the configuration. Course discovery always covers the importer's folders and the + * studio's published worktrees, after the configured dirs, without duplicates (compared as absolute paths). + * Tests and dev entries use this to build an AppConfig the same way `loadConfig` does. + */ +export function resolveConfig(input: ConfigInput): AppConfig { + const dataDir = input.dataDir; + const importedCoursesDir = input.importedCoursesDir ?? resolve(dataDir, "courses"); + const studioWorktreesDir = input.studioWorktreesDir ?? resolve(dataDir, "worktrees"); + const courseDirs: string[] = []; + for (const d of [...(input.courseDirs ?? []), importedCoursesDir, studioWorktreesDir]) { + if (!courseDirs.some((c) => resolve(c) === resolve(d))) courseDirs.push(d); + } + const port = input.port ?? 3000; + return { + port, + databaseUrl: input.databaseUrl ?? "sqlite://./data/stcourses.db", + dataDir, + courseDirs, + secret: input.secret ?? DEV_SECRET, + publicUrl: input.publicUrl ?? `http://localhost:${port}`, + importedCoursesDir, + studioReposDir: input.studioReposDir ?? resolve(dataDir, "studio", "repos"), + studioWorktreesDir, + cacheDir: input.cacheDir ?? resolve(dataDir, "cache"), + importRoots: input.importRoots ?? [resolve(dataDir, "import-inbox")], + uploadMaxMb: input.uploadMaxMb ?? 100, + aiAllowedBaseUrls: input.aiAllowedBaseUrls ?? [], + transcribeUrl: input.transcribeUrl ?? null, + python: input.python ?? "python3", + }; +} + export function loadConfig(env: Record = process.env): AppConfig { const port = env.STC_PORT !== undefined && env.STC_PORT !== "" ? Number(env.STC_PORT) : 3000; if (!Number.isInteger(port) || port < 0) throw new Error(`Invalid STC_PORT: ${env.STC_PORT}`); @@ -11,12 +52,40 @@ export function loadConfig(env: Record = process.env console.warn("[stc] STC_SECRET is not set; using an insecure development default"); secret = DEV_SECRET; } - return { + let uploadMaxMb = 100; + if (env.STC_UPLOAD_MAX_MB) { + uploadMaxMb = Number(env.STC_UPLOAD_MAX_MB); + if (!Number.isFinite(uploadMaxMb) || uploadMaxMb <= 0) throw new Error(`Invalid STC_UPLOAD_MAX_MB: ${env.STC_UPLOAD_MAX_MB}`); + } + const dataDir = env.STC_DATA_DIR || "./data"; + const roots = paths(env.STC_IMPORT_ROOTS); + return resolveConfig({ port, databaseUrl: env.STC_DATABASE_URL || "sqlite://./data/stcourses.db", - dataDir: env.STC_DATA_DIR || "./data", - courseDirs: (env.STC_COURSE_DIRS || "./overlays:./courses").split(":").filter(Boolean), + dataDir, + courseDirs: paths(env.STC_COURSE_DIRS || "./overlays:./courses"), secret, publicUrl: env.STC_PUBLIC_URL || `http://localhost:${port}`, + importRoots: roots.length ? roots.map((r) => resolve(r)) : undefined, + uploadMaxMb, + aiAllowedBaseUrls: urls(env.STC_AI_ALLOWED_BASE_URLS), + transcribeUrl: env.STC_TRANSCRIBE_URL || null, + python: env.STC_PYTHON || "python3", + }); +} + +/** The importer's derived-file layout under `cacheDir` (decks, cover candidates, transcripts), shared by every package. */ +export function cachePaths(config: Pick) { + const root = config.cacheDir; + const seg = (s: string) => { + if (!s || s === "." || s === ".." || s.includes("/") || s.includes("\\") || s.includes("\0")) throw new Error(`invalid cache path segment "${s}"`); + return s; + }; + return { + root, + deckDir: (courseId: string, deckId: string) => resolve(root, "decks", seg(courseId), seg(deckId)), + coversDir: (courseId: string) => resolve(root, "covers", seg(courseId)), + transcriptFile: (courseId: string, lessonId: string) => resolve(root, "transcripts", seg(courseId), `${seg(lessonId)}.json`), + transcriptsDir: (courseId: string) => resolve(root, "transcripts", seg(courseId)), }; } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index d955c36..11619c1 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,4 +1,5 @@ -export { loadConfig } from "./config"; +export { loadConfig, resolveConfig, cachePaths, type ConfigInput } from "./config"; export { createJobQueue } from "./jobs"; -export { serveMedia, contentType, contentDisposition } from "./media"; +export { serveMedia, contentType, contentDisposition, mediaSecurityHeaders, MEDIA_SANDBOX_CSP } from "./media"; export { createApp, canView, type Routes } from "./app"; +export { composeServer, type ComposeOptions, type ComposedServer } from "./compose"; diff --git a/packages/core/src/media.ts b/packages/core/src/media.ts index 941c1ed..c1a5ec3 100644 --- a/packages/core/src/media.ts +++ b/packages/core/src/media.ts @@ -1,9 +1,23 @@ const TYPES: Record = { mp4: "video/mp4", webm: "video/webm", pdf: "application/pdf", png: "image/png", jpg: "image/jpeg", jpeg: "image/jpeg", svg: "image/svg+xml", md: "text/markdown; charset=utf-8", txt: "text/plain; charset=utf-8", zip: "application/zip", - js: "text/javascript; charset=utf-8", ts: "text/plain; charset=utf-8", + js: "text/javascript; charset=utf-8", mjs: "text/javascript; charset=utf-8", ts: "text/plain; charset=utf-8", + html: "text/html; charset=utf-8", htm: "text/html; charset=utf-8", xhtml: "application/xhtml+xml", xml: "application/xml", }; +/** Types a browser could run script from. Course media is author content on the app origin, so these get a sandbox. */ +const ACTIVE = new Set(["svg", "html", "htm", "xhtml", "xml", "js", "mjs"]); +/** Opaque origin, no script, no plugins: an svg still renders as an image, an html page as inert markup. */ +export const MEDIA_SANDBOX_CSP = "sandbox; default-src 'none'; img-src 'self' data:; style-src 'unsafe-inline'; media-src 'self'"; + +/** Headers every media response carries; plus a sandbox CSP for script-capable types. */ +export function mediaSecurityHeaders(path: string): Record { + const ext = path.slice(path.lastIndexOf(".") + 1).toLowerCase(); + const h: Record = { "X-Content-Type-Options": "nosniff" }; + if (ACTIVE.has(ext)) h["Content-Security-Policy"] = MEDIA_SANDBOX_CSP; + return h; +} + export function contentType(path: string): string { const ext = path.slice(path.lastIndexOf(".") + 1).toLowerCase(); return TYPES[ext] ?? "application/octet-stream"; @@ -42,11 +56,14 @@ export async function serveMedia(req: Request, absPath: string, opts: { download const headers: Record = { "Content-Type": contentType(absPath), "Accept-Ranges": "bytes", + ...mediaSecurityHeaders(absPath), }; if (opts.download) headers["Content-Disposition"] = contentDisposition(name); const range = req.headers.get("range"); const parsed = range ? parseRange(range, size) : null; - if (parsed === "invalid") return new Response(null, { status: 416, headers: { "Content-Range": `bytes */${size}`, "Accept-Ranges": "bytes" } }); + if (parsed === "invalid" || (parsed && size === 0)) { + return new Response(null, { status: 416, headers: { "Content-Range": `bytes */${size}`, "Accept-Ranges": "bytes", "X-Content-Type-Options": "nosniff" } }); + } if (!parsed) { headers["Content-Length"] = String(size); return new Response(req.method === "HEAD" ? null : file, { status: 200, headers }); diff --git a/packages/core/test/app.test.ts b/packages/core/test/app.test.ts index 6a8545c..ad6f1c1 100644 --- a/packages/core/test/app.test.ts +++ b/packages/core/test/app.test.ts @@ -97,5 +97,13 @@ test("canView matrix", async () => { const pub = ctx.courses.get("sample")!, priv = ctx.courses.get("secret")!; expect(canView(null, pub, [])).toBe(true); expect(canView(null, priv, [])).toBe(false); + const u = (roles: AuthUser["roles"], groups: string[] = []): AuthUser => ({ id: "u1", email: "u@x", name: "U", roles, groups }); + expect(canView(u(["learner"]), priv, [])).toBe(false); + expect(canView(u(["owner"]), priv, [])).toBe(true); + expect(canView(u(["author"]), priv, [])).toBe(true); // R14: the global author role views every course + expect(canView(u(["learner"]), priv, [], true)).toBe(true); // per-course author + expect(canView(u(["learner"]), priv, [{ principal_type: "user", principal_id: "u1" }])).toBe(true); + expect(canView(u(["learner"], ["g"]), priv, [{ principal_type: "group", principal_id: "g" }])).toBe(true); + expect(canView(u(["learner"], ["h"]), priv, [{ principal_type: "group", principal_id: "g" }])).toBe(false); await db.close(); }); diff --git a/packages/core/test/compose.test.ts b/packages/core/test/compose.test.ts new file mode 100644 index 0000000..213877b --- /dev/null +++ b/packages/core/test/compose.test.ts @@ -0,0 +1,51 @@ +import { expect, test } from "bun:test"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import type { AuthUser, Feature, ServerContext } from "@stc/contracts"; +import { composeServer, resolveConfig } from "../src"; + +const FIXTURES = resolve(import.meta.dir, "../../../fixtures/courses"); +const config = () => resolveConfig({ dataDir: mkdtempSync(join(tmpdir(), "stc-compose-")), databaseUrl: "sqlite://:memory:", courseDirs: [FIXTURES] }); + +test("factories get the context; the queue is built from the union of their jobs; auth and stop are honoured", async () => { + const seen: string[] = []; + let stopped = 0; + const user: AuthUser = { id: "u", email: "u@x", name: "U", roles: ["owner"], groups: [] }; + let early: unknown = null; + const a = (ctx: ServerContext): Feature => { + // the handle exists while factories run, but enqueuing must wait for the real queue + try { void ctx.jobs.enqueue("b.work", {}); } catch (e) { early = e; } + return { + name: "a", + routes: { "/api/a": { GET: async (_r, rc) => Response.json({ who: rc.user?.id ?? null, courses: ctx.courses.list().length }) } }, + jobs: { "a.work": async (p) => { seen.push(`a:${JSON.stringify(p)}`); return null; } }, + auth: { userFromRequest: async () => user }, + stop: () => { stopped++; }, + }; + }; + const b = (ctx: ServerContext): Feature => ({ + name: "b", + routes: { "/api/b": { POST: async () => Response.json({ id: await ctx.jobs.enqueue("a.work", { n: 1 }) }) } }, + jobs: { "b.work": async () => null }, + }); + const s = await composeServer(config(), [a, b]); + try { + expect(String(early)).toMatch(/not ready/); + expect(await (await s.app.fetch(new Request("http://x/api/a"))).json()).toEqual({ who: "u", courses: 1 }); + const { id } = (await (await s.app.fetch(new Request("http://x/api/b", { method: "POST" }))).json()) as { id: string }; + for (let i = 0; i < 200 && (await s.ctx.jobs.get(id))?.status !== "done"; i++) await Bun.sleep(10); + expect((await s.ctx.jobs.get(id))?.status).toBe("done"); + expect(seen).toEqual(['a:{"n":1}']); + } finally { + await s.stop(); + } + expect(stopped).toBe(1); +}); + +test("duplicate job kinds and two auth services are refused", async () => { + const f = (name: string, extra: Partial) => (): Feature => ({ name, routes: {}, ...extra }); + await expect(composeServer(config(), [f("x", { jobs: { k: async () => null } }), f("y", { jobs: { k: async () => null } })])).rejects.toThrow(/Duplicate job kind "k"/); + const auth = { userFromRequest: async () => null }; + await expect(composeServer(config(), [f("x", { auth }), f("y", { auth })])).rejects.toThrow(/AuthService/); +}); diff --git a/packages/core/test/config.test.ts b/packages/core/test/config.test.ts index 3305ea7..85c3d15 100644 --- a/packages/core/test/config.test.ts +++ b/packages/core/test/config.test.ts @@ -1,5 +1,6 @@ import { expect, test, spyOn } from "bun:test"; -import { loadConfig } from "../src"; +import { resolve } from "node:path"; +import { cachePaths, loadConfig, resolveConfig } from "../src"; test("defaults", () => { const warn = spyOn(console, "warn").mockImplementation(() => {}); @@ -7,21 +8,59 @@ test("defaults", () => { expect(c.port).toBe(3000); expect(c.databaseUrl).toBe("sqlite://./data/stcourses.db"); expect(c.dataDir).toBe("./data"); - expect(c.courseDirs).toEqual(["./overlays", "./courses"]); + // configured dirs first, then the importer's folders and the studio's published worktrees + expect(c.courseDirs).toEqual(["./overlays", "./courses", resolve("./data/courses"), resolve("./data/worktrees")]); + expect(c.importedCoursesDir).toBe(resolve("./data/courses")); + expect(c.studioReposDir).toBe(resolve("./data/studio/repos")); + expect(c.studioWorktreesDir).toBe(resolve("./data/worktrees")); + expect(c.cacheDir).toBe(resolve("./data/cache")); + expect(c.importRoots).toEqual([resolve("./data/import-inbox")]); + expect(c.uploadMaxMb).toBe(100); + expect(c.aiAllowedBaseUrls).toEqual([]); + expect(c.transcribeUrl).toBeNull(); + expect(c.python).toBe("python3"); expect(c.publicUrl).toBe("http://localhost:3000"); expect(warn).toHaveBeenCalledTimes(1); warn.mockRestore(); }); test("colon splitting and overrides", () => { - const c = loadConfig({ STC_PORT: "8080", STC_COURSE_DIRS: "/a b:/c", STC_SECRET: "s", STC_PUBLIC_URL: "https://x.test" }); + const c = loadConfig({ + STC_PORT: "8080", STC_COURSE_DIRS: "/a b:/c", STC_SECRET: "s", STC_PUBLIC_URL: "https://x.test", STC_DATA_DIR: "/d", + STC_IMPORT_ROOTS: "/in box:/other", STC_UPLOAD_MAX_MB: "5", STC_AI_ALLOWED_BASE_URLS: "http://gw.lan:8080, https://x.test", + STC_TRANSCRIBE_URL: "http://whisper.lan/v1/audio/transcriptions", STC_PYTHON: "/opt/py/bin/python", + }); expect(c.port).toBe(8080); - expect(c.courseDirs).toEqual(["/a b", "/c"]); + expect(c.courseDirs).toEqual(["/a b", "/c", "/d/courses", "/d/worktrees"]); expect(c.secret).toBe("s"); expect(c.publicUrl).toBe("https://x.test"); + expect(c.importRoots).toEqual(["/in box", "/other"]); + expect(c.uploadMaxMb).toBe(5); + expect(c.aiAllowedBaseUrls).toEqual(["http://gw.lan:8080", "https://x.test"]); + expect(c.transcribeUrl).toBe("http://whisper.lan/v1/audio/transcriptions"); + expect(c.python).toBe("/opt/py/bin/python"); +}); + +test("configured dirs that already name the derived dirs are not repeated", () => { + const c = loadConfig({ STC_SECRET: "s", STC_DATA_DIR: "/d", STC_COURSE_DIRS: "/d/courses:/x" }); + expect(c.courseDirs).toEqual(["/d/courses", "/x", "/d/worktrees"]); + expect(resolveConfig({ dataDir: "/d", courseDirs: ["/d/worktrees/"] }).courseDirs).toEqual(["/d/worktrees/", "/d/courses"]); +}); + +test("invalid upload cap throws", () => { + expect(() => loadConfig({ STC_SECRET: "s", STC_UPLOAD_MAX_MB: "lots" })).toThrow(/STC_UPLOAD_MAX_MB/); }); test("production without a secret throws", () => { expect(() => loadConfig({ NODE_ENV: "production" })).toThrow(/STC_SECRET/); expect(loadConfig({ NODE_ENV: "production", STC_SECRET: "x" }).secret).toBe("x"); }); + +test("cachePaths: the importer's layout, no escaping segments", () => { + const p = cachePaths({ cacheDir: "/d/cache" }); + expect(p.deckDir("c", "deck1")).toBe("/d/cache/decks/c/deck1"); + expect(p.coversDir("c")).toBe("/d/cache/covers/c"); + expect(p.transcriptFile("c", "m1-l1")).toBe("/d/cache/transcripts/c/m1-l1.json"); + expect(() => p.coversDir("..")).toThrow(); + expect(() => p.deckDir("c", "a/b")).toThrow(); +}); diff --git a/packages/core/test/media.test.ts b/packages/core/test/media.test.ts index 10fdb74..0060c1c 100644 --- a/packages/core/test/media.test.ts +++ b/packages/core/test/media.test.ts @@ -57,3 +57,36 @@ test("non-ASCII filename gets a valid filename*", async () => { test("missing file is 404", async () => { expect((await serveMedia(req(), join(dir, "nope.mp4"))).status).toBe(404); }); + +test("every response is nosniff; script-capable types get a sandbox CSP", async () => { + const d = mkdtempSync(join(tmpdir(), "stc-media-csp-")); + const { MEDIA_SANDBOX_CSP } = await import("../src"); + for (const [name, body, sandboxed] of [ + ["x.svg", "", true], + ["page.html", "", true], + ["Page.HTM", "

      x

      ", true], + ["code.js", "alert(1)", true], + ["mod.mjs", "export {}", true], + ["notes.md", "# hi", false], + ["clip.mp4", "x", false], + ] as const) { + writeFileSync(join(d, name), body); + for (const r of [await serveMedia(req(), join(d, name)), await serveMedia(req("bytes=0-0"), join(d, name))]) { + expect(r.headers.get("x-content-type-options")).toBe("nosniff"); + expect([name, r.headers.get("content-security-policy")]).toEqual([name, sandboxed ? MEDIA_SANDBOX_CSP : null]); + } + } + expect(MEDIA_SANDBOX_CSP).toMatch(/^sandbox;/); + expect(MEDIA_SANDBOX_CSP).not.toContain("allow-scripts"); +}); + +test("a range on an empty file is 416, never a negative Content-Range", async () => { + const d = mkdtempSync(join(tmpdir(), "stc-media-empty-")); + writeFileSync(join(d, "empty.txt"), ""); + for (const range of ["bytes=0-", "bytes=-5", "bytes=0-0"]) { + const r = await serveMedia(req(range), join(d, "empty.txt")); + expect(r.status).toBe(416); + expect(r.headers.get("content-range")).toBe("bytes */0"); + } + expect((await serveMedia(req(), join(d, "empty.txt"))).status).toBe(200); +}); diff --git a/packages/course-format/package.json b/packages/course-format/package.json index 935c4e7..b3d7584 100644 --- a/packages/course-format/package.json +++ b/packages/course-format/package.json @@ -6,5 +6,9 @@ "license": "AGPL-3.0-only", "dependencies": { "zod": "^4.6.7" + }, + "exports": { + ".": "./src/index.ts", + "./markdown": "./src/markdown.ts" } } diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index ec4d4e5..7129602 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -1,5 +1,5 @@ export { connect, type Db } from "./connect"; -export { migrate } from "./migrate"; +export { migrate, type PackageMigrations } from "./migrate"; export { progressRepo, type LessonProgress, type ProgressState } from "./repos/progress"; export { notesRepo, type Note, type NoteInput } from "./repos/notes"; export { attemptsRepo, type Attempt } from "./repos/attempts"; diff --git a/packages/db/src/migrate.ts b/packages/db/src/migrate.ts index be5f38f..7f71ffc 100644 --- a/packages/db/src/migrate.ts +++ b/packages/db/src/migrate.ts @@ -9,19 +9,34 @@ function statements(text: string): string[] { return noComments.split(";").map((s) => s.trim()).filter(Boolean); } -export async function migrate(db: Db): Promise { +/** Migration folders contributed by feature packages, keyed by package name. */ +export type PackageMigrations = Record; + +/** + * Applies pending migrations: core (`packages/db/migrations`) first, then each package's folders in alphabetical + * order of package name, files in name order within a folder. `schema_migrations` is keyed by file name, so file + * names must be unique across packages (each package prefixes its files, e.g. `0001_learner_auth.sql`). + */ +export async function migrate(db: Db, packages: PackageMigrations = {}): Promise { await db.sql.unsafe("CREATE TABLE IF NOT EXISTS schema_migrations (name TEXT PRIMARY KEY, applied_at TEXT NOT NULL)"); const done = new Set((await db.sql`SELECT name FROM schema_migrations`).map((r: { name: string }) => r.name)); - const files = readdirSync(MIGRATIONS_DIR).filter((f) => f.endsWith(".sql")).sort(); + const dirs = [MIGRATIONS_DIR, ...Object.keys(packages).sort().flatMap((k) => packages[k]!)]; + const seen = new Map(); const applied: string[] = []; - for (const name of files) { - if (done.has(name)) continue; - const stmts = statements(readFileSync(join(MIGRATIONS_DIR, name), "utf8")); - await db.sql.begin(async (tx) => { - for (const s of stmts) await tx.unsafe(s); - await tx`INSERT INTO schema_migrations (name, applied_at) VALUES (${name}, ${new Date().toISOString()})`; - }); - applied.push(name); + for (const dir of dirs) { + for (const name of readdirSync(dir).filter((f) => f.endsWith(".sql")).sort()) { + const other = seen.get(name); + if (other && other !== dir) throw new Error(`migration ${name} exists in both ${other} and ${dir}`); + seen.set(name, dir); + if (done.has(name)) continue; + const stmts = statements(readFileSync(join(dir, name), "utf8")); + await db.sql.begin(async (tx) => { + for (const s of stmts) await tx.unsafe(s); + await tx`INSERT INTO schema_migrations (name, applied_at) VALUES (${name}, ${new Date().toISOString()})`; + }); + done.add(name); + applied.push(name); + } } return applied; } diff --git a/packages/db/test/migrate.test.ts b/packages/db/test/migrate.test.ts index 8619728..23f3301 100644 --- a/packages/db/test/migrate.test.ts +++ b/packages/db/test/migrate.test.ts @@ -30,3 +30,22 @@ forEachDb("all tables exist", async (db) => { await one("collection_courses", s`INSERT INTO collection_courses VALUES ('co','c',0)`); await one("jobs", s`INSERT INTO jobs VALUES ('j','k','queued',0,NULL,'{}',NULL,NULL,${t},${t})`); }); + +forEachDb("package migrations run after core, in package-name order, once", async (db) => { + const { mkdtempSync, writeFileSync, mkdirSync } = await import("node:fs"); + const { join } = await import("node:path"); + const { tmpdir } = await import("node:os"); + const root = mkdtempSync(join(tmpdir(), "stc-mig-")); + const a = join(root, "a"), z = join(root, "z"); + mkdirSync(a); mkdirSync(z); + // z's table is referenced by a's second file only through ordering by package name: "alpha" runs before "zeta". + writeFileSync(join(z, "0001_zeta.sql"), "CREATE TABLE zeta_t (id TEXT PRIMARY KEY);"); + writeFileSync(join(a, "0001_alpha.sql"), "CREATE TABLE alpha_t (id TEXT PRIMARY KEY);"); + writeFileSync(join(a, "0002_alpha.sql"), "INSERT INTO alpha_t VALUES ('x');"); + expect(await migrate(db, { zeta: [z], alpha: [a] })).toEqual(["0001_alpha.sql", "0002_alpha.sql", "0001_zeta.sql"]); + expect(await migrate(db, { zeta: [z], alpha: [a] })).toEqual([]); + expect((await db.sql`SELECT id FROM alpha_t`).length).toBe(1); + // the same file name in two packages is refused rather than silently skipped + writeFileSync(join(z, "0001_alpha.sql"), "SELECT 1;"); + await expect(migrate(db, { zeta: [z], alpha: [a] })).rejects.toThrow(/exists in both/); +}); diff --git a/packages/feature-ai/src/feature.ts b/packages/feature-ai/src/feature.ts index d17f3a7..13b5180 100644 --- a/packages/feature-ai/src/feature.ts +++ b/packages/feature-ai/src/feature.ts @@ -24,8 +24,6 @@ export interface AiFeatureOptions { /** Overrides provider construction (tests use a fake AiProvider). */ providerFactory?: (cfg: ProviderConfig, apiKey: string | null) => AiProvider; mcpTimeoutMs?: number; - /** Environment to read STC_AI_ALLOWED_BASE_URLS from. Default: process.env. */ - env?: Record; /** Server-side log for provider errors that are not shown to users. Default: console.error. */ log?: (message: string) => void; } @@ -147,8 +145,9 @@ export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feat const progress = progressRepo(ctx.db); const sink = opts.draftSink ?? createFsDraftSink(ctx.config.dataDir); const log = opts.log ?? ((m: string) => console.error(`[ai] ${m}`)); - const env = opts.env ?? process.env; - const allowedOrigins = async () => [...new Set([...parseAllowedOrigins(env.STC_AI_ALLOWED_BASE_URLS ?? ""), ...(await settings.getServer()).allowedBaseUrls])]; + // AppConfig.aiAllowedBaseUrls (STC_AI_ALLOWED_BASE_URLS) plus the admin's list in the server AI settings. + const configOrigins = parseAllowedOrigins((ctx.config.aiAllowedBaseUrls ?? []).join(",")); + const allowedOrigins = async () => [...new Set([...configOrigins, ...(await settings.getServer()).allowedBaseUrls])]; /** * Providers for a user's own key never follow redirects, and their errors reach the user only as the HTTP * status plus a fixed sentence (the full text is logged here). Server-scope providers are the admin's own. @@ -186,9 +185,8 @@ export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feat }; /** * Generation and the studio file listing: the user must hold an authoring right (owner, the `author` role, or a - * `course_authors` row) AND see the course under core's `canView`, the same visibility rule as everywhere else. - * A global `author` therefore cannot reach a restricted course without an ACL grant or a course_authors row - * (see contract request 8). + * `course_authors` row) AND see the course under core's `canView`, the one visibility rule (R14; it lets the + * global `author` role view every course). */ const canAuthor = async (user: AuthUser, course: LoadedCourse) => { const isAuthorRow = await acl.isAuthor(course.id, user.id); diff --git a/packages/feature-ai/src/web/GeneratePage.tsx b/packages/feature-ai/src/web/GeneratePage.tsx index c96c0fe..fd53529 100644 --- a/packages/feature-ai/src/web/GeneratePage.tsx +++ b/packages/feature-ai/src/web/GeneratePage.tsx @@ -178,7 +178,11 @@ export default function GeneratePage() { {job.status === "failed" ?

      {job.error}

      : null} {job.result ? ( <> - {job.result.branch ? "Committed to your draft branch " : "Saved as a draft at "}{job.result.location}. Review it in the studio before publishing. + {job.result.branch ? ( + Committed to your draft branch {job.result.location}. Review it in the studio before publishing. + ) : ( + This course is not edited in the studio, so the draft was saved as files on the server at {job.result.location}. Nothing in the course changed; copy what you want into it yourself. + )}
        {job.result.paths.map((p) =>
      • {p}draft
        • )}
      {job.result.warnings.length ? (
      diff --git a/packages/feature-ai/src/web/index.ts b/packages/feature-ai/src/web/index.ts index 181fc97..9fb7424 100644 --- a/packages/feature-ai/src/web/index.ts +++ b/packages/feature-ai/src/web/index.ts @@ -4,7 +4,7 @@ export const aiWeb: WebFeature = { name: "ai", routes: [ { path: "/c/:courseId/assistant", element: () => import("./AssistantPage") }, - { path: "/admin/ai", element: () => import("./AdminAiPage"), nav: { label: "AI", area: "admin" } }, + { path: "/admin/ai", element: () => import("./AdminAiPage"), nav: { label: "AI", area: "admin", roles: ["owner"] } }, { path: "/account/ai", element: () => import("./AccountAiPage") }, { path: "/studio/generate", element: () => import("./GeneratePage") }, ], diff --git a/packages/feature-ai/src/web/urls.ts b/packages/feature-ai/src/web/urls.ts index d08fb57..375f206 100644 --- a/packages/feature-ai/src/web/urls.ts +++ b/packages/feature-ai/src/web/urls.ts @@ -1,11 +1,2 @@ -/** - * Learner URLs the AI pages link to. The learner app (WP-A) owns these routes; keep every builder here so - * Phase 2 can change the shape in one place (contract request 7). - */ -export function courseUrl(courseId: string): string { - return `/c/${encodeURIComponent(courseId)}`; -} - -export function lessonUrl(courseId: string, lessonId: string): string { - return `${courseUrl(courseId)}/${encodeURIComponent(lessonId)}`; -} +// Learner URLs the AI pages link to; the learner owns the routes (shared helpers in @stc/contracts, request 7). +export { courseUrl, lessonUrl } from "@stc/contracts"; diff --git a/packages/feature-ai/test/feature.test.ts b/packages/feature-ai/test/feature.test.ts index 96fa0bd..148ad52 100644 --- a/packages/feature-ai/test/feature.test.ts +++ b/packages/feature-ai/test/feature.test.ts @@ -2,8 +2,8 @@ import { expect } from "bun:test"; import { cpSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; -import type { AuthUser, JobRecord, ServerContext } from "@stc/contracts"; -import { createApp, createJobQueue } from "@stc/core"; +import type { AppConfig, AuthUser, JobRecord, ServerContext } from "@stc/contracts"; +import { createApp, createJobQueue, resolveConfig } from "@stc/core"; import { createCourseRepository } from "@stc/course-format"; import type { Db } from "@stc/db"; import { forEachDb } from "../../db/test/helpers"; @@ -18,7 +18,7 @@ const USERS: Record = { auth: { id: "auth", email: "w@x", name: "Writer", roles: ["author"], groups: [] }, }; -async function setup(db: Db, turns: Parameters[0] = [() => ({ text: "ok" })], extra: Partial = {}) { +async function setup(db: Db, turns: Parameters[0] = [() => ({ text: "ok" })], extra: Partial = {}, configOverrides: Partial = {}) { const dir = mkdtempSync(join(tmpdir(), "stc-ai-feat-")); cpSync(SAMPLE, join(dir, "courses/sample"), { recursive: true }); cpSync(SAMPLE, join(dir, "courses/secret"), { recursive: true }); @@ -29,7 +29,7 @@ async function setup(db: Db, turns: Parameters[0] = [() => const keys: (string | null)[] = []; const ctx = { db, courses, - config: { port: 0, databaseUrl: "", dataDir: join(dir, "data"), courseDirs: [], secret: "test-secret", publicUrl: "" }, + config: resolveConfig({ port: 0, databaseUrl: "", dataDir: join(dir, "data"), courseDirs: [], secret: "test-secret", publicUrl: "", ...configOverrides }), auth: { userFromRequest: async (req: Request) => USERS[req.headers.get("x-user") ?? ""] ?? null }, } as unknown as ServerContext; const feature = aiFeature(ctx, { providerFactory: (_cfg, key) => { keys.push(key); return fake.provider; }, ...extra }); @@ -159,7 +159,7 @@ forEachDb("generation is for authors and writes drafts, not course files", async // SSRF guard: a learner's own key (BYOK) may only go to the official endpoints or admin-allowed origins. forEachDb("BYOK base URLs are limited to official endpoints and allowed origins", async (db) => { - const { call } = await setup(db, undefined, { env: { STC_AI_ALLOWED_BASE_URLS: "http://ollama.internal:11434, https://llm.example.org" } }); + const { call } = await setup(db, undefined, {}, { aiAllowedBaseUrls: ["http://ollama.internal:11434", "https://llm.example.org"] }); const put = (body: unknown) => call("PUT", "/api/ai/account/key", "ann", body); // Internal or arbitrary hosts are refused for either kind. for (const baseUrl of ["http://127.0.0.1:8080/v1", "http://169.254.169.254/latest", "http://localhost:11434/v1", "https://api.anthropic.com.evil.test", "http://ollama.internal:11435/v1", "https://ollama.internal:11434/v1", "file:///etc/passwd"]) { @@ -203,7 +203,7 @@ forEachDb("provider errors on a user's own key show the status and a fixed messa seen.push({ url: String(input), redirect: init?.redirect }); return new Response(JSON.stringify({ error: { message: "INTERNAL-SECRET-BODY from upstream" } }), { status: 502, headers: { "content-type": "application/json" } }); }) as typeof fetch; - const { call } = await setup(db, undefined, { providerFactory: undefined, fetch: fakeFetch, log: (m) => logs.push(m), env: { STC_AI_ALLOWED_BASE_URLS: "http://ollama.internal:11434" } }); + const { call } = await setup(db, undefined, { providerFactory: undefined, fetch: fakeFetch, log: (m) => logs.push(m) }, { aiAllowedBaseUrls: ["http://ollama.internal:11434"] }); expect((await call("PUT", "/api/ai/account/key", "ann", { kind: "openai", model: "m", apiKey: "k", baseUrl: "http://ollama.internal:11434/v1" })).status).toBe(200); const text = await (await call("POST", "/api/ai/courses/sample/chat", "ann", { message: "hello" })).text(); expect(text).not.toContain("INTERNAL-SECRET-BODY"); @@ -219,22 +219,18 @@ forEachDb("provider errors on a user's own key show the status and a fixed messa expect(t2).toContain("502: INTERNAL-SECRET-BODY"); }); -forEachDb("authoring uses core canView: a global author needs a grant for a restricted course", async (db) => { +forEachDb("authoring uses core canView: the global author role reaches restricted courses (R14)", async (db) => { const { call } = await setup(db); await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "anthropic", apiKey: "k" }, mcpServers: [] }); const body = { courseId: "secret", kind: "questions", style: "pure-snake", topic: "Adding", sources: [{ kind: "lesson", lessonId: "m2-text" }] }; const courses = async (u: string) => ((await (await call("GET", "/api/ai/studio/courses", u)).json()) as { id: string }[]).map((c) => c.id).sort(); - // Public course: the author role is enough. Restricted course: not without a grant. - expect(await courses("auth")).toEqual(["sample"]); - expect((await call("GET", "/api/ai/studio/sources?courseId=secret", "auth")).status).toBe(403); - expect((await call("POST", "/api/ai/studio/generate", "auth", body)).status).toBe(403); + // The global author role views every course under core canView, so it may author the restricted one too. + expect(await courses("auth")).toEqual(["sample", "secret"]); + expect((await call("GET", "/api/ai/studio/sources?courseId=secret", "auth")).status).toBe(200); + expect((await call("POST", "/api/ai/studio/generate", "auth", body)).status).toBe(202); // The owner sees everything. expect(await courses("owner")).toEqual(["sample", "secret"]); expect((await call("POST", "/api/ai/studio/generate", "owner", body)).status).toBe(202); - // An ACL grant (core canView) lets the global author in. - await db.sql`INSERT INTO course_acl (course_id, principal_type, principal_id) VALUES ('secret', 'user', 'auth')`; - expect(await courses("auth")).toEqual(["sample", "secret"]); - expect((await call("POST", "/api/ai/studio/generate", "auth", body)).status).toBe(202); // A course_authors row makes a learner an author of that course only. expect((await call("POST", "/api/ai/studio/generate", "bob", body)).status).toBe(403); await db.sql`INSERT INTO course_authors (course_id, user_id) VALUES ('secret', 'bob')`; diff --git a/packages/feature-ai/test/web.test.tsx b/packages/feature-ai/test/web.test.tsx index 683652d..231dbd3 100644 --- a/packages/feature-ai/test/web.test.tsx +++ b/packages/feature-ai/test/web.test.tsx @@ -32,7 +32,7 @@ test("markdown renders code and text without raw HTML", () => { test("web routes", () => { expect(aiWeb.routes.map((r) => r.path)).toEqual(["/c/:courseId/assistant", "/admin/ai", "/account/ai", "/studio/generate"]); - expect(aiWeb.routes.find((r) => r.path === "/admin/ai")!.nav).toEqual({ label: "AI", area: "admin" }); + expect(aiWeb.routes.find((r) => r.path === "/admin/ai")!.nav).toEqual({ label: "AI", area: "admin", roles: ["owner"] }); }); test("lesson links go through one builder", async () => { @@ -44,3 +44,13 @@ test("lesson links go through one builder", async () => { expect(src).toContain("lessonUrl(courseId, c.lessonId)"); expect(src).not.toMatch(/`\/c\//); }); + +test("citation links use the learner's registered lesson route", async () => { + const { learnerWeb } = await import("@stc/feature-learner/web"); + const { matchPath } = await import("react-router-dom"); + const { lessonUrl } = await import("../src/web/urls"); + const url = lessonUrl("pure snake", "m1-ü/x"); + const hit = learnerWeb.routes.map((r) => matchPath(r.path, url)).find((m) => m && m.pattern.path === "/c/:courseId/:lessonId"); + // matchPath leaves params encoded (the router decodes the pathname before matching). + expect(Object.fromEntries(Object.entries(hit?.params ?? {}).map(([k, v]) => [k, decodeURIComponent(v!)]))).toEqual({ courseId: "pure snake", lessonId: "m1-ü/x" }); +}); diff --git a/packages/feature-import/src/context.ts b/packages/feature-import/src/context.ts index caeb3d2..d0eff37 100644 --- a/packages/feature-import/src/context.ts +++ b/packages/feature-import/src/context.ts @@ -6,17 +6,24 @@ import { slugify, uniqueId } from "./util/text"; import { extractFrame, type Tools } from "./media"; export interface ImportEnv { [k: string]: string | undefined } +/** `env` supplies only tool paths and transcription credentials (STC_FFMPEG, STC_FFPROBE, STC_TRANSCRIBE_KEY, _MODEL). */ export interface ImportOptions { env?: ImportEnv } export interface Deps { ctx: ServerContext; env: ImportEnv; tools: Tools; + /** AppConfig.python (STC_PYTHON). */ python: string; + /** AppConfig.importRoots (STC_IMPORT_ROOTS). */ + importRoots: string[]; + /** AppConfig.transcribeUrl (STC_TRANSCRIBE_URL). */ + transcribeUrl: string | null; } -export const coursesDir = (d: Deps) => resolve(d.ctx.config.dataDir, "courses"); -export const cacheDir = (d: Deps) => resolve(d.ctx.config.dataDir, "cache"); +export const coursesDir = (d: Deps) => resolve(d.ctx.config.importedCoursesDir ?? resolve(d.ctx.config.dataDir, "courses")); +export const cacheDir = (d: Deps) => resolve(d.ctx.config.cacheDir ?? resolve(d.ctx.config.dataDir, "cache")); +const studioRepo = (d: Deps, id: string) => join(d.ctx.config.studioReposDir ?? resolve(d.ctx.config.dataDir, "studio", "repos"), `${id}.git`); export const deckCacheDir = (d: Deps, courseId: string, deckId: string) => join(cacheDir(d), "decks", courseId, deckId); export const coverCacheDir = (d: Deps, courseId: string) => join(cacheDir(d), "covers", courseId); export const transcriptCacheDir = (d: Deps, courseId: string) => join(cacheDir(d), "transcripts", courseId); @@ -27,11 +34,14 @@ export const isPlainId = (s: string) => /^[a-z0-9][a-z0-9-]*$/.test(s); export async function freshCourseId(d: Deps, wanted: string | undefined, title: string): Promise { if (wanted !== undefined) { if (!isPlainId(wanted)) throw new Error(`invalid courseId "${wanted}" (use lowercase letters, digits and dashes)`); - if (await exists(join(coursesDir(d), wanted)) || d.ctx.courses.get(wanted)) throw new Error(`a course with id "${wanted}" already exists`); + // Studio repos share the id space (ruling R9): refuse an id the studio already uses, even before it is published. + if (await exists(join(coursesDir(d), wanted)) || await exists(studioRepo(d, wanted)) || d.ctx.courses.get(wanted)) throw new Error(`a course with id "${wanted}" already exists`); return wanted; } const taken = new Set(d.ctx.courses.list().map((c) => c.id)); for (const e of await readdir(coursesDir(d)).catch(() => [] as string[])) taken.add(e); + const repos = d.ctx.config.studioReposDir ?? resolve(d.ctx.config.dataDir, "studio", "repos"); + for (const e of await readdir(repos).catch(() => [] as string[])) if (e.endsWith(".git")) taken.add(e.slice(0, -4)); return uniqueId(slugify(title), (x) => taken.has(x)); } diff --git a/packages/feature-import/src/index.ts b/packages/feature-import/src/index.ts index 367fe43..0262bea 100644 --- a/packages/feature-import/src/index.ts +++ b/packages/feature-import/src/index.ts @@ -1,3 +1,4 @@ +import { resolve } from "node:path"; import type { Feature, JobHandler, ServerContext } from "@stc/contracts"; import type { Deps, ImportOptions } from "./context"; import { importCodeRepo } from "./jobs/code-repo"; @@ -14,15 +15,22 @@ export { deckCacheDir, coverCacheDir, transcriptCacheDir } from "./context"; /** * `ctx` is only read when a job runs or a request arrives, never while this function runs, so the server can pass * its context object before the job queue (which needs these handlers) exists. - * Environment: STC_PYTHON (Python with PyMuPDF, default python3), STC_FFMPEG / STC_FFPROBE, STC_TRANSCRIBE_URL (+ _KEY, _MODEL). + * Config: AppConfig.python, importRoots, transcribeUrl. Environment: STC_FFMPEG / STC_FFPROBE, STC_TRANSCRIBE_KEY / _MODEL. */ export function importFeature(ctx: ServerContext, opts: ImportOptions = {}): Feature { const deps = (): Deps => { const env = opts.env ?? process.env; - return { ctx, env, tools: { ffmpeg: env.STC_FFMPEG || "ffmpeg", ffprobe: env.STC_FFPROBE || "ffprobe" }, python: env.STC_PYTHON || "python3" }; + const c = ctx.config; + return { + ctx, env, + tools: { ffmpeg: env.STC_FFMPEG || "ffmpeg", ffprobe: env.STC_FFPROBE || "ffprobe" }, + python: c.python || "python3", + importRoots: c.importRoots?.length ? c.importRoots : [resolve(c.dataDir, "import-inbox")], + transcribeUrl: c.transcribeUrl ?? null, + }; }; const job = (fn: (d: Deps, p: unknown, jc: Parameters[1]) => Promise): JobHandler => (p, jc) => fn(deps(), p, jc); - ensureRoots({ ctx, env: opts.env ?? process.env }); + ensureRoots(deps()); return { name: "import", routes: importRoutes(deps), @@ -37,3 +45,6 @@ export function importFeature(ctx: ServerContext, opts: ImportOptions = {}): Fea } export { importWeb } from "./web"; + +/** The importer adds no tables. */ +export const migrationsDirs: string[] = []; diff --git a/packages/feature-import/src/jobs/common.ts b/packages/feature-import/src/jobs/common.ts index f8158fc..e685e81 100644 --- a/packages/feature-import/src/jobs/common.ts +++ b/packages/feature-import/src/jobs/common.ts @@ -37,7 +37,7 @@ export async function adoptCover(d: Deps, courseId: string, courseDir: string, c export async function maybeQueueTranscription(d: Deps, courseId: string, wanted: boolean, warnings: string[]): Promise { if (!wanted) return; - if (!d.env.STC_TRANSCRIBE_URL) { + if (!d.transcribeUrl) { warnings.push("transcription requested but STC_TRANSCRIBE_URL is not set; skipped"); return; } diff --git a/packages/feature-import/src/jobs/transcribe.ts b/packages/feature-import/src/jobs/transcribe.ts index 7afd802..dfbafb4 100644 --- a/packages/feature-import/src/jobs/transcribe.ts +++ b/packages/feature-import/src/jobs/transcribe.ts @@ -33,7 +33,7 @@ export function segmentsToMarkdown(title: string, segments: Segment[], fallbackT */ export async function transcribeCourse(d: Deps, payload: unknown, jc: JobContext): Promise { const p = (payload ?? {}) as Record; - const url = d.env.STC_TRANSCRIBE_URL; + const url = d.transcribeUrl; const courseId = optStr(p, "courseId") ?? ""; if (!url) return { courseId, skipped: true, reason: "STC_TRANSCRIBE_URL is not set", transcribed: 0, warnings: [] }; const course = await managedCourse(d, courseId); diff --git a/packages/feature-import/src/roots.ts b/packages/feature-import/src/roots.ts index 621c96f..a33fdd0 100644 --- a/packages/feature-import/src/roots.ts +++ b/packages/feature-import/src/roots.ts @@ -5,19 +5,17 @@ import type { Deps } from "./context"; export const OUTSIDE = "path is outside the allowed import folders"; -/** STC_IMPORT_ROOTS (":"-separated), default /import-inbox. */ -export function importRoots(d: Pick): string[] { - const raw = d.env.STC_IMPORT_ROOTS; - const list = raw ? raw.split(":").filter(Boolean) : [resolve(d.ctx.config.dataDir, "import-inbox")]; - return list.map((r) => resolve(r)); +/** AppConfig.importRoots: STC_IMPORT_ROOTS (":"-separated), default /import-inbox. */ +export function importRoots(d: Pick): string[] { + return d.importRoots.map((r) => resolve(r)); } -export function ensureRoots(d: Pick): void { +export function ensureRoots(d: Pick): void { for (const r of importRoots(d)) { try { mkdirSync(r, { recursive: true }); } catch { /* reported when used */ } } } /** Real path (symlinks followed) must lie inside the real path of one root. Throws OUTSIDE otherwise. */ -export async function assertAllowedPath(d: Pick, path: string): Promise { +export async function assertAllowedPath(d: Pick, path: string): Promise { let real: string; try { real = await realpath(resolve(path)); } catch { throw new Error(OUTSIDE); } for (const root of importRoots(d)) { diff --git a/packages/feature-import/src/routes.ts b/packages/feature-import/src/routes.ts index c44570f..01384eb 100644 --- a/packages/feature-import/src/routes.ts +++ b/packages/feature-import/src/routes.ts @@ -26,7 +26,7 @@ export function importRoutes(deps: () => Deps): Feature["routes"] { const d = deps(); const dir = coursesDir(d); const served = d.ctx.config.courseDirs.some((c) => resolve(c) === dir); - return json({ transcribe: Boolean(d.env.STC_TRANSCRIBE_URL), coursesDir: dir, coursesDirServed: served, importRoots: importRoots(d) }); + return json({ transcribe: Boolean(d.transcribeUrl), coursesDir: dir, coursesDirServed: served, importRoots: importRoots(d) }); }), }, "/api/import/jobs": { @@ -42,7 +42,7 @@ export function importRoutes(deps: () => Deps): Feature["routes"] { const options: Record = {}; for (const k of OPTION_KEYS) if (body?.options?.[k] !== undefined && body.options[k] !== "") options[k] = body.options[k]; if (kind === "transcribe") { - if (!d.env.STC_TRANSCRIBE_URL) return json({ error: "STC_TRANSCRIBE_URL is not set" }, 409); + if (!d.transcribeUrl) return json({ error: "STC_TRANSCRIBE_URL is not set" }, 409); if (typeof options.courseId !== "string") return json({ error: "courseId is required" }, 400); return json({ id: await d.ctx.jobs.enqueue("import.transcribe", { courseId: options.courseId }) }, 202); } diff --git a/packages/feature-import/src/web.ts b/packages/feature-import/src/web.ts index 369fa85..5a013b4 100644 --- a/packages/feature-import/src/web.ts +++ b/packages/feature-import/src/web.ts @@ -3,5 +3,5 @@ import type { WebFeature } from "@stc/contracts"; /** Browser-safe entry (no server imports): apps/web imports this as "@stc/feature-import/web". */ export const importWeb: WebFeature = { name: "import", - routes: [{ path: "/admin/import", element: () => import("./web/ImportPage"), nav: { label: "Import", area: "admin" } }], + routes: [{ path: "/admin/import", element: () => import("./web/ImportPage"), nav: { label: "Import", area: "admin", roles: ["owner"] } }], }; diff --git a/packages/feature-import/test/harness.ts b/packages/feature-import/test/harness.ts index ff22cc2..fe2a8ea 100644 --- a/packages/feature-import/test/harness.ts +++ b/packages/feature-import/test/harness.ts @@ -2,6 +2,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import type { JobContext, ServerContext } from "@stc/contracts"; import { createCourseRepository } from "@stc/course-format"; +import { resolveConfig } from "@stc/core"; import type { Deps } from "../src/context"; import { PYTHON, tmp } from "./fixtures"; @@ -13,10 +14,14 @@ export async function makeEnv(env: Record = {}): Pro const queued: TestEnv["queued"] = []; const progress: TestEnv["progress"] = []; const ctx = { - config: { port: 0, databaseUrl: "sqlite://:memory:", dataDir, courseDirs: [join(dataDir, "courses")], secret: "t", publicUrl: "http://x" }, + config: resolveConfig({ databaseUrl: "sqlite://:memory:", dataDir, courseDirs: [join(dataDir, "courses")], secret: "t", publicUrl: "http://x" }), courses, jobs: { enqueue: async (kind: string, payload: unknown) => { queued.push({ kind, payload }); return "j1"; }, get: async () => null, list: async () => [] }, } as unknown as ServerContext; const jc: JobContext = { signal: new AbortController().signal, reportProgress: async (f, m) => { progress.push([f, m]); } }; - return { deps: { ctx, env: { STC_IMPORT_ROOTS: tmpdir(), ...env }, tools: { ffmpeg: "ffmpeg", ffprobe: "ffprobe" }, python: PYTHON }, dataDir, queued, jc, progress }; + const deps: Deps = { + ctx, env, tools: { ffmpeg: "ffmpeg", ffprobe: "ffprobe" }, python: PYTHON, + importRoots: (env.STC_IMPORT_ROOTS ?? tmpdir()).split(":"), transcribeUrl: env.STC_TRANSCRIBE_URL ?? null, + }; + return { deps, dataDir, queued, jc, progress }; } diff --git a/packages/feature-import/test/packt-video.test.ts b/packages/feature-import/test/packt-video.test.ts index 6ca5ab5..5d0e681 100644 --- a/packages/feature-import/test/packt-video.test.ts +++ b/packages/feature-import/test/packt-video.test.ts @@ -58,3 +58,13 @@ test("transcribe option queues the transcription job when a URL is configured", const r = await importPacktVideo(env.deps, { path: zip, options: { transcribe: true } }, env.jc); expect(env.queued).toEqual([{ kind: "import.transcribe", payload: { courseId: r.courseId } }]); }); + +test("an id the studio already uses (studio/repos/.git) is refused, and never picked automatically", async () => { + const { mkdir } = await import("node:fs/promises"); + const env = await makeEnv(); + await mkdir(join(env.deps.ctx.config.studioReposDir, "taken.git"), { recursive: true }); + const zip = await makePacktZip(await tmp()); + await expect(importPacktVideo(env.deps, { path: zip, options: { title: "X", courseId: "taken" } }, env.jc)).rejects.toThrow(/already exists/); + const r = await importPacktVideo(env.deps, { path: zip, options: { title: "Taken" } }, env.jc); + expect(r.courseId).toBe("taken-2"); +}); diff --git a/packages/feature-import/test/routes.test.ts b/packages/feature-import/test/routes.test.ts index 3d2f359..5fc3994 100644 --- a/packages/feature-import/test/routes.test.ts +++ b/packages/feature-import/test/routes.test.ts @@ -2,7 +2,7 @@ import { expect } from "bun:test"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AuthUser, ServerContext } from "@stc/contracts"; -import { createApp, createJobQueue } from "@stc/core"; +import { createApp, createJobQueue, resolveConfig } from "@stc/core"; import { createCourseRepository } from "@stc/course-format"; import { forEachDb } from "../../db/test/helpers"; import { importFeature } from "../src"; @@ -16,11 +16,16 @@ async function setup(db: any, env: Record = {}) { const dataDir = await tmp("stc-data-"); const courses = await createCourseRepository([join(dataDir, "courses")]); const ctx = { - config: { port: 0, databaseUrl: "", dataDir, courseDirs: [join(dataDir, "courses")], secret: "t", publicUrl: "" }, + config: resolveConfig({ + databaseUrl: "", dataDir, courseDirs: [join(dataDir, "courses")], secret: "t", publicUrl: "", + python: process.env.STC_PYTHON || "python3", + importRoots: "STC_IMPORT_ROOTS" in env ? (env.STC_IMPORT_ROOTS ? [env.STC_IMPORT_ROOTS] : undefined) : [tmpdir()], + transcribeUrl: env.STC_TRANSCRIBE_URL ?? null, + }), db, courses, auth: { userFromRequest: async (req: Request) => ({ owner, learner }[req.headers.get("x-user") ?? ""] ?? null) }, } as unknown as ServerContext; - const feature = importFeature(ctx, { env: { STC_PYTHON: process.env.STC_PYTHON, STC_IMPORT_ROOTS: tmpdir(), ...env } }); + const feature = importFeature(ctx); const queue = createJobQueue(db, feature.jobs!, { pollMs: 10 }); ctx.jobs = queue; const app = createApp(ctx, [feature]); diff --git a/packages/feature-learner/package.json b/packages/feature-learner/package.json index bf47b54..f64690f 100644 --- a/packages/feature-learner/package.json +++ b/packages/feature-learner/package.json @@ -13,7 +13,7 @@ }, "exports": { ".": "./src/index.ts", - "./web": "./web/src/index.ts" + "./web": "./src/web/index.ts" }, "devDependencies": { "@types/sanitize-html": "^2.16.2" diff --git a/packages/feature-learner/src/index.ts b/packages/feature-learner/src/index.ts index c01208d..cad2f30 100644 --- a/packages/feature-learner/src/index.ts +++ b/packages/feature-learner/src/index.ts @@ -1,6 +1,7 @@ // Server export { migrationsDirs, runMigrations } from "./server/migrate"; export { learnerFeature } from "./server/feature"; +export { userIdentity } from "./server/identity"; // Browser (also available without the server half as "@stc/feature-learner/web") export { learnerWeb, LessonView, type LessonViewProps, ensureLearnerStyles } from "./web"; // Shared JSON shapes diff --git a/packages/feature-learner/src/server/access.ts b/packages/feature-learner/src/server/access.ts index 64fbbec..e0a7bc9 100644 --- a/packages/feature-learner/src/server/access.ts +++ b/packages/feature-learner/src/server/access.ts @@ -30,8 +30,8 @@ export function canEdit(user: AuthUser | null, courseId: string, idx: AccessInde /** * Learners see courses with a manifest that core `canView` allows; owners also see broken ones. The author input - * is per-course authorship only, the same as core's media route, so every route applies one rule. (The global - * `author` role grants viewing only once Phase 2 extends `canView`; contract request learner §6.) + * is per-course authorship (course_authors), the same as core's media route; core `canView` itself lets the owner + * and the global `author` role view every course (R14), so every route applies one rule. */ export function visible(user: AuthUser | null, c: LoadedCourse, idx: AccessIndex): boolean { if (!c.manifest) return isOwner(user); diff --git a/packages/feature-learner/src/server/catalog.ts b/packages/feature-learner/src/server/catalog.ts index e2395b7..3db40cf 100644 --- a/packages/feature-learner/src/server/catalog.ts +++ b/packages/feature-learner/src/server/catalog.ts @@ -7,6 +7,7 @@ import type { } from "../shared/types"; import { mediaUrl } from "../shared/types"; import { accessIndex, courseFor, isOwner, visible } from "./access"; +import { coverUrlFor } from "./covers"; import { moduleLocks } from "./gating"; import { json, notFound, param } from "./http"; @@ -93,7 +94,7 @@ export function libraryRoutes(ctx: ServerContext): Feature["routes"] { const doneCount = lessons.filter((l) => progress[l.id]?.state === "done").length; return { id: m.id, title: m.title, pitch: m.pitch ?? null, authors: m.authors ?? [], - coverUrl: m.cover && ctx.courses.resolvePath(m.id, m.cover) ? mediaUrl(m.id, m.cover) : null, + coverUrl: coverUrlFor(ctx, m), accent: m.accent ?? null, formats: formatsOf(m), lessonCount: lessons.length, minutes, collections: m.collections ?? [], tracks: (m.tracks ?? []).map((t) => t.label), source: m.source?.kind ?? null, addedAt: addedAt(c), status: c.status, progress: lessons.length ? doneCount / lessons.length : 0, doneCount, @@ -204,7 +205,7 @@ export function libraryRoutes(ctx: ServerContext): Feature["routes"] { })); const payload: CoursePayload = { id: m.id, title: m.title, pitch: m.pitch ?? null, authors: m.authors ?? [], accent: m.accent ?? null, - coverUrl: m.cover && ctx.courses.resolvePath(m.id, m.cover) ? mediaUrl(m.id, m.cover) : null, + coverUrl: coverUrlFor(ctx, m), tracks: m.tracks ?? [], gating: m.gating, source: m.source?.kind ?? null, status: course.status, modules, progress, canEdit: found.canEdit, signedIn: !!rc.user, diagnostics: found.canEdit ? course.diagnostics.map((d) => ({ level: d.level, lessonId: d.lessonId, path: d.path, message: d.message })) : [], diff --git a/packages/feature-learner/src/server/covers.ts b/packages/feature-learner/src/server/covers.ts new file mode 100644 index 0000000..22c2f5a --- /dev/null +++ b/packages/feature-learner/src/server/covers.ts @@ -0,0 +1,47 @@ +import { readdirSync } from "node:fs"; +import { join } from "node:path"; +import type { Feature, ServerContext } from "@stc/contracts"; +import { cachePaths, serveMedia } from "@stc/core"; +import type { Manifest } from "@stc/course-format"; +import { mediaUrl } from "../shared/types"; +import { courseFor } from "./access"; +import { notFound } from "./http"; + +const CANDIDATE = /^(frame-\d+|deck-[A-Za-z0-9._-]+)\.jpg$/; + +/** The importer's first cover candidate (`/covers//frame-N.jpg | deck-.jpg`), if any. */ +export function coverCandidate(ctx: ServerContext, courseId: string): string | null { + let dir: string; + try { dir = cachePaths(ctx.config).coversDir(courseId); } catch { return null; } + let names: string[]; + try { names = readdirSync(dir).filter((n) => CANDIDATE.test(n)); } catch { return null; } + if (!names.length) return null; + // Prefer video frames (the importer's own first choice), then deck covers; numeric order within frames. + names.sort((a, b) => (a.startsWith("frame-") === b.startsWith("frame-") ? a.localeCompare(b, "en", { numeric: true }) : a.startsWith("frame-") ? -1 : 1)); + return join(dir, names[0]!); +} + +export const coverRoute = (courseId: string) => `/api/courses/${encodeURIComponent(courseId)}/cover`; + +/** The manifest's cover if it resolves inside the course; else the importer's cached candidate; else null. */ +export function coverUrlFor(ctx: ServerContext, m: Manifest): string | null { + if (m.cover && ctx.courses.resolvePath(m.id, m.cover)) return mediaUrl(m.id, m.cover); + return coverCandidate(ctx, m.id) ? coverRoute(m.id) : null; +} + +/** Serves the cached cover candidate to users who may view the course (core canView; 404 otherwise). */ +export function coverRoutes(ctx: ServerContext): Feature["routes"] { + return { + "/api/courses/:courseId/cover": { + GET: async (req, rc) => { + const found = await courseFor(ctx, rc.user, rc.params.courseId ?? ""); + if (!found) return notFound(); + const file = coverCandidate(ctx, found.course.id); + if (!file) return notFound(); + const res = await serveMedia(req, file); + res.headers.set("Cache-Control", "private, max-age=300"); + return res; + }, + }, + }; +} diff --git a/packages/feature-learner/src/server/feature.ts b/packages/feature-learner/src/server/feature.ts index ebfe694..186e158 100644 --- a/packages/feature-learner/src/server/feature.ts +++ b/packages/feature-learner/src/server/feature.ts @@ -1,9 +1,9 @@ -import type { AuthService, Feature, RouteHandler, ServerContext } from "@stc/contracts"; +import type { AuthService, Feature, ServerContext } from "@stc/contracts"; import { createAuth } from "./auth"; -import { runMigrations } from "./migrate"; import { json } from "./http"; import { adminRoutes } from "./admin"; import { libraryRoutes } from "./catalog"; +import { coverRoutes } from "./covers"; import { lessonRoutes } from "./lesson"; import { userDataRoutes } from "./userdata"; import { codeRoutes } from "./code"; @@ -16,9 +16,8 @@ import { runnerFrameRoutes } from "./runnerFrame"; * Returns the routes plus the `AuthService` that replaces the server's anonymous one. */ export function learnerFeature(ctx: ServerContext): Feature & { auth: AuthService } { - const ready = runMigrations(ctx.db); - ready.catch((e) => console.error("[learner] migrations failed", e)); - const auth = createAuth(ctx, ready); + // The server runs this package's migrations (migrationsDirs) in core migrate() before any feature is built. + const auth = createAuth(ctx, Promise.resolve()); const routes: Feature["routes"] = { "/api/auth/*": { GET: (req, rc) => auth.handler(req, rc.user), @@ -30,6 +29,7 @@ export function learnerFeature(ctx: ServerContext): Feature & { auth: AuthServic }, ...adminRoutes(ctx, auth.service.invalidate), ...libraryRoutes(ctx), + ...coverRoutes(ctx), ...lessonRoutes(ctx), ...userDataRoutes(ctx), ...codeRoutes(ctx), @@ -37,11 +37,5 @@ export function learnerFeature(ctx: ServerContext): Feature & { auth: AuthServic ...assetRoutes(), ...runnerFrameRoutes(), }; - // Every handler waits for the learner migrations (a no-op once Phase 2 runs them in migrate()). - for (const methods of Object.values(routes)) { - for (const [m, h] of Object.entries(methods)) { - (methods as Record)[m] = async (req, rc) => { await ready; return h!(req, rc); }; - } - } return { name: "learner", routes, auth: auth.service }; } diff --git a/packages/feature-learner/src/server/identity.ts b/packages/feature-learner/src/server/identity.ts new file mode 100644 index 0000000..2c892a6 --- /dev/null +++ b/packages/feature-learner/src/server/identity.ts @@ -0,0 +1,9 @@ +import type { Db } from "@stc/db"; + +/** A user's display name and email (for commit authorship, e.g. the AI generator's studio drafts); null if unknown. */ +export async function userIdentity(db: Db, userId: string): Promise<{ name: string; email: string } | null> { + const rows = await db.sql`SELECT name, email FROM auth_user WHERE id = ${userId}`; + const r = rows[0] as { name: string | null; email: string } | undefined; + if (!r) return null; + return { name: r.name?.trim() || r.email, email: r.email }; +} diff --git a/packages/feature-learner/src/server/migrate.ts b/packages/feature-learner/src/server/migrate.ts index ecf343d..1dfacba 100644 --- a/packages/feature-learner/src/server/migrate.ts +++ b/packages/feature-learner/src/server/migrate.ts @@ -1,33 +1,10 @@ -import { readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; -import type { Db } from "@stc/db"; +import { migrate, type Db } from "@stc/db"; -/** Folders holding this package's migrations; Phase 2 wires these into `migrate()`. */ +/** Folders holding this package's migrations; the server passes them to core `migrate()` (contract request 2). */ export const migrationsDirs: string[] = [join(import.meta.dir, "..", "..", "migrations")]; -function statements(text: string): string[] { - const noComments = text.split("\n").filter((l) => !l.trim().startsWith("--")).join("\n"); - return noComments.split(";").map((s) => s.trim()).filter(Boolean); -} - -/** - * Applies the learner migrations with the same bookkeeping as `@stc/db`'s `migrate()` - * (table `schema_migrations`, keyed by file name). Safe to call repeatedly. - */ -export async function runMigrations(db: Db, dirs: string[] = migrationsDirs): Promise { - await db.sql.unsafe("CREATE TABLE IF NOT EXISTS schema_migrations (name TEXT PRIMARY KEY, applied_at TEXT NOT NULL)"); - const done = new Set((await db.sql`SELECT name FROM schema_migrations`).map((r: { name: string }) => r.name)); - const applied: string[] = []; - for (const dir of dirs) { - for (const name of readdirSync(dir).filter((f) => f.endsWith(".sql")).sort()) { - if (done.has(name)) continue; - const stmts = statements(readFileSync(join(dir, name), "utf8")); - await db.sql.begin(async (tx) => { - for (const s of stmts) await tx.unsafe(s); - await tx`INSERT INTO schema_migrations (name, applied_at) VALUES (${name}, ${new Date().toISOString()})`; - }); - applied.push(name); - } - } - return applied; +/** Core migrations plus the learner's, through core `migrate()` (tests and tools; the server migrates once at startup). */ +export function runMigrations(db: Db): Promise { + return migrate(db, { "feature-learner": migrationsDirs }); } diff --git a/packages/feature-learner/src/web/api.ts b/packages/feature-learner/src/web/api.ts index 9826e1a..034b514 100644 --- a/packages/feature-learner/src/web/api.ts +++ b/packages/feature-learner/src/web/api.ts @@ -182,8 +182,7 @@ export function fmtSize(n: number | null | undefined): string { if (n < 1024 * 1024) return `${Math.round(n / 1024)} KB`; return `${(n / 1024 / 1024).toFixed(1)} MB`; } -export const lessonHref = (courseId: string, lessonId: string) => `/c/${encodeURIComponent(courseId)}/${encodeURIComponent(lessonId)}`; -export const courseHref = (courseId: string) => `/c/${encodeURIComponent(courseId)}`; +export { lessonUrl as lessonHref, courseUrl as courseHref } from "@stc/contracts"; export async function copyText(text: string): Promise { try { await navigator.clipboard.writeText(text); return true; } catch { diff --git a/packages/feature-learner/src/web/index.ts b/packages/feature-learner/src/web/index.ts index 4fcd26f..4144ed4 100644 --- a/packages/feature-learner/src/web/index.ts +++ b/packages/feature-learner/src/web/index.ts @@ -1,6 +1,7 @@ -import type { WebFeature } from "@stc/contracts"; +import { COURSE_ROUTE, LESSON_ROUTE, type WebFeature } from "@stc/contracts"; export { LessonView, type LessonViewProps } from "./lesson/LessonView"; +export type { LessonPayload } from "../shared/types"; export { ensureStyles as ensureLearnerStyles } from "./api"; /** Learner routes: library, course pages, lessons, notes and the account pages. */ @@ -8,9 +9,9 @@ export const learnerWeb: WebFeature = { name: "learner", routes: [ { path: "/", element: () => import("./library/LibraryPage") }, - { path: "/c/:courseId", element: () => import("./course/pages").then((m) => ({ default: m.CoursePage })) }, + { path: COURSE_ROUTE, element: () => import("./course/pages").then((m) => ({ default: m.CoursePage })) }, { path: "/c/:courseId/notes", element: () => import("./course/pages").then((m) => ({ default: m.NotesPage })) }, - { path: "/c/:courseId/:lessonId", element: () => import("./course/pages").then((m) => ({ default: m.LessonPage })) }, + { path: LESSON_ROUTE, element: () => import("./course/pages").then((m) => ({ default: m.LessonPage })) }, { path: "/account", element: () => import("./account/AccountPage") }, { path: "/account/admin", element: () => import("./account/AdminPage") }, ], diff --git a/packages/feature-learner/src/web/lesson/LessonView.tsx b/packages/feature-learner/src/web/lesson/LessonView.tsx index 2e4735b..7b1ac43 100644 --- a/packages/feature-learner/src/web/lesson/LessonView.tsx +++ b/packages/feature-learner/src/web/lesson/LessonView.tsx @@ -24,6 +24,8 @@ export interface LessonViewProps { signedIn?: boolean; /** Studio preview: no notes, no saved attempts, no progress reports. */ preview?: boolean; + /** URL for a course-relative file in the lesson text (the studio preview points these at draft files). */ + mediaUrl?: (courseRel: string) => string; onProgress?: (state: ProgressState, positionSeconds?: number) => void; /** An exercise attempt was saved (the server already updated progress). */ onSaved?: () => void; @@ -85,12 +87,12 @@ function TrackSwitch({ lesson, track, onChange }: { lesson: LessonPayload; track ); } -function useLessonHtml(lesson: LessonPayload, track: string | null): string | null { +function useLessonHtml(lesson: LessonPayload, track: string | null, media?: (courseRel: string) => string): string | null { return useMemo(() => { const md = (track && lesson.trackMarkdown[track]) || lesson.markdown; if (!md) return null; - return renderLessonHtml(md, { courseId: lesson.courseId, baseFile: lesson.baseFile, tracks: lesson.tracks, track, lessonFiles: lesson.lessonFiles, stripFirstHeading: true }); - }, [lesson, track]); + return renderLessonHtml(md, { courseId: lesson.courseId, baseFile: lesson.baseFile, tracks: lesson.tracks, track, lessonFiles: lesson.lessonFiles, stripFirstHeading: true, mediaUrl: media }); + }, [lesson, track, media]); } function Decks({ lesson, notes, initialSlide, keyboard }: { lesson: LessonPayload; notes: NotesApi | null; initialSlide?: { deckId: string; page: number } | null; keyboard: boolean }) { @@ -109,7 +111,7 @@ function Decks({ lesson, notes, initialSlide, keyboard }: { lesson: LessonPayloa function TextLesson(p: LessonViewProps & { chap: string; track: string | null; notes: NotesApi }) { const { lesson, track } = p; - const html = useLessonHtml(lesson, track); + const html = useLessonHtml(lesson, track, p.mediaUrl); const cmd = localCommandFor(lesson.localCommand, track); const minutes = lesson.lesson.minutes; const hasMeta = !!minutes || lesson.tracks.length > 1; @@ -228,9 +230,9 @@ function useVideoProgress(v: React.RefObject, p: Lesson function ExerciseLesson(p: LessonViewProps & { chap: string; track: string | null }) { const { lesson, track } = p; - const html = useLessonHtml(lesson, track); + const html = useLessonHtml(lesson, track, p.mediaUrl); const hintsHtml = useMemo(() => (lesson.hints.length - ? renderHintsHtml(lesson.hints, { courseId: lesson.courseId, baseFile: lesson.baseFile, tracks: lesson.tracks, track, lessonFiles: lesson.lessonFiles }) + ? renderHintsHtml(lesson.hints, { courseId: lesson.courseId, baseFile: lesson.baseFile, tracks: lesson.tracks, track, lessonFiles: lesson.lessonFiles, mediaUrl: p.mediaUrl }) : null), [lesson, track]); const cmd = localCommandFor(lesson.localCommand, track); return ( diff --git a/packages/feature-learner/src/web/markdown.ts b/packages/feature-learner/src/web/markdown.ts index fa9e8d2..d05dd7a 100644 --- a/packages/feature-learner/src/web/markdown.ts +++ b/packages/feature-learner/src/web/markdown.ts @@ -24,6 +24,8 @@ export interface RenderOptions { lessonFiles?: Record; /** Drop a leading heading (the page shows the lesson title itself). */ stripFirstHeading?: boolean; + /** URL for a course-relative file; default `/api/media//` (the studio preview serves draft files). */ + mediaUrl?: (courseRel: string) => string; } const escapeHtml = (s: string) => s.replace(/&/g, "&").replace(//g, ">").replace(/"/g, """); @@ -226,7 +228,7 @@ export function resolveHref(href: string, opts: RenderOptions, kind: "link" | "i if (rel === null) return null; const lessonId = kind === "link" ? opts.lessonFiles?.[rel] : undefined; if (lessonId) return { url: `/c/${encodeURIComponent(opts.courseId)}/${encodeURIComponent(lessonId)}${hash ? `#${hash}` : ""}`, external: false }; - return { url: mediaUrl(opts.courseId, rel) + (hash ? `#${hash}` : ""), external: false }; + return { url: (opts.mediaUrl ? opts.mediaUrl(rel) : mediaUrl(opts.courseId, rel)) + (hash ? `#${hash}` : ""), external: false }; } export function renderLessonHtml(md: string, opts: RenderOptions): string { diff --git a/packages/feature-learner/test/app.ts b/packages/feature-learner/test/app.ts index 9bc48b6..706573f 100644 --- a/packages/feature-learner/test/app.ts +++ b/packages/feature-learner/test/app.ts @@ -2,7 +2,7 @@ import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import type { AppConfig, JobQueue, ServerContext } from "@stc/contracts"; -import { createApp } from "@stc/core"; +import { createApp, resolveConfig } from "@stc/core"; import { createCourseRepository } from "@stc/course-format"; import type { Db } from "@stc/db"; import { learnerFeature } from "../src/server/feature"; @@ -13,10 +13,10 @@ export const ORIGIN = "http://localhost:3000"; const fakeJobs: JobQueue = { enqueue: async () => "j", get: async () => null, list: async () => [] }; export async function makeApp(db: Db, courseDirs: string[] = [FIXTURES]) { - const config: AppConfig = { + const config: AppConfig = resolveConfig({ port: 0, databaseUrl: "", dataDir: mkdtempSync(join(tmpdir(), "stc-learner-")), courseDirs, secret: "test-secret-0123456789abcdef0123456789", publicUrl: ORIGIN, - }; + }); const ctx: ServerContext = { config, db, courses: await createCourseRepository(courseDirs), jobs: fakeJobs, auth: { userFromRequest: async () => null }, diff --git a/packages/feature-learner/test/covers.test.ts b/packages/feature-learner/test/covers.test.ts new file mode 100644 index 0000000..5ae7ee5 --- /dev/null +++ b/packages/feature-learner/test/covers.test.ts @@ -0,0 +1,40 @@ +import { expect } from "bun:test"; +import { join } from "node:path"; +import { forEachDb } from "./helpers"; +import { FIXTURES, makeApp } from "./app"; +import { extraCourses } from "./courses"; + +const JPEG = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0, 0x10, 0x4a, 0x46, 0x49, 0x46, 0, 1, 0xff, 0xd9]); + +forEachDb("the library falls back to the importer's cover candidate, served only to users who can view the course", async (db) => { + const dir = await extraCourses(); + const { call, signUp, ctx } = await makeApp(db, [FIXTURES, dir]); + const covers = (id: string) => join(ctx.config.cacheDir, "covers", id); + await Bun.write(join(covers("sample"), "deck-d1.jpg"), JPEG); + await Bun.write(join(covers("sample"), "frame-2.jpg"), JPEG); + await Bun.write(join(covers("sample"), "frame-10.jpg"), new Uint8Array([1])); + await Bun.write(join(covers("secret"), "frame-1.jpg"), JPEG); + await Bun.write(join(covers("sample"), "notes.txt"), "not a cover"); + + const lib = async (cookie?: string) => ((await (await call("GET", "/api/library", { cookie })).json()) as { courses: { id: string; coverUrl: string | null }[] }).courses; + const sample = (await lib()).find((c) => c.id === "sample")!; + expect(sample.coverUrl).toBe("/api/courses/sample/cover"); + const r = await call("GET", "/api/courses/sample/cover"); + expect(r.status).toBe(200); + expect(r.headers.get("content-type")).toBe("image/jpeg"); + expect(r.headers.get("x-content-type-options")).toBe("nosniff"); + expect(new Uint8Array(await r.arrayBuffer())).toEqual(JPEG); // frame-2 (frames first, numeric order), not frame-10 or the deck + expect((await (await call("GET", "/api/courses/sample")).json()).coverUrl).toBe("/api/courses/sample/cover"); + + // restricted: 404 for anonymous users and learners without access, 200 for the owner + const owner = await signUp("owner@x.test"); + const learner = await signUp("lea@x.test"); + expect((await call("GET", "/api/courses/secret/cover")).status).toBe(404); + expect((await call("GET", "/api/courses/secret/cover", { cookie: learner })).status).toBe(404); + expect((await call("GET", "/api/courses/secret/cover", { cookie: owner })).status).toBe(200); + expect((await lib(owner)).find((c) => c.id === "secret")!.coverUrl).toBe("/api/courses/secret/cover"); + // no candidate: no cover, and the route is 404 + expect((await lib(owner)).find((c) => c.id === "gated")!.coverUrl).toBeNull(); + expect((await call("GET", "/api/courses/gated/cover", { cookie: owner })).status).toBe(404); + expect((await call("GET", "/api/courses/..%2Fsample/cover", { cookie: owner })).status).toBe(404); +}); diff --git a/packages/feature-learner/test/visibility.test.ts b/packages/feature-learner/test/visibility.test.ts index 1d9ec5f..9b3aef1 100644 --- a/packages/feature-learner/test/visibility.test.ts +++ b/packages/feature-learner/test/visibility.test.ts @@ -3,12 +3,13 @@ import { forEachDb } from "./helpers"; import { FIXTURES, makeApp } from "./app"; import { extraCourses } from "./courses"; -forEachDb("one visibility rule: the global author role alone does not open a restricted course (I1)", async (db) => { +forEachDb("one visibility rule (core canView, R14): owner, the global author role and per-course authors open a restricted course; learners do not", async (db) => { const dir = await extraCourses(); const { call, signUp } = await makeApp(db, [FIXTURES, dir]); const owner = await signUp("owner@x.test"); const author = await signUp("author@x.test"); const coAuthor = await signUp("co@x.test"); + const learner = await signUp("lea@x.test"); const users = (await (await call("GET", "/api/admin/users", { cookie: owner })).json()) as { id: string; email: string }[]; const id = (e: string) => users.find((u) => u.email === e)!.id; expect((await call("PUT", `/api/admin/users/${id("author@x.test")}/roles`, { cookie: owner, body: { roles: ["author"] } })).status).toBe(200); @@ -16,12 +17,15 @@ forEachDb("one visibility rule: the global author role alone does not open a res const paths = ["/api/courses/secret", "/api/courses/secret/lessons/s1", "/api/media/secret/a.md"]; for (const p of paths) { - expect([p, (await call("GET", p, { cookie: author })).status]).toEqual([p, 404]); + expect([p, (await call("GET", p, { cookie: author })).status]).toEqual([p, 200]); expect([p, (await call("GET", p, { cookie: coAuthor })).status]).toEqual([p, 200]); + expect([p, (await call("GET", p, { cookie: learner })).status]).toEqual([p, 404]); + expect([p, (await call("GET", p)).status]).toEqual([p, 404]); } const lib = async (c: string) => ((await (await call("GET", "/api/library", { cookie: c })).json()) as { courses: { id: string }[] }).courses.map((x) => x.id); - expect(await lib(author)).not.toContain("secret"); + expect(await lib(author)).toContain("secret"); expect(await lib(coAuthor)).toContain("secret"); - const search = (await (await call("GET", "/api/library/search?q=hidden", { cookie: author })).json()) as unknown[]; + expect(await lib(learner)).not.toContain("secret"); + const search = (await (await call("GET", "/api/library/search?q=hidden", { cookie: learner })).json()) as unknown[]; expect(search).toEqual([]); }); diff --git a/packages/feature-learner/web/src/index.ts b/packages/feature-learner/web/src/index.ts deleted file mode 100644 index 4c833d1..0000000 --- a/packages/feature-learner/web/src/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -// Browser entry "@stc/feature-learner/web". The repo maps "@stc/*" to "packages/*/src", so this subpath -// resolves here (tsc and Bun alike). It re-exports the web half only: the package index also carries the -// server (Better Auth, SSO), which cannot be bundled for the browser. -export * from "../../src/web"; diff --git a/packages/feature-studio/src/server/draft-writer.ts b/packages/feature-studio/src/server/draft-writer.ts index a4100d3..ca63e5e 100644 --- a/packages/feature-studio/src/server/draft-writer.ts +++ b/packages/feature-studio/src/server/draft-writer.ts @@ -1,4 +1,5 @@ import type { AppConfig } from "@stc/contracts"; +import { storeDirs } from "./paths"; import { createStudioStore, StudioError, safeAuthor, type Identity } from "./store"; /** Programmatic draft API for other packages (WP-D's studio generator). Writes only to `draft/`, never publishes. */ @@ -16,8 +17,11 @@ export interface DraftWriterOptions { identity?: (authorId: string) => Promise | Identity | null; } -export function createDraftWriter(ctx: { config: Pick }, opts: DraftWriterOptions = {}): DraftWriter { - const store = createStudioStore({ dataDir: ctx.config.dataDir }); +export function createDraftWriter( + ctx: { config: Pick & Partial> }, + opts: DraftWriterOptions = {}, +): DraftWriter { + const store = createStudioStore(storeDirs(ctx.config)); return { async writeDraftFiles(courseId, authorId, files, message) { if (!store.isManaged(courseId)) throw new StudioError(`course ${courseId} is not managed by the studio`, 404); diff --git a/packages/feature-studio/src/server/feature.ts b/packages/feature-studio/src/server/feature.ts index e691487..7a321ae 100644 --- a/packages/feature-studio/src/server/feature.ts +++ b/packages/feature-studio/src/server/feature.ts @@ -1,21 +1,20 @@ import { existsSync, realpathSync } from "node:fs"; import { basename, extname, resolve, sep } from "node:path"; import type { AuthUser, Feature, RequestContext, RouteHandler, ServerContext } from "@stc/contracts"; -import { serveMedia } from "@stc/core"; +import { canView, serveMedia } from "@stc/core"; import { ManifestSchema, parseManifest, resolveInside, type LoadedCourse } from "@stc/course-format"; import { aclRepo } from "@stc/db"; import { createDraftWriter, type DraftWriter } from "./draft-writer"; -import { studioWorktreesDir } from "./paths"; +import { storeDirs, studioWorktreesDir } from "./paths"; import { checkRelPath, createStudioStore, MANIFEST, ownerRequiredFor, StudioError, type FileWrite, type StudioStore } from "./store"; import { createPublishedWatcher, type PublishedWatcher } from "./watch"; export const OUTSIDE_MESSAGE = "This course is managed outside STCourses"; export const IMPORTED_MESSAGE = "Imported course: read-only in the studio for now"; -/** Upload cap in bytes: STC_UPLOAD_MAX_MB, default 100 (review I2). */ -export function uploadMaxBytes(env: Record = process.env): number { - const mb = Number(env.STC_UPLOAD_MAX_MB ?? "100"); - return (Number.isFinite(mb) && mb > 0 ? mb : 100) * 1024 * 1024; +/** Upload cap in bytes from AppConfig.uploadMaxMb (STC_UPLOAD_MAX_MB, default 100; review I2). */ +export function uploadMaxBytes(mb: number | undefined): number { + return (typeof mb === "number" && Number.isFinite(mb) && mb > 0 ? mb : 100) * 1024 * 1024; } /** Server-side allowlist by extension for uploads: text, documents, images, media, archives and code. */ @@ -73,19 +72,29 @@ function manifestToYaml(m: unknown): string { } export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = {}): StudioFeature { - const store = createStudioStore({ dataDir: ctx.config.dataDir }); + const store = createStudioStore(storeDirs(ctx.config)); const watcher = createPublishedWatcher(store, ctx.courses, { intervalMs: opts.watchIntervalMs }); const acl = aclRepo(ctx.db); - const wtDir = resolve(studioWorktreesDir(ctx.config.dataDir)); + const wtDir = resolve(ctx.config.studioWorktreesDir ?? studioWorktreesDir(ctx.config.dataDir)); const draftWriter = createDraftWriter(ctx); const hasRole = (u: AuthUser) => u.roles.includes("owner") || u.roles.includes("author"); const authoredCourses = async (u: AuthUser): Promise => (await ctx.db.sql`SELECT course_id FROM course_authors WHERE user_id = ${u.id}`).map((r: { course_id: string }) => r.course_id); - const canEdit = async (u: AuthUser, courseId: string) => hasRole(u) || (await acl.isAuthor(courseId, u.id)); + /** + * An authoring right (owner, the `author` role, or a course_authors row) and, for a course the server has loaded, + * core `canView` (the one visibility rule, R14). A studio course not loaded yet (e.g. its published manifest is + * broken) has no visibility to check. + */ + const canEdit = async (u: AuthUser, courseId: string) => { + const isAuthor = await acl.isAuthor(courseId, u.id); + if (!hasRole(u) && !isAuthor) return false; + const c = ctx.courses.get(courseId); + return !c || canView(u, c, await acl.listForCourse(courseId), isAuthor); + }; const isStudioCourse = (c: LoadedCourse) => resolve(c.manifestPath).startsWith(wtDir + sep); const who = (u: AuthUser) => ({ id: u.id, name: u.name || u.email || u.id, email: u.email || `${u.id}@users.stcourses.invalid`, owner: u.roles.includes("owner") }); - const importedRoot = (() => { const d = resolve(ctx.config.dataDir, "courses"); return existsSync(d) ? realpathSync(d) : d; })(); + const importedRoot = (() => { const d = resolve(ctx.config.importedCoursesDir ?? resolve(ctx.config.dataDir, "courses")); return existsSync(d) ? realpathSync(d) : d; })(); /** Why a course that is not studio-managed is read-only here. */ const readOnlyMessage = (c: LoadedCourse): string => { const kind = c.manifest?.source?.kind; @@ -250,7 +259,7 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = { "/api/studio/courses/:courseId/upload": { POST: course(async (req, _rc, user, courseId) => { - const max = uploadMaxBytes(); + const max = uploadMaxBytes(ctx.config.uploadMaxMb); const declared = Number(req.headers.get("content-length") ?? "0"); if (declared > max) return fail(413, `file too large (limit ${Math.round(max / 1048576)} MB)`); const bytes = await readCapped(req, max); diff --git a/packages/feature-studio/src/server/paths.ts b/packages/feature-studio/src/server/paths.ts index 4f954bc..e56283a 100644 --- a/packages/feature-studio/src/server/paths.ts +++ b/packages/feature-studio/src/server/paths.ts @@ -14,10 +14,18 @@ export function studioReposDir(dataDir: string): string { /** * The course dirs the CourseRepository must scan: the configured ones, the importer's plain folders in * `/courses`, and studio courses through their published worktree in `/worktrees`. - * Phase 0's server builds the repository from `config.courseDirs` only; see docs/contract-requests/studio.md. + * Core `loadConfig`/`resolveConfig` already include both in `config.courseDirs`; this stays for callers that build + * an AppConfig by hand. */ -export function studioCourseDirs(config: Pick): string[] { +export function studioCourseDirs(config: Pick & Partial>): string[] { const dirs = config.courseDirs.map((d) => resolve(d)); - for (const d of [resolve(config.dataDir, "courses"), resolve(studioWorktreesDir(config.dataDir))]) if (!dirs.includes(d)) dirs.push(d); + const imported = config.importedCoursesDir ?? resolve(config.dataDir, "courses"); + const worktrees = config.studioWorktreesDir ?? studioWorktreesDir(config.dataDir); + for (const d of [resolve(imported), resolve(worktrees)]) if (!dirs.includes(d)) dirs.push(d); return dirs; } + +/** Store options from the server configuration (core AppConfig owns the layout). */ +export function storeDirs(config: Pick & Partial>) { + return { dataDir: config.dataDir, reposDir: config.studioReposDir, importedDir: config.importedCoursesDir, worktreesDir: config.studioWorktreesDir }; +} diff --git a/packages/feature-studio/src/server/store.ts b/packages/feature-studio/src/server/store.ts index 6d1c3fe..638822a 100644 --- a/packages/feature-studio/src/server/store.ts +++ b/packages/feature-studio/src/server/store.ts @@ -123,6 +123,12 @@ export type Actor = Identity & { id: string; owner?: boolean }; export interface StudioStoreOptions { dataDir: string; + /** AppConfig.studioReposDir (default `/studio/repos`). */ + reposDir?: string; + /** AppConfig.importedCoursesDir (default `/courses`): ids taken by the importer. */ + importedDir?: string; + /** AppConfig.studioWorktreesDir (default `/worktrees`). */ + worktreesDir?: string; publishedBranch?: string; /** Tests only: allow local-path remotes (the HTTP feature never sets this). */ allowLocalRemotes?: boolean; @@ -132,9 +138,9 @@ export function createStudioStore(opts: StudioStoreOptions) { const dataDir = opts.dataDir; const main = opts.publishedBranch ?? "main"; // Studio repos live apart from the importer's plain folders in /courses (controller ruling, review I4). - const reposDir = join(dataDir, "studio", "repos"); - const importedDir = join(dataDir, "courses"); - const worktreesDir = join(dataDir, "worktrees"); + const reposDir = opts.reposDir ?? join(dataDir, "studio", "repos"); + const importedDir = opts.importedDir ?? join(dataDir, "courses"); + const worktreesDir = opts.worktreesDir ?? join(dataDir, "worktrees"); const remoteGuard = opts.allowLocalRemotes ? ["-c", "protocol.ext.allow=never", "-c", "protocol.fd.allow=never"] : REMOTE_GUARD; const paths = (courseId: string) => { diff --git a/packages/feature-studio/src/ui/editors/LessonEditor.tsx b/packages/feature-studio/src/ui/editors/LessonEditor.tsx index 0c630a6..7f5834d 100644 --- a/packages/feature-studio/src/ui/editors/LessonEditor.tsx +++ b/packages/feature-studio/src/ui/editors/LessonEditor.tsx @@ -82,7 +82,7 @@ export function LessonEditor({ ws, lessonId, onGone }: { ws: Workspace; lessonId

      Preview

      - ws.resolveUrl(new URL(rel, `http://x/${mdPath}`).pathname.slice(1))} /> + ws.resolveUrl(new URL(rel, `http://x/${mdPath}`).pathname.slice(1))} />
      ) : null} diff --git a/packages/feature-studio/src/ui/learnerPreview.tsx b/packages/feature-studio/src/ui/learnerPreview.tsx new file mode 100644 index 0000000..210c32f --- /dev/null +++ b/packages/feature-studio/src/ui/learnerPreview.tsx @@ -0,0 +1,28 @@ +import { useMemo } from "react"; +import { extractSection } from "@stc/course-format/markdown"; +import { LessonView, type LessonPayload } from "@stc/feature-learner/web"; +import type { LessonPreviewProps } from "./preview"; + +/** + * The studio's live preview, rendered by the learner's real `LessonView` (contract request studio 3) in preview + * mode: no notes, attempts or progress. Draft files (images, links) resolve through the studio's raw-file route. + * Video lessons preview their transcript; exercises preview their instructions. + */ +export default function LearnerLessonPreview(props: LessonPreviewProps) { + const { courseId, markdown, lesson, file, resolveCourseUrl, resolveUrl } = props; + const payload = useMemo(() => { + const anchored = lesson?.anchor ? extractSection(markdown, lesson.anchor) : null; + const body = lesson?.anchor && anchored === null ? `> Heading anchor \`#${lesson.anchor}\` was not found in this file.\n\n${markdown}` : anchored ?? markdown; + return { + courseId, courseTitle: "", + lesson: { id: lesson?.id ?? "preview", title: lesson?.title ?? "", type: lesson?.type === "exercise" ? "exercise" : "text", minutes: null }, + module: { id: "", title: "Preview", index: 0 }, + locked: false, + markdown: body, baseFile: resolveCourseUrl ? file ?? null : null, trackMarkdown: {}, tracks: [], + video: null, transcript: null, hints: [], decks: [], code: null, attachments: [], browser: null, localCommand: null, + prev: null, next: null, lessonFiles: {}, + }; + }, [courseId, markdown, lesson?.id, lesson?.title, lesson?.type, lesson?.anchor, file, resolveCourseUrl]); + // With the file known, links resolve course-relative; otherwise relative to the markdown file (the v1 contract). + return
      ; +} diff --git a/packages/feature-studio/src/ui/preview.tsx b/packages/feature-studio/src/ui/preview.tsx index dfc4faf..0799419 100644 --- a/packages/feature-studio/src/ui/preview.tsx +++ b/packages/feature-studio/src/ui/preview.tsx @@ -1,16 +1,21 @@ -import type { ComponentType } from "react"; +import { lazy, Suspense, type ComponentType } from "react"; import { renderMarkdown } from "./markdown"; /** - * Live-preview adapter. v1 renders with a small built-in Markdown renderer; Phase 2 swaps in the learner - * package's `LessonView` via `setLessonPreview` (docs/contract-requests/studio.md). + * Live-preview adapter. The default is the learner's `LessonView` (./learnerPreview, loaded lazily so the + * lesson renderer is fetched only when an editor opens); `setLessonPreview` can replace it, and the small built-in + * Markdown renderer stays as `SimpleMarkdownPreview` and as the loading fallback. */ export interface LessonPreviewProps { courseId: string; markdown: string; lesson?: { id: string; title: string; type: string; anchor?: string } | null; - /** Maps a course-relative path (images, attachments) to a URL that serves the draft file. */ + /** Maps a path relative to the markdown file (images, attachments) to a URL that serves the draft file. */ resolveUrl: (rel: string) => string; + /** Course-relative path of the markdown file. */ + file?: string; + /** Maps a course-relative path to a URL that serves the draft file (preferred over `resolveUrl` when given). */ + resolveCourseUrl?: (courseRel: string) => string; } export type LessonPreviewComponent = ComponentType; @@ -18,11 +23,12 @@ export function SimpleMarkdownPreview({ markdown, resolveUrl }: LessonPreviewPro return
      ; } -let current: LessonPreviewComponent = SimpleMarkdownPreview; +const LearnerPreview = lazy(() => import("./learnerPreview")); +let current: LessonPreviewComponent = LearnerPreview; export function setLessonPreview(c: LessonPreviewComponent): void { current = c; } export function LessonPreview(props: LessonPreviewProps) { const C = current; - return ; + return }>; } diff --git a/packages/feature-studio/src/web.ts b/packages/feature-studio/src/web.ts index d60a5a4..8134982 100644 --- a/packages/feature-studio/src/web.ts +++ b/packages/feature-studio/src/web.ts @@ -4,7 +4,7 @@ import type { WebFeature } from "@stc/contracts"; export const studioWeb: WebFeature = { name: "studio", routes: [ - { path: "/studio", element: () => import("./ui/StudioHome"), nav: { label: "Studio", area: "header" } }, + { path: "/studio", element: () => import("./ui/StudioHome"), nav: { label: "Studio", area: "header", roles: ["owner", "author"] } }, { path: "/studio/:courseId", element: () => import("./ui/CourseStudio") }, ], }; diff --git a/packages/feature-studio/test/app-helpers.ts b/packages/feature-studio/test/app-helpers.ts index e7960af..592bbca 100644 --- a/packages/feature-studio/test/app-helpers.ts +++ b/packages/feature-studio/test/app-helpers.ts @@ -1,6 +1,6 @@ import { resolve } from "node:path"; -import type { AuthUser, ServerContext } from "@stc/contracts"; -import { createApp } from "@stc/core"; +import type { AppConfig, AuthUser, ServerContext } from "@stc/contracts"; +import { createApp, resolveConfig } from "@stc/core"; import { createCourseRepository } from "@stc/course-format"; import { connect, migrate } from "@stc/db"; import { studioFeature } from "../src/server/feature"; @@ -15,7 +15,7 @@ export const users: Record = { }; /** A server context like main.ts builds, with header-selected users (`x-user: ada`). */ -export async function setupApp(extraCourseDirs: string[] = [], dbUrl = "sqlite://:memory:", prepare?: (dataDir: string) => void) { +export async function setupApp(extraCourseDirs: string[] = [], dbUrl = "sqlite://:memory:", prepare?: (dataDir: string) => void, configOverrides: Partial = {}) { const dataDir = tempDir(); prepare?.(dataDir); let dropSchema = async () => {}; @@ -31,7 +31,7 @@ export async function setupApp(extraCourseDirs: string[] = [], dbUrl = "sqlite:/ } const db = await connect(dbUrl); await migrate(db); - const config = { port: 0, databaseUrl: "sqlite://:memory:", dataDir, courseDirs: extraCourseDirs, secret: "t", publicUrl: "http://x" }; + const config = resolveConfig({ port: 0, databaseUrl: "sqlite://:memory:", dataDir, courseDirs: extraCourseDirs, secret: "t", publicUrl: "http://x", ...configOverrides }); const courses = await createCourseRepository(studioCourseDirs(config)); const ctx = { config, db, courses, jobs: null as never, diff --git a/packages/feature-studio/test/preview.test.tsx b/packages/feature-studio/test/preview.test.tsx new file mode 100644 index 0000000..2333f0d --- /dev/null +++ b/packages/feature-studio/test/preview.test.tsx @@ -0,0 +1,28 @@ +import { expect, test } from "bun:test"; +import { renderToString } from "react-dom/server"; +import LearnerLessonPreview from "../src/ui/learnerPreview"; + +const md = "# Intro\n\nHello ![pic](img/a.png)\n\n## Part two\n\nOnly this part.\n\n## Part three\n\nNot this.\n"; +const draft = (rel: string) => `/api/studio/courses/c1/raw/${rel}`; + +test("the studio preview renders through the learner LessonView, with draft URLs", () => { + const html = renderToString( "/wrong"} />); + expect(html).toContain('class="lr lr-lesson"'); + expect(html).toContain("Lesson one"); + expect(html).toContain('src="/api/studio/courses/c1/raw/lessons/img/a.png"'); + expect(html).not.toContain("/api/media/"); +}); + +test("an anchored lesson previews only its section; a missing anchor says so", () => { + const one = renderToString(); + expect(one).toContain("Only this"); + expect(one).not.toContain("Not this"); + const missing = renderToString(); + expect(missing).toContain("was not found"); +}); + +test("raw HTML in a draft is sanitised by the learner pipeline", () => { + const html = renderToString("} lesson={{ id: "l1", title: "T", type: "text" }} file="a.md" resolveCourseUrl={draft} resolveUrl={draft} />); + expect(html).not.toContain("onerror"); + expect(html).not.toContain("