lucasw89/STCourses
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge: Phase 1 feature packages and Phase 2 integration

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Lucas Winther
2026-10-30 15:51:19 +01:00
co-authored by Claude Opus 5.5
parent 3ca05fcfbc db5a604647
commit 58e0b20c95
230 changed files with 19118 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+13
View File
@@ -0,0 +1,13 @@
.git
.worktrees
.superpowers
node_modules
**/node_modules
data
reports
research_notes
docs
**/test
fixtures
*.log
.env
Dockerfile
+52
View File
@@ -0,0 +1,52 @@
# STCourses: the server and the SPA in one image. AGPL-3.0-only.
# Build: docker build -t stc-app .
# Run: see docker-compose.yml and docs/operations.md
FROM oven/bun:1.3
# Tools the importer and the studio call: ffmpeg/ffprobe (frames, durations), git (studio repos, code imports),
# unzip (Packt and code zips), python3 with PyMuPDF (PDF page images and text; STC_PYTHON points at it).
RUN apt-get update \
&& apt-get install -y --no-install-recommends ffmpeg git unzip ca-certificates python3 python3-pymupdf \
&& rm -rf /var/lib/apt/lists/* \
&& python3 -c "import pymupdf"
WORKDIR /app
# Dependencies first (cached unless a manifest or the lockfile changes).
COPY package.json bun.lock ./
COPY apps/server/package.json apps/server/
COPY apps/web/package.json apps/web/
COPY packages/contracts/package.json packages/contracts/
COPY packages/core/package.json packages/core/
COPY packages/course-format/package.json packages/course-format/
COPY packages/db/package.json packages/db/
COPY packages/ui/package.json packages/ui/
COPY packages/feature-learner/package.json packages/feature-learner/
COPY packages/feature-import/package.json packages/feature-import/
COPY packages/feature-studio/package.json packages/feature-studio/
COPY packages/feature-ai/package.json packages/feature-ai/
RUN bun install --production --frozen-lockfile
# Sources (Bun runs TypeScript directly; tsconfig.json carries the @stc/* path mapping).
COPY tsconfig.json ./
COPY apps apps
COPY packages packages
COPY overlays overlays
# /data holds the database (SQLite), imported courses, studio repos and caches. /courses is for course repos you mount.
RUN mkdir -p /data /courses && chown bun:bun /data
VOLUME /data
ENV NODE_ENV=production \
STC_PORT=3000 \
STC_DATA_DIR=/data \
STC_DATABASE_URL=sqlite:///data/stcourses.db \
STC_COURSE_DIRS=/app/overlays:/courses \
STC_IMPORT_ROOTS=/import \
STC_PYTHON=python3
USER bun
EXPOSE 3000
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
CMD bun -e "fetch('http://127.0.0.1:' + (process.env.STC_PORT || 3000) + '/api/health').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))"
CMD ["bun", "apps/server/src/main.ts"]
README.md
+5
View File
@@ -11,4 +11,9 @@ bun run typecheck # tsc --noEmit
bun run dev # start the server with hot reload
```
## Running it
`STC_SECRET=... docker compose up -d` (add `--profile postgres` and `STC_DATABASE_URL` for Postgres). Mounts,
configuration and the measured memory budget are in [docs/operations.md](docs/operations.md).
License: AGPL-3.0-only.
apps/server/src/features.ts
+53
View File
@@ -0,0 +1,53 @@
import type { FeatureFactory, ServerContext } from "@stc/contracts";
import type { PackageMigrations } from "@stc/db";
import { aiFeature, createFsDraftSink, migrationsDirs as aiMigrations, type AiFeatureOptions, type DraftSink } from "@stc/feature-ai";
import { importFeature, migrationsDirs as importMigrations } from "@stc/feature-import";
import { learnerFeature, migrationsDirs as learnerMigrations, userIdentity } from "@stc/feature-learner";
import { createDraftWriter, migrationsDirs as studioMigrations, studioFeature, type StudioFeature } from "@stc/feature-studio";
/** Every package's migration folders, keyed by package name (core runs first, then these in name order). */
export const packageMigrations: PackageMigrations = {
"feature-ai": aiMigrations,
"feature-import": importMigrations,
"feature-learner": learnerMigrations,
"feature-studio": studioMigrations,
};
/**
* Where the AI generator writes drafts: studio-managed courses get a commit on the requesting author's
* `draft/<userId>` branch, authored as that user's name and email; overlay and imported courses (which the studio
* does not manage) fall back to files under `<dataDir>/drafts/<courseId>/<run>/`. The generator page says which.
*/
export function aiDraftSink(ctx: ServerContext, studio: Pick<StudioFeature, "store">): DraftSink {
const writer = createDraftWriter(ctx, { identity: (userId) => userIdentity(ctx.db, userId) });
const files = createFsDraftSink(ctx.config.dataDir);
return {
writeDraftFiles: (courseId, authorId, list, message) =>
(studio.store.isManaged(courseId) ? writer : files).writeDraftFiles(courseId, authorId, list, message),
};
}
export interface FeatureSetOptions {
/** Studio published-worktree polling (default on). */
studioWatch?: boolean;
/** Test seam: replaces AI provider construction (e.g. a canned provider). Production leaves it unset. */
aiProviderFactory?: AiFeatureOptions["providerFactory"];
}
/**
* The product: learner (auth, library, reader), importer, studio and AI, in that order.
* Order matters: the AI factory needs the studio feature (its draft sink writes through the studio store), so the
* studio factory must run first. composeServer runs factories in list order; the AI factory checks this.
*/
export function productFeatures(opts: FeatureSetOptions = {}): FeatureFactory[] {
let studio: StudioFeature | null = null;
return [
learnerFeature,
(ctx) => importFeature(ctx),
(ctx) => (studio = studioFeature(ctx, { watch: opts.studioWatch ?? true })),
(ctx) => {
if (!studio) throw new Error("productFeatures: the studio factory must run before the AI factory");
return aiFeature(ctx, { draftSink: aiDraftSink(ctx, studio), ...(opts.aiProviderFactory ? { providerFactory: opts.aiProviderFactory } : {}) });
},
];
}
apps/server/src/main.ts
+28 -29
View File
@@ -1,44 +1,43 @@
import type { AppConfig, AuthService, Feature, JobHandler, ServerContext } from "@stc/contracts";
import { createApp, createJobQueue, loadConfig } from "@stc/core";
import { createCourseRepository } from "@stc/course-format";
import { connect, migrate } from "@stc/db";
import type { AppConfig, Feature, FeatureFactory } from "@stc/contracts";
import { composeServer, loadConfig, type ComposeOptions } from "@stc/core";
import indexHtml from "../../web/index.html";
import { packageMigrations, productFeatures } from "./features";
// Temporary: the learner package replaces this with real sessions.
const anonymousAuth: AuthService = { userFromRequest: async () => null };
export interface StartOptions extends ComposeOptions {}
export async function startServer(
config: AppConfig,
features: Feature[],
_webFeatures?: unknown,
): Promise<{ server: Bun.Server<undefined>; stop(): Promise<void> }> {
const db = await connect(config.databaseUrl);
await migrate(db);
const courses = await createCourseRepository(config.courseDirs);
const handlers: Record<string, JobHandler> = {};
for (const f of features) Object.assign(handlers, f.jobs);
const jobs = createJobQueue(db, handlers);
const ctx: ServerContext = { config, db, courses, jobs, auth: anonymousAuth };
const app = createApp(ctx, features);
jobs.start();
const server = Bun.serve({
port: config.port,
// Bun routes win over fetch; "/api/*" falls through to the composed app (404 for unknown API paths).
routes: { ...app.routes, "/api/*": (req) => app.fetch(req), "/*": indexHtml },
fetch: (req) => app.fetch(req),
});
/**
* Composes the features (core `composeServer`) and serves them with the SPA. Bun routes win over `fetch`;
* "/api/*" falls through to the composed app, so unknown API paths are 404 rather than the SPA.
*/
export async function startServer(config: AppConfig, features: (Feature | FeatureFactory)[], opts: StartOptions = {}) {
const composed = await composeServer(config, features, opts);
const { app } = composed;
let server: Bun.Server<undefined>;
try {
server = Bun.serve({
port: config.port,
routes: { ...app.routes, "/api/*": (req) => app.fetch(req), "/*": indexHtml },
fetch: (req) => app.fetch(req),
// Production (NODE_ENV=production, as in the Docker image) bundles the SPA once, minified, without HMR.
development: process.env.NODE_ENV !== "production",
});
} catch (e) {
await composed.stop();
throw e;
}
return {
server,
ctx: composed.ctx,
features: composed.features,
async stop() {
await server.stop(true);
await jobs.stop();
await db.close();
await composed.stop();
},
};
}
if (import.meta.main) {
const config = loadConfig();
const { server } = await startServer(config, []);
const { server } = await startServer(config, productFeatures(), { migrations: packageMigrations });
console.log(`STCourses listening on ${server.url}`);
}
apps/server/test/e2e.test.ts
+218
View File
@@ -0,0 +1,218 @@
// End-to-end (plan Task I3): the real server with every feature, wired exactly as in production
// (`productFeatures()` order and `packageMigrations`), against the fixtures plus a restricted copy, on SQLite and
// Postgres. It is also the visibility and cross-site regression suite for the composed server.
import { describe, expect, test } from "bun:test";
import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import type { AiProvider } from "@stc/contracts";
import { loadConfig } from "@stc/core";
import { createBrowserRunner } from "../../../packages/feature-learner/src/web/runner/runner";
import { packageMigrations, productFeatures } from "../src/features";
import { startServer } from "../src/main";
import { PG, pgSchema } from "./pg";
const FIXTURES = resolve(import.meta.dir, "../../../fixtures/courses");
const SAMPLE = join(FIXTURES, "sample");
const ORIGIN = "http://localhost:3000";
const JPEG = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0, 0x10, 0x4a, 0x46, 0x49, 0x46, 0, 1, 0xff, 0xd9]);
const cookieFrom = (r: Response) => r.headers.getSetCookie().map((c) => c.split(";")[0]).filter((c) => !c.endsWith("=")).join("; ");
/** A canned AI provider: chat answers with text; generation answers with one file block. */
const fakeAi: AiProvider = {
id: "fake",
async *chat(messages) {
const sys = messages[0]?.content ?? "";
if (sys.includes("=== FILE:")) yield { text: "=== FILE: lessons/generated.md ===\n# Generated\n\nA draft.\n=== END FILE ===\n" };
else yield { text: "An answer." };
},
};
/** A restricted copy of the sample course ("secret", titled "Hidden Vault"). */
function restrictedCourseDir(): string {
const dir = mkdtempSync(join(tmpdir(), "stc-e2e-courses-"));
cpSync(SAMPLE, join(dir, "secret"), { recursive: true });
const m = readFileSync(join(SAMPLE, "stcourse.yaml"), "utf8");
writeFileSync(join(dir, "secret/stcourse.yaml"), m.replace("id: sample", "id: secret").replace("title: Sample course", "title: Hidden Vault").replace("visibility: public", "visibility: restricted"));
return dir;
}
async function run(databaseUrl: string) {
const dataDir = mkdtempSync(join(tmpdir(), "stc-e2e-"));
const config = loadConfig({
STC_PORT: "0", STC_DATABASE_URL: databaseUrl, STC_SECRET: "e2e-secret-0123456789abcdef0123456789",
STC_DATA_DIR: dataDir, STC_COURSE_DIRS: `${FIXTURES}:${restrictedCourseDir()}`, STC_PUBLIC_URL: ORIGIN,
});
// The importer's cover candidate for the restricted course.
mkdirSync(join(config.cacheDir, "covers", "secret"), { recursive: true });
writeFileSync(join(config.cacheDir, "covers", "secret", "frame-1.jpg"), JPEG);
const s = await startServer(config, productFeatures({ studioWatch: false, aiProviderFactory: () => fakeAi }), { migrations: packageMigrations });
const base = s.server.url.toString().replace(/\/$/, "");
type Opts = { cookie?: string; body?: unknown; headers?: Record<string, string>; origin?: string | null };
const call = async (method: string, path: string, opts: Opts = {}) => {
const headers: Record<string, string> = { ...opts.headers };
if (opts.origin !== null) headers.origin = opts.origin ?? ORIGIN;
if (opts.cookie) headers.cookie = opts.cookie;
if (opts.body !== undefined) headers["content-type"] ??= "application/json";
const body = opts.body === undefined ? undefined : typeof opts.body === "string" ? opts.body : JSON.stringify(opts.body);
return fetch(base + path, { method, headers, body });
};
const json = async (method: string, path: string, opts: Opts = {}) => (await call(method, path, opts)).json() as Promise<any>;
const signUp = async (email: string, name: string) => {
const r = await call("POST", "/api/auth/sign-up/email", { body: { email, name, password: "correct horse battery" } });
expect(r.status).toBe(200);
return cookieFrom(r);
};
const until = async (f: () => Promise<boolean>) => { for (let i = 0; i < 400; i++) { if (await f()) return; await Bun.sleep(25); } throw new Error("timeout"); };
try {
// 1. the server is up with every feature, the fixture course and the restricted copy
expect(await json("GET", "/api/health")).toEqual({ ok: true, courses: 2, db: databaseUrl.startsWith("postgres") ? "postgres" : "sqlite" });
expect((await call("GET", "/c/sample")).headers.get("content-type")).toContain("text/html");
expect(s.features.map((f) => f.name)).toEqual(["learner", "import", "studio", "ai"]);
// 2. sign up: the first account is the owner
const owner = await signUp("ada@example.org", "Ada Lovelace");
const me = (await json("GET", "/api/me", { cookie: owner })) as { user: { id: string; roles: string[] } };
expect(me.user.roles).toEqual(["owner"]);
// 3. mark a lesson done
expect((await call("PUT", "/api/courses/sample/progress/m1-intro", { cookie: owner, body: { state: "done" } })).status).toBe(200);
// 4. a slide note on the video lesson's deck, and a bodyless DELETE over the wire
const note = await call("POST", "/api/courses/sample/notes", { cookie: owner, body: { lessonId: "m1-video", anchorKind: "slide", anchorValue: 2, deckId: "d1", body: "Slide two matters" } });
expect(note.status).toBe(201);
const scratch = (await json("POST", "/api/courses/sample/notes", { cookie: owner, body: { lessonId: "m1-intro", body: "scratch" } })) as { id: string };
expect((await call("DELETE", `/api/notes/${scratch.id}`, { cookie: owner })).status).toBe(200);
// 5. run the sample exercise with the learner's runner, then save the passing attempt
const lesson = (await json("GET", "/api/courses/sample/lessons/m2-ex-add", { cookie: owner })) as { browser: { starter: string; tests: string } };
const runner = createBrowserRunner("js");
expect((await runner.run(lesson.browser.starter, lesson.browser.tests, 5000)).some((t) => !t.passed)).toBe(true);
const code = lesson.browser.starter.replace("a - b", "a + b");
const results = await runner.run(code, lesson.browser.tests, 5000);
expect(results.length).toBeGreaterThan(0);
expect(results.every((t) => t.passed)).toBe(true);
expect((await call("PUT", "/api/courses/sample/lessons/m2-ex-add/attempt", { cookie: owner, body: { code, passed: true, result: results, track: "js" } })).status).toBe(200);
// 6. read progress and notes back
const progress = (await json("GET", "/api/courses/sample/progress", { cookie: owner })) as Record<string, { state: string }>;
expect(progress["m1-intro"]?.state).toBe("done");
expect(progress["m2-ex-add"]?.state).toBe("done");
const notes = (await json("GET", "/api/courses/sample/notes", { cookie: owner })) as { body: string; anchorKind: string; deckId: string }[];
expect(notes.map((n) => [n.body, n.anchorKind, n.deckId])).toEqual([["Slide two matters", "slide", "d1"]]);
// 7. a second account is a plain learner: own data only, and the owner/author features refuse it
const lea = await signUp("lea@example.org", "Lea");
expect(await json("GET", "/api/courses/sample/progress", { cookie: lea })).toEqual({});
expect((await call("GET", "/api/import/info", { cookie: lea })).status).toBe(403);
expect((await call("GET", "/api/studio/courses", { cookie: lea })).status).toBe(403);
expect((await call("GET", "/api/ai/admin/config", { cookie: lea })).status).toBe(403);
expect((await call("GET", "/api/import/info", { cookie: owner })).status).toBe(200);
expect((await call("GET", "/api/studio/courses", { cookie: owner })).status).toBe(200);
// 8. media: a real Range request gives 206 with the right bytes; nosniff everywhere
const whole = new Uint8Array(readFileSync(join(SAMPLE, "media/clip.mp4")));
const part = await call("GET", "/api/media/sample/media/clip.mp4", { cookie: lea, headers: { Range: "bytes=0-9" } });
expect(part.status).toBe(206);
expect(part.headers.get("content-range")).toBe(`bytes 0-9/${whole.length}`);
expect(new Uint8Array(await part.arrayBuffer())).toEqual(whole.slice(0, 10));
expect(part.headers.get("x-content-type-options")).toBe("nosniff");
expect((await call("GET", "/api/media/sample/media/clip.mp4", { headers: { Range: `bytes=${whole.length}-` } })).status).toBe(416);
// 9. AI on (canned provider), so chat and retrieval can be checked
expect((await call("PUT", "/api/ai/admin/config", { cookie: owner, body: { provider: { kind: "anthropic", apiKey: "k" }, mcpServers: [] } })).status).toBe(200);
// 10. the restricted course: the owner gets everything; a learner and anonymous users learn nothing
const restricted: [string, string, unknown?][] = [
["GET", "/api/courses/secret"],
["GET", "/api/courses/secret/lessons/m1-intro"],
["GET", "/api/media/secret/lessons/01-intro.md"],
["GET", "/api/courses/secret/cover"],
["GET", "/api/courses/secret/code/m1"],
["GET", "/api/courses/secret/code/m1/zip"],
["GET", "/api/courses/secret/notes"],
["GET", "/api/courses/secret/progress"],
["PUT", "/api/courses/secret/progress/m1-intro", { state: "done" }],
["POST", "/api/courses/secret/notes", { lessonId: "m1-intro", body: "x" }],
["PUT", "/api/courses/secret/lessons/m2-ex-add/attempt", { code: "x", passed: false }],
["POST", "/api/ai/courses/secret/chat", { message: "What is in this course?" }],
];
for (const [method, path, body] of restricted) {
const asOwner = await call(method, path, { cookie: owner, body });
expect([method, path, asOwner.status >= 200 && asOwner.status < 300]).toEqual([method, path, true]);
await asOwner.arrayBuffer();
const asLearner = await call(method, path, { cookie: lea, body });
expect([method, path, asLearner.status]).toEqual([method, path, 404]);
// Anonymous: 404 for content; 401 ("sign in first") on per-user routes, which answer the same for every course id.
const anon = await call(method, path, { body });
expect([method, path, [401, 404].includes(anon.status)]).toEqual([method, path, true]);
const text = await anon.text();
expect(text).not.toContain("Hidden Vault");
}
const ids = async (cookie?: string) => ((await json("GET", "/api/library", { cookie })) as { courses: { id: string }[] }).courses.map((c) => c.id).sort();
expect(await ids(owner)).toEqual(["sample", "secret"]);
expect(await ids(lea)).toEqual(["sample"]);
expect(await ids()).toEqual(["sample"]);
const search = async (cookie?: string) => JSON.stringify(await json("GET", "/api/library/search?q=vault", { cookie }));
expect(await search(owner)).toContain("secret");
expect(await search(lea)).toBe("[]");
expect(await search()).toBe("[]");
// AI retrieval: the course is unknown to the learner's assistant status, known to the owner's
expect((await json("GET", "/api/ai/status?courseId=secret", { cookie: owner })).course?.id).toBe("secret");
expect((await json("GET", "/api/ai/status?courseId=secret", { cookie: lea })).course).toBeUndefined();
expect((await json("GET", "/api/ai/status?courseId=secret")).course).toBeUndefined();
// 11. cross-site requests (CSRF): the owner's cookie is not enough
const pwned = join(dataDir, "PWNED");
const mcp = { transport: "stdio", command: "sh", args: ["-c", `touch ${pwned}`] };
const forged = await call("POST", "/api/ai/admin/mcp/test", { cookie: owner, origin: "http://evil.example", headers: { "content-type": "text/plain" }, body: JSON.stringify(mcp) });
expect(forged.status).toBe(403);
const forgedJson = await call("POST", "/api/ai/admin/mcp/test", { cookie: owner, origin: "http://evil.example", body: mcp });
expect(forgedJson.status).toBe(403);
const noOrigin = await call("POST", "/api/ai/admin/mcp/test", { cookie: owner, origin: null, body: mcp });
expect(noOrigin.status).toBe(403);
await Bun.sleep(200);
expect(existsSync(pwned)).toBe(false);
expect((await call("POST", "/api/courses/sample/notes", { cookie: owner, origin: "http://evil.example", headers: { "content-type": "text/plain" }, body: JSON.stringify({ lessonId: "m1-intro", body: "forged" }) })).status).toBe(403);
expect((await call("POST", "/api/courses/sample/notes", { cookie: owner, origin: null, headers: { referer: `${ORIGIN}/c/sample/m1-intro` }, body: { lessonId: "m1-intro", body: "via referer" } })).status).toBe(201);
expect((await call("POST", "/api/auth/sign-in/email", { origin: "http://evil.example", body: { email: "ada@example.org", password: "correct horse battery" } })).status).toBe(403);
expect((await json("GET", "/api/courses/sample/notes", { cookie: owner })).map((n: { body: string }) => n.body)).not.toContain("forged");
// 12. AI drafts through the real wiring: a studio course gets a commit on the author's draft branch, as the
// author; a course the studio does not manage gets files under <data>/drafts
expect((await call("POST", "/api/studio/courses", { cookie: owner, body: { id: "gen-target", title: "Generated" } })).status).toBe(201);
const generate = async (courseId: string, lessonId: string) => {
const r = await call("POST", "/api/ai/studio/generate", { cookie: owner, body: { courseId, kind: "document", style: "pure-snake", topic: "Adding", sources: [{ kind: "lesson", lessonId }] } });
expect(r.status).toBe(202);
const { jobId } = (await r.json()) as { jobId: string };
let job: any = null;
await until(async () => { job = await json("GET", `/api/ai/studio/jobs/${jobId}`, { cookie: owner }); return job.status === "done" || job.status === "failed"; });
expect([job.status, job.error]).toEqual(["done", null]);
return job.result as { branch?: string; location: string; paths: string[] };
};
const onStudio = await generate("gen-target", "welcome");
expect(onStudio.branch).toMatch(/^draft\//);
const repo = join(config.studioReposDir, "gen-target.git");
expect((await Bun.$`git --git-dir=${repo} log -1 ${"--format=%an <%ae>"} ${onStudio.branch!}`.text()).trim()).toBe("Ada Lovelace <ada@example.org>");
expect(await Bun.$`git --git-dir=${repo} show ${`${onStudio.branch}:lessons/generated.md`}`.text()).toContain("# Generated");
const onFiles = await generate("sample", "m1-intro");
expect(onFiles.branch).toBeUndefined();
expect(onFiles.location.startsWith(resolve(dataDir, "drafts", "sample"))).toBe(true);
expect(readFileSync(join(onFiles.location, "lessons/generated.md"), "utf8")).toContain("# Generated");
} finally {
await s.stop();
}
}
describe("end to end", () => {
test("sign up, mark done, slide note, exercise, progress, visibility, CSRF, AI drafts [sqlite]", () => run("sqlite://:memory:"), 90_000);
const pg = PG;
if (pg) {
test("sign up, mark done, slide note, exercise, progress, visibility, CSRF, AI drafts [postgres]", async () => {
const { url, drop } = await pgSchema(pg);
try { await run(url); } finally { await drop(); }
}, 90_000);
}
});
apps/server/test/pg.ts
+13
View File
@@ -0,0 +1,13 @@
import { connect } from "@stc/db";
export const PG = process.env.STC_TEST_PG_URL;
/** A fresh Postgres schema (search_path in the URL), dropped by `drop()`. Same isolation as packages/db's forEachDb. */
export async function pgSchema(base: string): Promise<{ url: string; drop(): Promise<void> }> {
const schema = `t_${Bun.randomUUIDv7().replaceAll("-", "")}`;
const admin = await connect(base);
await admin.sql.unsafe(`CREATE SCHEMA ${schema}`);
const u = new URL(base);
u.searchParams.set("options", `-c search_path=${schema}`);
return { url: u.toString(), drop: async () => { await admin.sql.unsafe(`DROP SCHEMA ${schema} CASCADE`); await admin.close(); } };
}
apps/server/test/server.test.ts
+4 -2
View File
@@ -1,5 +1,7 @@
import { afterAll, beforeAll, expect, test } from "bun:test";
import { resolve } from "node:path";
import { mkdtempSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { loadConfig } from "@stc/core";
import { startServer } from "../src/main";
@@ -7,7 +9,7 @@ let s: Awaited<ReturnType<typeof startServer>>;
let base: string;
beforeAll(async () => {
const config = loadConfig({
STC_PORT: "0", STC_DATABASE_URL: "sqlite://:memory:", STC_SECRET: "t",
STC_PORT: "0", STC_DATABASE_URL: "sqlite://:memory:", STC_SECRET: "t", STC_DATA_DIR: mkdtempSync(join(tmpdir(), "stc-server-")),
STC_COURSE_DIRS: resolve(import.meta.dir, "../../../fixtures/courses"),
});
s = await startServer(config, []);
apps/web/index.html
+4
View File
@@ -3,6 +3,10 @@
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- CSP (learner review C1, ruling R16). No inline script: the one hash is Bun's dev-server helper
(/_bun/unref beacon), which production builds do not inject. frame-ancestors cannot be set in a meta tag;
see docs/contract-requests/learner.md section 5. -->
<meta http-equiv="Content-Security-Policy" content="default-src 'self'; script-src 'self' 'sha256-ES8VF47udzjVFFPh4RhgcgjUrADRd1Ef0GD9UagS2mY='; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob: https:; media-src 'self' blob:; font-src 'self' data:; connect-src 'self'; worker-src 'self' blob:; frame-src 'self'; object-src 'none'; base-uri 'self'; form-action 'self'">
<title>STCourses</title>
<link rel="stylesheet" href="../../packages/ui/src/fonts.css">
<link rel="stylesheet" href="../../packages/ui/src/tokens.css">
apps/web/src/bundle.test.ts
+25
View File
@@ -0,0 +1,25 @@
import { expect, test } from "bun:test";
import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
// A subprocess, because HTML bundling inside a full `bun test` run is unreliable (see the learner's csp.test).
test("the SPA bundles for the browser with no server code in it", async () => {
const out = mkdtempSync(join(tmpdir(), "stc-spa-"));
try {
const root = resolve(import.meta.dir, "../../..");
const p = Bun.spawn(["bun", "build", "apps/web/index.html", "--outdir", out, "--target", "browser"], { cwd: root, stdout: "pipe", stderr: "pipe" });
const code = await p.exited;
expect(code === 0 ? "" : await new Response(p.stderr).text()).toBe("");
const js = readdirSync(out).filter((f) => f.endsWith(".js")).map((f) => readFileSync(join(out, f), "utf8")).join("\n");
expect(js.length).toBeGreaterThan(1000);
// Server-only modules of the feature packages and their dependencies.
for (const marker of ["better-auth", "X509Certificate", "createStudioStore", "importFeature(", "aiFeature(", "createJobQueue", "bun:sqlite"]) {
expect([marker, js.includes(marker)]).toEqual([marker, false]);
}
// Every feature's browser half is present.
for (const page of ["/admin/import", "/studio/:courseId", "/admin/ai", "/c/:courseId/:lessonId"]) expect(js).toContain(page);
} finally {
rmSync(out, { recursive: true, force: true });
}
}, 120_000);
apps/web/src/features.ts
+7 -2
View File
@@ -1,4 +1,9 @@
import type { WebFeature } from "@stc/contracts";
// Browser-only entries ("@stc/<package>/web", mapped in tsconfig paths and package.json exports). Never import a
// package's server index here: it pulls Better Auth, git and node:fs into the bundle.
import { learnerWeb } from "@stc/feature-learner/web";
import { importWeb } from "@stc/feature-import/web";
import { studioWeb } from "@stc/feature-studio/web";
import { aiWeb } from "@stc/feature-ai/web";
// Phase 1 packages append one import line each and push into this list.
export const webFeatures: WebFeature[] = [];
export const webFeatures: WebFeature[] = [learnerWeb, importWeb, studioWeb, aiWeb];
apps/web/src/main.tsx
+26 -6
View File
@@ -1,21 +1,41 @@
import { createRoot } from "react-dom/client";
import { lazy, Suspense } from "react";
import { createBrowserRouter, Link, Outlet, RouterProvider, type RouteObject } from "react-router-dom";
import { lazy, Suspense, useEffect, useState } from "react";
import { createBrowserRouter, Link, NavLink, Outlet, RouterProvider, useLocation, type RouteObject } from "react-router-dom";
import type { AuthUser } from "@stc/contracts";
import { AppShell } from "@stc/ui";
import { webFeatures } from "./features";
import { visibleNav } from "./nav";
function Home() {
return <p className="work">No features installed yet</p>;
}
/** The signed-in user (GET /api/me), refreshed on navigation so sign-in and sign-out update the nav. */
function useSessionUser(): AuthUser | null {
const { pathname } = useLocation();
const [user, setUser] = useState<AuthUser | null>(null);
useEffect(() => {
let live = true;
fetch("/api/me", { credentials: "same-origin" })
.then((r) => (r.ok ? r.json() : { user: null }))
.then((b: { user: AuthUser | null }) => { if (live) setUser(b.user ?? null); })
.catch(() => { if (live) setUser(null); });
return () => { live = false; };
}, [pathname]);
return user;
}
function Layout() {
const nav = webFeatures.flatMap((f) => f.routes.filter((r) => r.nav?.area === "header"));
// Links the user cannot use are hidden; the pages and APIs still enforce access themselves.
const nav = visibleNav(webFeatures, useSessionUser());
const header = (
<>
<Link to="/" style={{ fontWeight: 600, color: "var(--head)", textDecoration: "none" }}>STCourses</Link>
<nav style={{ display: "flex", gap: 14 }}>
{nav.map((r) => <Link key={r.path} to={r.path}>{r.nav!.label}</Link>)}
</nav>
{nav.length ? (
<nav className="stc-nav" aria-label="Areas">
{nav.map((r) => <NavLink key={r.path} to={r.path}>{r.nav!.label}</NavLink>)}
</nav>
) : null}
</>
);
return (
apps/web/src/nav.test.ts
+21
View File
@@ -0,0 +1,21 @@
import { expect, test } from "bun:test";
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { webFeatures } from "./features";
import { visibleNav } from "./nav";
const labels = (roles: ("owner" | "author" | "learner")[] | null) => visibleNav(webFeatures, roles ? { roles } : null).map((r) => r.nav!.label);
test("nav items are limited by role", () => {
expect(labels(null)).toEqual([]);
expect(labels(["learner"])).toEqual([]);
expect(labels(["author"])).toEqual(["Studio"]);
expect(labels(["owner"])).toEqual(["Studio", "Import", "AI"]);
});
test("features.ts imports only browser entries, by package name", () => {
const src = readFileSync(join(import.meta.dir, "features.ts"), "utf8");
const specs = [...src.matchAll(/from\s+"([^"]+)"/g)].map((m) => m[1]!);
for (const s of specs) expect(s === "@stc/contracts" || /^@stc\/feature-[a-z]+\/web$/.test(s)).toBe(true);
expect(specs.filter((s) => s.endsWith("/web")).length).toBe(4);
});
apps/web/src/nav.ts
+15
View File
@@ -0,0 +1,15 @@
import type { AuthUser, Role, WebFeature, WebRoute } from "@stc/contracts";
/** Admin-area links default to the owner; header links default to everyone. `nav.roles` overrides both. */
const DEFAULT_ROLES: Record<"header" | "admin", Role[] | undefined> = { header: undefined, admin: ["owner"] };
/** The nav links this user may use, header links first, then admin links. */
export function visibleNav(features: WebFeature[], user: Pick<AuthUser, "roles"> | null): WebRoute[] {
const all = features.flatMap((f) => f.routes.filter((r) => r.nav));
const allowed = (r: WebRoute) => {
const roles = r.nav!.roles ?? DEFAULT_ROLES[r.nav!.area];
if (!roles) return true;
return !!user && roles.some((x) => user.roles.includes(x));
};
return [...all.filter((r) => r.nav!.area === "header" && allowed(r)), ...all.filter((r) => r.nav!.area === "admin" && allowed(r))];
}
bun.lock
+158
View File
@@ -57,6 +57,16 @@
"packages/feature-learner": {
"name": "@stc/feature-learner",
"version": "0.0.0",
"dependencies": {
"@better-auth/sso": "^1.7.14",
"better-auth": "^1.7.14",
"marked": "^18.1.2",
"pdfjs-dist": "^6.4.299",
"sanitize-html": "^2",
},
"devDependencies": {
"@types/sanitize-html": "^2.16.2",
},
},
"packages/feature-studio": {
"name": "@stc/feature-studio",
@@ -68,14 +78,68 @@
},
},
"packages": {
"@authenio/xml-encryption": ["@authenio/xml-encryption@6.0.2", "", { "dependencies": { "@xmldom/xmldom": "^0.8.13", "escape-html": "^1.0.3" } }, "sha512-/jv5qO935fH4Ma+fxh5rJLRYZvUM57fd/I74IiSoR8cFm+sgjszXFEEn1PkqZhhBD4MVkH2YT+/UDY+pnW0S0Q=="],
"@better-auth/core": ["@better-auth/core@1.7.14", "", { "dependencies": { "@standard-schema/spec": "^1.1.0", "zod": "^4.5.4" }, "peerDependencies": { "@better-auth/utils": "^0.4.2 || ^0.5.0", "@better-fetch/fetch": "1.3.3", "@opentelemetry/api": "^1.9.0", "better-call": "1.4.4", "jose": "^6.1.0", "kysely": "^0.28.5 || ^0.29.0", "nanostores": "^1.0.1" }, "optionalPeers": ["@opentelemetry/api"] }, "sha512-ByhdrGVwp/el+hoTYKSVOHIH23/lQj9TQ1txdGcWOg22tnfr3hJ6ukzySooeYj/R36XsQgBhkJqIp8mKsNeErA=="],
"@better-auth/drizzle-adapter": ["@better-auth/drizzle-adapter@1.7.14", "", { "peerDependencies": { "@better-auth/core": "^1.7.14", "@better-auth/utils": "^0.4.2 || ^0.5.0", "drizzle-orm": "^0.45.2 || >=1.0.0-rc.1 <2.0.0" }, "optionalPeers": ["drizzle-orm"] }, "sha512-5RC8Ec5Ur15FWsS4As9/VrlRbB3+9EfShxIpA5A1FcD6IgPxFvzQNkAm2+FYx9LUSqx47NPf9hBTG2GUCMP9Ig=="],
"@better-auth/kysely-adapter": ["@better-auth/kysely-adapter@1.7.14", "", { "peerDependencies": { "@better-auth/core": "^1.7.14", "@better-auth/utils": "^0.4.2 || ^0.5.0", "kysely": "^0.28.17 || ^0.29.0" }, "optionalPeers": ["kysely"] }, "sha512-4wkqj8a8JVpWDYLlgzfbk7eOaWCFkL6HHQd6CLamFKaC8vNsXew1x4v5dE5O6dGr8j4g5qMISS+BAvOuIvBevg=="],
"@better-auth/memory-adapter": ["@better-auth/memory-adapter@1.7.14", "", { "peerDependencies": { "@better-auth/core": "^1.7.14", "@better-auth/utils": "^0.4.2 || ^0.5.0" } }, "sha512-Ojv9BcmFyHuTHha9ZM/NUW1ut9J84wLngta+07Es0Ejn8kekqP5aHBpZd5Z8AvADs5iJUOWvQGFmGlPWAKJgBg=="],
"@better-auth/mongo-adapter": ["@better-auth/mongo-adapter@1.7.14", "", { "peerDependencies": { "@better-auth/core": "^1.7.14", "@better-auth/utils": "^0.4.2 || ^0.5.0", "mongodb": "^6.0.0 || ^7.0.0" }, "optionalPeers": ["mongodb"] }, "sha512-6IA0RSPZFMdRwwnLZ9oWnDIiD0y0o0azTHUfa01BNU2AFJgMDkeiASUV3bKcJBskzQoyQrff74eFYTsTSeeVUQ=="],
"@better-auth/prisma-adapter": ["@better-auth/prisma-adapter@1.7.14", "", { "peerDependencies": { "@better-auth/core": "^1.7.14", "@better-auth/utils": "^0.4.2 || ^0.5.0", "@prisma/client": "^5.0.0 || ^6.0.0 || ^7.0.0", "prisma": "^5.0.0 || ^6.0.0 || ^7.0.0" }, "optionalPeers": ["@prisma/client", "prisma"] }, "sha512-mF6t5xqeZNWzUe0ErmrSdzQmtslJscmu9X8Mp5QmROlpnnAzvSc70RILu8lHms8CePt7N34+KRH1QvYjGFrVGQ=="],
"@better-auth/sso": ["@better-auth/sso@1.7.14", "", { "dependencies": { "@xmldom/xmldom": "^0.9.12", "fast-xml-parser": "^5.8.0", "jose": "^6.2.3", "samlify": "^2.13.1", "tldts": "^7.4.3", "zod": "^4.5.4" }, "peerDependencies": { "@better-auth/core": "^1.7.14", "@better-auth/utils": "^0.4.2 || ^0.5.0", "@better-fetch/fetch": "1.3.3", "better-auth": "^1.7.14", "better-call": "1.4.4" } }, "sha512-sfUzpm6GO5l5MSUA17qRO/E4INthqF+KRmymtj4Hhm+Xsx6CoRgr0BVIDfAic3tSqVhObcVYiiiZ38dXXhkCCw=="],
"@better-auth/telemetry": ["@better-auth/telemetry@1.7.14", "", { "peerDependencies": { "@better-auth/core": "^1.7.14", "@better-auth/utils": "^0.4.2 || ^0.5.0", "@better-fetch/fetch": "1.3.3" } }, "sha512-PtKFzsH7gje/WJuUPDjckOoNFazsq8NMt3AdiXIPzHl7tZULcNGTKAX8K7+yn0rG6cO/4mtMgpOjhtilElOnmg=="],
"@better-auth/utils": ["@better-auth/utils@0.5.1", "", { "dependencies": { "@noble/hashes": "^2.0.1" } }, "sha512-WETqw5vQ52uQ/xwWR4E8rTgT6fDEr/wnS+otDNkBCa/69281FCF32OKDdQUcahMCnjsvqXBNiEvAQHFB0EVRRg=="],
"@better-fetch/fetch": ["@better-fetch/fetch@1.3.3", "", {}, "sha512-U6SlDt/GUWtK2JAxS/NhdoYTEim+EL1G5byz2TUj83KjLZBUpYzb5SmX0aXaCnbGkAjFKnV/o/0+cyXMbEcqqg=="],
"@fontsource/ibm-plex-mono": ["@fontsource/ibm-plex-mono@5.3.0", "", {}, "sha512-eTgnZjZEGk1QtD3ZstF+Vclo2HLAni8YMy34/DxllwZvyz1lR/1RF/xTiAquOBO7MvqBx8D2Ig2WCPMVfdZu7Q=="],
"@fontsource/ibm-plex-sans": ["@fontsource/ibm-plex-sans@5.3.0", "", {}, "sha512-CbE4CbbEEZJX860XyUiRpsksXIQR8Rp2XDva2VO53NJox9tVNtusrysd2x5YkUEY3ErQ66W1IiiQL8/wihhw5w=="],
"@fontsource/literata": ["@fontsource/literata@5.3.0", "", {}, "sha512-9JGMT5+YfBK49thoMq9ASaMRJErHT4V4QuMhAEwKHRLreOsIMP2xP6ZRLhPJ0e6YlET+kjWTJuN7LAxt9/gWGQ=="],
"@napi-rs/canvas": ["@napi-rs/canvas@1.0.13", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "1.0.13", "@napi-rs/canvas-darwin-arm64": "1.0.13", "@napi-rs/canvas-darwin-x64": "1.0.13", "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.13", "@napi-rs/canvas-linux-arm64-gnu": "1.0.13", "@napi-rs/canvas-linux-arm64-musl": "1.0.13", "@napi-rs/canvas-linux-riscv64-gnu": "1.0.13", "@napi-rs/canvas-linux-x64-gnu": "1.0.13", "@napi-rs/canvas-linux-x64-musl": "1.0.13", "@napi-rs/canvas-win32-arm64-msvc": "1.0.13", "@napi-rs/canvas-win32-x64-msvc": "1.0.13" } }, "sha512-b87ugZpQ5ViwqVBkjervxpwa0GmJPsKHLBNoZN6GpjWeHzTbDUGryriq+2DYW2XbB+xCqSIINQysC0/I87nPbw=="],
"@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@1.0.13", "", { "os": "android", "cpu": "arm64" }, "sha512-trn7AaDzQ+yeeA0ezhkC7cB1ZDPtp4UYk0EvB5F8qtYL9jEVWtVrU30WOpI52DszoTxaTsCgf5PP4ZVaOlOIHQ=="],
"@napi-rs/canvas-darwin-arm64": ["@napi-rs/canvas-darwin-arm64@1.0.13", "", { "os": "darwin", "cpu": "arm64" }, "sha512-6Y2djEetL8copxfNK6hg7oig/xqTwOchv19hBOm+zBNdKc/02Z3kjZN5wrLzv/l5O04/bAQS6I/AlQpwBlXgPg=="],
"@napi-rs/canvas-darwin-x64": ["@napi-rs/canvas-darwin-x64@1.0.13", "", { "os": "darwin", "cpu": "x64" }, "sha512-OZoPwwu4TMMg8GTCFBrg8UFExwQTU/55i3uTC/izDat6PHE+ro9dimyQoOboFf91+NY7Pnn70ScR8f8I8+eZXw=="],
"@napi-rs/canvas-linux-arm-gnueabihf": ["@napi-rs/canvas-linux-arm-gnueabihf@1.0.13", "", { "os": "linux", "cpu": "arm" }, "sha512-dZSYaKROuswU2eUWG5aBKndNVe68elo1YX93xtRax7q658tj8rYE+HwduovWJgs6qwv1r1/CLR60nDeBbyjvyg=="],
"@napi-rs/canvas-linux-arm64-gnu": ["@napi-rs/canvas-linux-arm64-gnu@1.0.13", "", { "os": "linux", "cpu": "arm64" }, "sha512-W6FzYE8ADHI93In1vi80KBL6btDtAMuj50Ch5ptkWS8xuntyzbR4h5JDUmXoakxZn2yMInksudTEqVSWoFqFGg=="],
"@napi-rs/canvas-linux-arm64-musl": ["@napi-rs/canvas-linux-arm64-musl@1.0.13", "", { "os": "linux", "cpu": "arm64" }, "sha512-+6fQ+rR49hkAa4voh2stLIwQ+4t+gmoseM8Bc53/DlMXb73IKymDPWK9A/NtWeqTW4ijWppYuyuxS2hPI8tzNA=="],
"@napi-rs/canvas-linux-riscv64-gnu": ["@napi-rs/canvas-linux-riscv64-gnu@1.0.13", "", { "os": "linux", "cpu": "none" }, "sha512-17XEMO3t9RW/TtHtSFf2AkaSWuCWdzbPeidz7zrOFawHxoLSJCnpTI1hN5aUvrZNnkNNmt/vZVJCh0ryUaik3A=="],
"@napi-rs/canvas-linux-x64-gnu": ["@napi-rs/canvas-linux-x64-gnu@1.0.13", "", { "os": "linux", "cpu": "x64" }, "sha512-lc48IwknVHBsNzXn9Xs5OkUsRX38Ml3Z1aOybQ51mMEzZckm8+4A0qROlyYHEoBPzr1kL3IU8ZUAqDb7wtdX8w=="],
"@napi-rs/canvas-linux-x64-musl": ["@napi-rs/canvas-linux-x64-musl@1.0.13", "", { "os": "linux", "cpu": "x64" }, "sha512-bGY1r26S2bs8UNFJcPWcCSNdCFbKep5PATsvtgtC9A5u6SRgr2/jJE/RlhHL5m9ZpixNDhVkLY/5h16Kx5V9YA=="],
"@napi-rs/canvas-win32-arm64-msvc": ["@napi-rs/canvas-win32-arm64-msvc@1.0.13", "", { "os": "win32", "cpu": "arm64" }, "sha512-2vRPBZoaIvJrUkDhZTE2Hn0YrnpJRCWVM3EJACoBKjuWI6WgPxGtUoRMsxRviCsyks8bKTSFdyw14CellnG2qA=="],
"@napi-rs/canvas-win32-x64-msvc": ["@napi-rs/canvas-win32-x64-msvc@1.0.13", "", { "os": "win32", "cpu": "x64" }, "sha512-DUlW3lZPz63kYsf8mxgyyZuevhlH0T3/BGP9l+NazhB0F1+gL6D4hqZdnhZpw24DlWmTpATDcyYSZlJOi+DdfQ=="],
"@noble/ciphers": ["@noble/ciphers@2.4.0", "", {}, "sha512-AnjFn0Jv92laAkvMrghlFZq4qQCIN/4DxFV/eooqtC2YTjB7kBeLMS2T9KJX4Dn+ZVXLOwK0lSgqDtx9gvxtiw=="],
"@noble/hashes": ["@noble/hashes@2.4.0", "", {}, "sha512-X5XaVWZIBCT7HHZGm5I7ZQXDwLG+bGXuSrMQAW+7Zvl87h1kmc1ZB1VSRJcpUfoUrGQp4Fkoxm5kZ+Ms+aW+eA=="],
"@nodable/entities": ["@nodable/entities@3.1.0", "", {}, "sha512-LsS/DjHr+uDM647Gru/cA8+J3a3HfhttwCKLyuoyN7yXTFCxKBtSgMQBvrW9yNPa2/zDRUNuGPaH5QUFoa4arQ=="],
"@remix-run/router": ["@remix-run/router@1.23.4", "", {}, "sha512-q7j5geK7xs3UJSdm9/iytUNclBnLmYx1EnSeCFXHPeutdqgIMeFeHtUZgS3EhlKxdBEAu8OwtJCwmLrEzpSs7Q=="],
"@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="],
"@stc/contracts": ["@stc/contracts@workspace:packages/contracts"],
"@stc/core": ["@stc/core@workspace:packages/core"],
@@ -104,10 +168,72 @@
"@types/react-dom": ["@types/react-dom@19.3.0", "", { "peerDependencies": { "@types/react": "^19.3.0" } }, "sha512-ZI7bU42mZXXKHn/qNLEw2IrbiINU7X5+vfgdixBHkCNpYWXjKgfQ/P+uyGb5CjOLB9UcnTeg3rylQtV2hym44Q=="],
"@types/sanitize-html": ["@types/sanitize-html@2.16.2", "", { "dependencies": { "htmlparser2": "^12.0" } }, "sha512-kuAbgDCFv418flrp3z4o/4ufLA/5ovmXRH0L419iZCPGO7w00znjeDI5Wy/KZWNIBpDQc7hucCK+J5almB6FyA=="],
"@xmldom/is-dom-node": ["@xmldom/is-dom-node@1.0.1", "", {}, "sha512-CJDxIgE5I0FH+ttq/Fxy6nRpxP70+e2O048EPe85J2use3XKdatVM7dDVvFNjQudd9B49NPoZ+8PG49zj4Er8Q=="],
"@xmldom/xmldom": ["@xmldom/xmldom@0.9.12", "", {}, "sha512-5AXjrcMClTryPe9LgZrygpB1lj7s0S9E0+W+AHaVKAVyHanafK86iPSvG5xHVSp/jC+VH1UXu0TAEmY279xH7A=="],
"anynum": ["anynum@1.0.1", "", {}, "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A=="],
"better-auth": ["better-auth@1.7.14", "", { "dependencies": { "@better-auth/core": "1.7.14", "@better-auth/drizzle-adapter": "1.7.14", "@better-auth/kysely-adapter": "1.7.14", "@better-auth/memory-adapter": "1.7.14", "@better-auth/mongo-adapter": "1.7.14", "@better-auth/prisma-adapter": "1.7.14", "@better-auth/telemetry": "1.7.14", "@better-auth/utils": "0.5.1", "@better-fetch/fetch": "1.3.3", "@noble/ciphers": "^2.2.0", "@noble/hashes": "^2.2.0", "better-call": "1.4.4", "defu": "^6.1.4", "jose": "^6.2.3", "kysely": "^0.28.17 || ^0.29.0", "nanostores": "^1.3.0", "zod": "^4.5.4" }, "peerDependencies": { "@lynx-js/react": "*", "@prisma/client": "^5.0.0 || ^6.0.0 || ^7.0.0", "@sveltejs/kit": "^2.0.0 || ^3.0.0", "@tanstack/react-start": "^1.0.0", "@tanstack/solid-start": "^1.0.0", "drizzle-kit": ">=0.31.4 || >=1.0.0-beta.1", "drizzle-orm": "^0.45.2 || >=1.0.0-rc.1 <2.0.0", "mongodb": "^6.0.0 || ^7.0.0", "mysql2": "^3.0.0", "next": "^14.0.0 || ^15.0.0 || ^16.0.0", "pg": "^8.0.0", "prisma": "^5.0.0 || ^6.0.0 || ^7.0.0", "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0", "solid-js": "^1.0.0", "svelte": "^4.0.0 || ^5.0.0", "vitest": "^2.0.0 || ^3.0.0 || ^4.0.0 || ^5.0.0", "vue": "^3.0.0" }, "optionalPeers": ["@lynx-js/react", "@prisma/client", "@sveltejs/kit", "@tanstack/react-start", "@tanstack/solid-start", "drizzle-kit", "drizzle-orm", "mongodb", "mysql2", "next", "pg", "prisma", "react", "react-dom", "solid-js", "svelte", "vitest", "vue"] }, "sha512-QxDCI0pOCCHDYmruisKhKyrvOJ6Icv9xg0RA6ZUysfKFXENYwhNNpwUlkE/NmViH2nDMgkKcymmLr55pnpR9vw=="],
"better-call": ["better-call@1.4.4", "", { "dependencies": { "@better-auth/utils": "^0.5.0", "@better-fetch/fetch": "^1.3.1", "rou3": "^0.9.1", "set-cookie-parser": "^3.1.2" }, "peerDependencies": { "zod": "^4.0.0" }, "optionalPeers": ["zod"] }, "sha512-Gq4XdGFy4dmUxFEAX2nj0ycFcFbD9ZWh8YROouZoyXeF1M11TpYbplk0pDByTFF8Wz+MYa6W9052c9m7dD+mfw=="],
"bun-types": ["bun-types@1.4.4", "", { "dependencies": { "@types/node": "*", "undici-types": "*" } }, "sha512-yjrJh5oGX7mMPKGd19YXyIjmEuj4G4zNC30bs7PM9Az7NQ3ndF3j6hbeRdvJTRQEhJ/FYQwW0a2moweEi2hMxQ=="],
"csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="],
"dayjs": ["dayjs@1.11.23", "", {}, "sha512-QDTCU0M0MxR3hQfnlDJfwekQiaanm1ubOD231u73WBckQ/fsamwRLiE2GBz6D3a/xF1NgfiDLJjXBa1hYOYTtQ=="],
"defu": ["defu@6.1.7", "", {}, "sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ=="],
"dom-serializer": ["dom-serializer@3.1.1", "", { "dependencies": { "domelementtype": "^3.0.0", "domhandler": "^6.0.0", "entities": "^8.0.0" } }, "sha512-4MEa38/QexBob6gFNwu+EGdWvhJ1OKuNwdYY3Y3NyeWDQfnGeDYQUDfIRzWu5B5gsv03so2Uxd28YC6zrsx3Lw=="],
"domelementtype": ["domelementtype@3.0.0", "", {}, "sha512-umCQid3jKbDmVjx8jGaW7uUykm4DEUeyV21hPxNMo2nV955DhUThwqyOIDtreepP31hl84X7G5U9ZfsWvIB3Pg=="],
"domhandler": ["domhandler@6.0.1", "", { "dependencies": { "domelementtype": "^3.0.0" } }, "sha512-gYzvtM72ZtxQO0T048kd6HWSbbGCNOUwcnfQ01cqIJ4X2IYKFFHZ5mKvrQETcFXxsRObZulDaKmy//R7TPtsBg=="],
"domutils": ["domutils@4.0.2", "", { "dependencies": { "dom-serializer": "^3.0.0", "domelementtype": "^3.0.0", "domhandler": "^6.0.0" } }, "sha512-qI4JLRKnSzqFqr7hAlS5xQDusBCjKSEG4t4+7aNrIQMHBcsC2TGEhuyABJdYkgSewL57PNLYEiibY2iPKhKpaA=="],
"entities": ["entities@8.1.0", "", {}, "sha512-kxL7msIffSuh9aaFAMD7rxAIuTRMAHMeBtgHW2yUdWw732ZNh4MehkF2gdjvtdmikkaIP9bFDDJOPlsvm7avrA=="],
"escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="],
"escape-string-regexp": ["escape-string-regexp@4.0.0", "", {}, "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="],
"fast-xml-builder": ["fast-xml-builder@1.3.1", "", { "dependencies": { "path-expression-matcher": "^1.6.2", "xml-naming": "^0.3.0" } }, "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug=="],
"fast-xml-parser": ["fast-xml-parser@5.11.2", "", { "dependencies": { "@nodable/entities": "^3.0.1", "fast-xml-builder": "^1.2.0", "is-unsafe": "^2.0.0", "path-expression-matcher": "^1.6.2", "strnum": "^2.4.2", "xml-naming": "^0.3.0" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-R9iDuNrQYeQut46cn2r2wHKn4HYzVDvDm5J1wW+koZewykv0yuO2HChTYeZtrULyHIDM9cj9TUXloCsueCQUog=="],
"htmlparser2": ["htmlparser2@12.0.0", "", { "dependencies": { "domelementtype": "^3.0.0", "domhandler": "^6.0.0", "domutils": "^4.0.2", "entities": "^8.0.0" } }, "sha512-Tz7u1i95/g2x2jz81+x0FBVhBhY5aRTvD3tXXdFaljuNdzDLJ8UGNRrTcj2cgQvAg3iW/h77Fz15nLW0L0CrZw=="],
"is-plain-object": ["is-plain-object@5.1.0", "", {}, "sha512-bUi/yjmtKYcRVUtWRGr0UA6xEFh2I6zWUwMrUXB3s7bmYCaZ8a+0ZsTRkrawh/mzlSD1Y0Ph8bp/U+TvBpWDNw=="],
"is-unsafe": ["is-unsafe@2.0.2", "", {}, "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ=="],
"jose": ["jose@6.2.14", "", {}, "sha512-3JasLuki1NRLQvY4wCoC37KZk4qxoBx2k2aFqLxT84IezqW0bkJKnNtxnZj/Fu1xsQivoc/lytxaRHJoGLjJ1A=="],
"kysely": ["kysely@0.29.7", "", {}, "sha512-fmIq+7kv4WbFm2tcfznRzikqB4cuNpgZ40PSU2fAJ2/OO5XhojqnTPrmgrCjYSRJVwBln03XcRXJGnw9H099IQ=="],
"launder": ["launder@1.7.2", "", { "dependencies": { "dayjs": "^1.11.23" } }, "sha512-DLg3HPnHUfBi5/MxMLmJD12dmlMpFEC2HgMW6vkZ/9JR0RU00kXoRGvDxAvFmdO0o610OA77i4FgNTLucmhDVg=="],
"marked": ["marked@18.1.2", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-DfOnvLAQ6WYeWxWO1u4RyFOeqyHup6dks5PPqnoFiooU7ipRvidO/2j8plhAHIiZg574E9ZlzSjUiPMLAzFkgg=="],
"nanoid": ["nanoid@3.3.20", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ=="],
"nanostores": ["nanostores@1.5.5", "", {}, "sha512-FixtE239Gl6Gz0ZGCK0WzvNwObNCIwVT+L7zQcf1QgcA6AzBxblzqe9OT88qiucNkhsjSOqiogSaF5jVbJiqSA=="],
"parse-srcset": ["parse-srcset@1.0.2", "", {}, "sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q=="],
"path-expression-matcher": ["path-expression-matcher@1.6.2", "", {}, "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ=="],
"pdfjs-dist": ["pdfjs-dist@6.4.299", "", { "optionalDependencies": { "@napi-rs/canvas": "^1.0.10" } }, "sha512-AVl138zALtfaAPvADulE0PZThbYzCBS79nL4pOSL/6Sm/4AH5A21BD9VHt97OlCuzJuCpmeZtAtkinisF4Vb1g=="],
"picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="],
"postcss": ["postcss@8.5.32", "", { "dependencies": { "nanoid": "^3.3.20", "picocolors": "^1.1.1", "source-map-js": "^1.2.2" } }, "sha512-LJlKCCAUFKOuZxipUdYw1NzIITEsPcgyOG9LkvRJvuKj/bRzbuRY0ARXvL0pWGpsZvn8jku+LMep2eU0seHp8g=="],
"react": ["react@19.3.1", "", {}, "sha512-QNhWdt7nrgVWWQG58QuGsAR3jv3Hbe4mb4vz7w2csnHavciZJgda1CPNaldF6eVxv90ud3jUXpKlHL69gO38yw=="],
"react-dom": ["react-dom@19.3.1", "", { "dependencies": { "scheduler": "^0.28.0" }, "peerDependencies": { "react": "^19.3.1" } }, "sha512-Y0bARz53lhEC1NMCASs0/j+faB/zqt3R8hpvg1e8+61ZVJ4UL0UdC4cBbXfZMcIuhrfgeILTLe+ZeBCavQPtKQ=="],
@@ -116,14 +242,46 @@
"react-router-dom": ["react-router-dom@6.30.6", "", { "dependencies": { "@remix-run/router": "1.23.4", "react-router": "6.30.6" }, "peerDependencies": { "react": ">=16.8", "react-dom": ">=16.8" } }, "sha512-0RHKZz7wwffvkU+2MFVT2NnjK44ssLEV+m0CAJaS2Ksmorrwj7WxH00jO0SOCW26/tINUnJHToXblDs33I38YQ=="],
"rou3": ["rou3@0.9.2", "", {}, "sha512-3SOzvaAg8rkHrXtRjpCvCvbyO5to9oOO27Z/XqHEYXfMRVSw/qMIVdmaOk9W2lcRLtR6dlqTjo9hDeJk70QBYQ=="],
"samlify": ["samlify@2.15.1", "", { "dependencies": { "@authenio/xml-encryption": "^6.0.2", "@xmldom/xmldom": "^0.9.12", "xml": "^1.0.1", "xml-crypto": "^6.4.0", "xml-escape": "^1.1.0", "xpath": "^0.0.35" } }, "sha512-peZitMO2cAnf0pY45H0yw0Drqmby7U91yRovH3AH8TdwYCNRnmidVORXIjcK+21cBk3Mjl8EVz8Nvuw/v20rAw=="],
"sanitize-html": ["sanitize-html@2.18.1", "", { "dependencies": { "escape-string-regexp": "^4.0.0", "htmlparser2": "^12.0.0", "is-plain-object": "^5.1.0", "launder": "^1.7.2", "parse-srcset": "^1.0.2", "postcss": "^8.5.27" } }, "sha512-2X5lC8fYla8YaT8g73Nj+HcLWXan5dLyCWLgVvaOmfw7JI78IRsyNtIDytTHdn7hBQbIkCah7Zbmd5jabzeDig=="],
"scheduler": ["scheduler@0.28.0", "", {}, "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw=="],
"set-cookie-parser": ["set-cookie-parser@3.2.1", "", {}, "sha512-Npd1Pt+4zb3BrSZDPd2qFwRpS6b1EUEasRrO8xS0zMJi0fApXXK5YuzP0minyJnxwnFHoVoJwA55q0A0cu63gQ=="],
"source-map-js": ["source-map-js@1.2.2", "", {}, "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw=="],
"strnum": ["strnum@2.5.1", "", { "dependencies": { "anynum": "^1.0.1" } }, "sha512-UM4nTLWzQiDBseVxUGxN0wJzIk0UU89Qi7pKXz9qISo+lhviJikTuikBsK/oFRlpvYHvSdz0TVoLyQ+sEpYwxA=="],
"tldts": ["tldts@7.4.19", "", { "dependencies": { "tldts-core": "^7.4.19" }, "bin": { "tldts": "bin/cli.js" } }, "sha512-48dH6aSpQnpbzDUc4vIQZ1VjB8tkK+e8c1yx05/SZK+rkssgjEOlxDd/yU8/a4vGYQ9vRgJr9dwMq9TJ0b3X2Q=="],
"tldts-core": ["tldts-core@7.4.20", "", {}, "sha512-C630TWXZCZmLq0F2bW4QqwnaaxCQ/svpAqvJwnC4JvgEM5ohhBnGsEzTLSOXcN+xjz+kEaHS5uoWIw2klnqd0A=="],
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
"undici-types": ["undici-types@8.12.3", "", {}, "sha512-EXE4ydL3S76dFb27UFjNVZHTyv/TLdATj/KUe29ptdcg6qXL48J2tZ3126E5kICj5jn781OR+XlVH1N6Mz8jbA=="],
"xml": ["xml@1.0.1", "", {}, "sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw=="],
"xml-crypto": ["xml-crypto@6.4.0", "", { "dependencies": { "@xmldom/is-dom-node": "^1.0.1", "@xmldom/xmldom": "^0.8.15", "xpath": "^0.0.33" } }, "sha512-d21ModhBO/+QWWAmOjyDeQJuHNmsR/HtOxyBchiTbKfGNKuHYwQNd+SlLhwyyrwoAGLp4Qc6BQ7nAtnM47dHSQ=="],
"xml-escape": ["xml-escape@1.1.0", "", {}, "sha512-B/T4sDK8Z6aUh/qNr7mjKAwwncIljFuUP+DO/D5hloYFj+90O88z8Wf7oSucZTHxBAsC1/CTP4rtx/x1Uf72Mg=="],
"xml-naming": ["xml-naming@0.3.0", "", {}, "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ=="],
"xpath": ["xpath@0.0.35", "", {}, "sha512-8O+m2N04SwLpGw/1Grz2HD8ejnWDV7M1dMPgKKuOjhYh5W9H4pel/+06kInohIdpS2FeSbL4i8ID8c4tpxwq9Q=="],
"zod": ["zod@4.6.7", "", {}, "sha512-g8W2D2kU6a8untc053fX1uVBKzM7yqAxW/3zwk83+6RyF2+SKOuV+c0wBDhUwzV1PaGapmZH+rmcUzPf+3CGwg=="],
"@authenio/xml-encryption/@xmldom/xmldom": ["@xmldom/xmldom@0.8.15", "", {}, "sha512-/5NV/vDALVFDXgLmfsy9TRCBlKwO2LNBFzpzvb9iIj+jR+eSc6DLYYvVOdivT/jm7MtU6TebYuRmzEOI7w40UA=="],
"@types/node/undici-types": ["undici-types@8.9.0", "", {}, "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg=="],
"xml-crypto/@xmldom/xmldom": ["@xmldom/xmldom@0.8.15", "", {}, "sha512-/5NV/vDALVFDXgLmfsy9TRCBlKwO2LNBFzpzvb9iIj+jR+eSc6DLYYvVOdivT/jm7MtU6TebYuRmzEOI7w40UA=="],
"xml-crypto/xpath": ["xpath@0.0.33", "", {}, "sha512-NNXnzrkDrAzalLhIUc01jO2mOzXGXh1JwPgkihcLLzw98c0WgYDmmjSh1Kl3wzaxSVWMuA+fe0WTWOBDWCBmNA=="],
}
}
docker-compose.yml
+72
View File
@@ -0,0 +1,72 @@
# STCourses with Docker Compose. Project name "stc", so containers are stc-app-1 and stc-postgres-1.
#
# SQLite (default): STC_SECRET=... docker compose up -d
# Postgres: STC_SECRET=... STC_DATABASE_URL=postgres://stc:stc@postgres:5432/stc \
# docker compose --profile postgres up -d
#
# Course repos go in ./courses (or set STC_COURSES_HOST); files to import go in ./import-inbox (or STC_IMPORT_HOST).
# Both are mounted read-only: originals are never changed. See docs/operations.md.
name: stc
services:
app:
build: .
image: stc-app
restart: unless-stopped
ports:
- "${STC_HOST_PORT:-3000}:3000"
environment:
STC_SECRET: ${STC_SECRET:?set STC_SECRET to a long random string}
STC_PUBLIC_URL: ${STC_PUBLIC_URL:-http://localhost:3000}
STC_DATABASE_URL: ${STC_DATABASE_URL:-sqlite:///data/stcourses.db}
STC_UPLOAD_MAX_MB: ${STC_UPLOAD_MAX_MB:-100}
STC_AI_ALLOWED_BASE_URLS: ${STC_AI_ALLOWED_BASE_URLS:-}
STC_TRANSCRIBE_URL: ${STC_TRANSCRIBE_URL:-}
STC_TRANSCRIBE_KEY: ${STC_TRANSCRIBE_KEY:-}
STC_TRANSCRIBE_MODEL: ${STC_TRANSCRIBE_MODEL:-}
STC_ALLOWED_ORIGINS: ${STC_ALLOWED_ORIGINS:-}
# Sign-in providers (unset ones stay off). Social: STC_AUTH_<GITHUB|GOOGLE|MICROSOFT|GITLAB|DISCORD>_ID/_SECRET.
STC_AUTH_GITHUB_ID: ${STC_AUTH_GITHUB_ID:-}
STC_AUTH_GITHUB_SECRET: ${STC_AUTH_GITHUB_SECRET:-}
STC_AUTH_GOOGLE_ID: ${STC_AUTH_GOOGLE_ID:-}
STC_AUTH_GOOGLE_SECRET: ${STC_AUTH_GOOGLE_SECRET:-}
STC_AUTH_MICROSOFT_ID: ${STC_AUTH_MICROSOFT_ID:-}
STC_AUTH_MICROSOFT_SECRET: ${STC_AUTH_MICROSOFT_SECRET:-}
STC_AUTH_GITLAB_ID: ${STC_AUTH_GITLAB_ID:-}
STC_AUTH_GITLAB_SECRET: ${STC_AUTH_GITLAB_SECRET:-}
STC_AUTH_DISCORD_ID: ${STC_AUTH_DISCORD_ID:-}
STC_AUTH_DISCORD_SECRET: ${STC_AUTH_DISCORD_SECRET:-}
STC_AUTH_OIDC_ISSUER: ${STC_AUTH_OIDC_ISSUER:-}
STC_AUTH_OIDC_ID: ${STC_AUTH_OIDC_ID:-}
STC_AUTH_OIDC_SECRET: ${STC_AUTH_OIDC_SECRET:-}
STC_AUTH_OIDC_NAME: ${STC_AUTH_OIDC_NAME:-}
STC_AUTH_OIDC_SCOPES: ${STC_AUTH_OIDC_SCOPES:-}
STC_AUTH_OIDC_DISCOVERY_URL: ${STC_AUTH_OIDC_DISCOVERY_URL:-}
STC_AUTH_GROUPS_CLAIM: ${STC_AUTH_GROUPS_CLAIM:-}
depends_on:
postgres:
condition: service_healthy
required: false # only with --profile postgres
volumes:
- stc-data:/data
- ${STC_COURSES_HOST:-./courses}:/courses:ro
- ${STC_IMPORT_HOST:-./import-inbox}:/import:ro
postgres:
image: postgres:17
profiles: ["postgres"]
restart: unless-stopped
environment:
POSTGRES_USER: stc
POSTGRES_PASSWORD: ${STC_PG_PASSWORD:-stc}
POSTGRES_DB: stc
volumes:
- stc-pg:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U stc -d stc"]
interval: 5s
retries: 20
volumes:
stc-data:
stc-pg:
docs/contract-requests/ai.md
+81
View File
@@ -0,0 +1,81 @@
# Contract requests from WP-D (AI)
Phase 1 worked around each of these inside `packages/feature-ai`. Phase 2 decides.
## 1. Feature factories in `startServer` (apps/server)
`aiFeature(ctx, opts?)` needs the `ServerContext` (db, courses, jobs, auth, config), but `startServer(config, features)` builds the
context itself and takes ready-made `Feature` objects. So `apps/server/src/main.ts` is not wired yet (the web side is:
one line in `apps/web/src/features.ts`). Proposal: `startServer(config, factories: ((ctx: ServerContext) => Feature)[])`,
then append `(ctx) => aiFeature(ctx, { draftSink: createDraftWriter(ctx) })`. Jobs must be registered after the
factories run (the job handler `ai.generate` lives on the returned feature).
Local workaround: `packages/feature-ai/scripts/dev.ts` builds the context and composes the feature for the dev server on port 3464.
## 2. Studio `DraftWriter` as the generator's sink (no adapter needed)
The studio exports `createDraftWriter(ctx, { identity? })` whose `writeDraftFiles(courseId, authorId, files, message)`
returns `{ branch, commit }` and commits only to `draft/<authorId>`. The AI package's `DraftSink` has the same method
and returns `DraftWriteResult = { location?, branch?, commit? }`, so the studio writer is assignable as is:
```ts
aiFeature(ctx, { draftSink: createDraftWriter(ctx, { identity: (id) => /* user name + email */ null }) })
```
`generateDraft` then reports `location: "draft/<author> @ <short commit>"` plus `branch` and `commit`; the generate page
says "Committed to your draft branch". Test: `test/generator.test.ts` ("a studio DraftWriter is a DraftSink as is").
Open point: the studio writer throws 404 for courses it does not manage (overlay courses such as Pure Snake). Phase 2
can wrap it to fall back to `createFsDraftSink(dataDir)` (drafts under `<dataDir>/drafts/<courseId>/<run>/`), or make
generation for unmanaged courses an explicit error. Until wired, the filesystem sink is the default.
## 3. Tool-call fields on `ChatMessage`
`ChatMessage` has `toolCallId` and `name` but no structured slot for an assistant turn with several tool calls or with
thinking blocks. The package encodes one tool call per assistant message (`content` = JSON input) and the Anthropic
provider keeps each response's raw content blocks (thinking with signature, text, tool_use) in memory by tool_use id
and replays them unchanged. Proposal: `ChatMessage.toolCalls?: { id, name, input }[]` and an opaque
`providerData?: unknown` the provider can round-trip.
## 4. Per-course mentor toggle in `ai_settings`
The frozen key is `(scope, owner_id)` with one `mentor_off` column, so a per-course toggle cannot be a row. The flag
lives in the user row's `config_json.mentorOff[courseId]`; `mentor_off` stays 0 (reserved for "off everywhere").
Proposal: add `course_id TEXT NOT NULL DEFAULT ''` to the key, or a small `ai_mentor_prefs(user_id, course_id, off)` table.
## 5. Citations on `ai_messages`
`ai_messages` has no column for an assistant message's sources. They are stored as an extra row with role `sources`
(JSON) right after the assistant row and folded back when reading. Proposal: `ai_messages.meta_json TEXT`.
## 6. Derived text cache layout (shared with WP-B)
Retrieval reads the importer's layout:
- `<dataDir>/cache/decks/<courseId>/<deckId>/text.json` = `[{ page, text }]` (page images alongside);
- `<dataDir>/cache/transcripts/<courseId>/<lessonId>.json` = `{ lessonId, text, segments: [{ start, end, text }] }`
(skipped when the lesson has a `transcript` file in the course, which the job writes too);
- plus plain `.txt/.md/.vtt/.srt` under `<dataDir>/cache/<courseId>/**` (older layout, kept for hand-made caches).
Proposal: export these paths from `@stc/core` (e.g. `cachePaths(dataDir)`) so both packages share one definition.
## 7. Learner lesson URL
Citations carry `lessonId`; the assistant links them to `/c/<courseId>/<lessonId>`. Every such link goes through
`lessonUrl(courseId, lessonId)` / `courseUrl(courseId)` in `packages/feature-ai/src/web/urls.ts` (also exported from
`@stc/feature-ai/web`), so a different WP-A route is a one-line change. Proposal: move that helper into
`@stc/contracts/web` so WP-A owns the route shape.
## 8. A global `author` role in core's `canView`
Generation and the studio file listing now require an authoring right (owner, the `author` role or a `course_authors`
row) and core's `canView(user, course, acl, isAuthor)`. `canView` only knows per-course authors (`isAuthor`), so a user
with the global `author` role cannot reach a `restricted` course without an ACL grant or a `course_authors` row.
Spec section 6 allows author rights "on all courses or per course", and the studio branch (`feature-studio`
`server/feature.ts:82-85`) currently grants the global role on every course. Proposal: core exports one
`canAuthor(user, course, acl, isAuthor)` (or `canView` treats `roles.includes("author")` like `isAuthor`), and both
the studio and the AI package call it. Until then the AI package follows core's `canView` (the stricter rule).
## 9. `STC_AI_ALLOWED_BASE_URLS` in `AppConfig`
Learners' own keys (BYOK) go only to `https://api.anthropic.com`, `https://api.openai.com/v1` or an origin on the
allowlist (SSRF guard). The allowlist is `STC_AI_ALLOWED_BASE_URLS` (comma-separated exact origins) plus the admin
setting `allowedBaseUrls` in the server AI config. `AppConfig` is frozen, so the package reads the variable from
`process.env` (injectable through `aiFeature(ctx, { env })`). Proposal: add `aiAllowedBaseUrls: string[]` to
`AppConfig` and `loadConfig`.
docs/contract-requests/import.md
+12
View File
@@ -0,0 +1,12 @@
# Contract requests from WP-B (Importer)
1. **`loadConfig` should add `<dataDir>/courses` to `courseDirs`.** The importer writes courses there. Worked around in `apps/server/src/main.ts` (the main guard pushes it) and in `packages/feature-import/dev/serve.ts`. `/api/import/info` reports `coursesDirServed` and the UI warns when it is false.
2. **`startServer` accepts feature factories** (`Feature | ((ctx) => Feature)`). A `Feature` needs `ServerContext`, but `ServerContext.jobs` is built from the features' job handlers. `importFeature(ctx)` only reads `ctx` while a job or request runs, so `main.ts` now builds `ctx` first, resolves the factories, then assigns `ctx.jobs`. The edit is about ten lines in `apps/server/src/main.ts`; Phase 2 should keep or replace it.
3. **Web entry for features.** `apps/web` must not pull a package's server code into the browser bundle. `@stc/feature-import` exposes a browser-safe entry as `./web` (package.json `exports`), but the root tsconfig `paths` (`@stc/*` to `packages/*/src`) cannot resolve `@stc/feature-import/web`, so `apps/web/src/features.ts` imports it by relative path. Suggest a tsconfig `paths` entry per feature, or a `src/web.ts` convention.
4. **Derived cache layout (no contract defines one).** Written by the importer, to be read by the learner and AI packages:
- `<dataDir>/cache/decks/<courseId>/<deckId>/p-001.jpg ...` (1280px wide JPEGs) and `text.json` (`[{ page, text }]`);
- `<dataDir>/cache/covers/<courseId>/frame-N.jpg` and `deck-<deckId>.jpg` (cover candidates);
- `<dataDir>/cache/transcripts/<courseId>/<lessonId>.json` (`{ lessonId, text, segments: [{ start, end, text }] }`).
Suggest a `cacheDir`/`deckPageUrl` helper in core, and a media route for `cache/decks` so the deck viewer can load pages.
5. **Video lessons have no description field.** Packt clips come with a one-line description. It is dropped, because `transcript` is the only Markdown a video lesson can carry and the transcription job writes it.
6. **`STC_IMPORT_ROOTS`** (":"-separated, default `<dataDir>/import-inbox`, created when `importFeature` is built) is read inside the import package. Core config could own it later.
docs/contract-requests/learner.md
+57
View File
@@ -0,0 +1,57 @@
# Contract requests from WP-A (learner)
## 1. Feature factories and the auth service (`apps/server/src/main.ts`, `@stc/contracts`)
**Problem.** Phase 0 passes `Feature[]` to `startServer`, but `ServerContext` is built inside `startServer`, so a feature that needs `ctx` (every Phase 1 package exports `xFeature(ctx)`) cannot be listed. There is also no way for a feature to replace the anonymous `AuthService`.
**Workaround in `feat/learner` (applied, small):** `startServer` accepts `FeatureEntry = Feature | ((ctx) => Feature & { auth?: AuthService })`. Factories run after `ctx` exists (with `ctx.jobs` set right after they return, so factories must not use `jobs` synchronously), and a feature carrying `auth` replaces `ctx.auth`. The `main` entry lists `learnerFeature`.
**Proposal.** Move both into the contract:
```ts
export type FeatureFactory = (ctx: ServerContext) => Feature;
export interface Feature { /* … */ auth?: AuthService; migrationsDirs?: string[] }
```
and let `startServer` take `Array<Feature | FeatureFactory>`.
## 2. Migrations per package
`migrate()` only reads `packages/db/migrations`. The learner package exports `migrationsDirs` and, until Phase 2 wires it in, runs its own migrations lazily inside `learnerFeature(ctx)` through `runMigrations(db, dirs)`, which uses the same `schema_migrations` table and file-name keys. When `migrate()` learns about `migrationsDirs`, the lazy call becomes a no-op; it can then be removed.
## 3. Package entry for the browser
**Problem.** The package index exports both the server (`learnerFeature`, which pulls in Better Auth and `@better-auth/sso`) and the browser half (`learnerWeb`, `LessonView`). A browser bundle of the index fails: `@better-auth/sso` imports `X509Certificate` from `node:crypto`, which has no browser polyfill. The root `tsconfig.json` maps `@stc/*` to `packages/*/src`, so package.json `exports` subpaths are not used either.
**Workaround in `feat/learner` (applied).** `LessonView` and `learnerWeb` are exported from the index (for server-side and type use) and from a browser-only entry, `@stc/feature-learner/web`. That subpath resolves through the existing mapping to `packages/feature-learner/web/src/index.ts`, a one-line re-export of `src/web`. `apps/web/src/features.ts` imports `learnerWeb` from it, and the studio should import `LessonView` from it too.
**Proposal.** Add `"@stc/*/web": ["packages/*/src/web"]` ahead of the general mapping in `tsconfig.json` (Bun honours tsconfig paths), and let every feature package expose `src/web/index.ts`. The shim directory can then be deleted. This also settles the importer's relative import in `features.ts`.
## 4. LessonView for the studio preview
`import { LessonView } from "@stc/feature-learner/web"`. Props: `lesson: LessonPayload` (the JSON from `GET /api/courses/:id/lessons/:lessonId`), plus optional `look`, `track`, `onTrackChange`, `preview`. With `preview` set, it makes no notes, attempt or progress requests; it still loads code listings and decks through the normal media routes. It injects its own stylesheet (scoped under `.lr`) and works with or without a router.
## 5. Content-Security-Policy on the SPA (review C1, ruling R16)
**Applied in `feat/learner`.** The SPA's CSP is a `<meta http-equiv="Content-Security-Policy">` tag in `apps/web/index.html`, ahead of every script. `Bun.serve` serves the HTML import directly, as in Phase 0; there is no proxy and no change to proxy environment variables. The policy:
```
default-src 'self'; script-src 'self' 'sha256-<Bun dev helper>'; style-src 'self' 'unsafe-inline';
img-src 'self' data: blob: https:; media-src 'self' blob:; font-src 'self' data:; connect-src 'self';
worker-src 'self' blob:; frame-src 'self'; object-src 'none'; base-uri 'self'; form-action 'self'
```
- **No inline script.**
- The one hash is the inline helper Bun's dev server injects (the `/_bun/unref` beacon). Production serving does not inject it.
- If a Bun upgrade changes the helper, the only effect is a dev-only CSP console error; `test/csp.test.ts` (dev mode) will then fail and name it.
- **Dev HMR works:** the HMR websocket `ws://<host>/_bun/hmr` is same-origin, and Chrome matches it with `connect-src 'self'` (checked: handshake 101, no violations).
- **The runner frame** (`/api/learner/runner-frame`) keeps its own header-based CSP, including `sandbox allow-scripts` and `frame-ancestors 'self'`.
**Proposal for core (Phase 2).**
- A meta tag cannot carry `frame-ancestors`, `sandbox` or `report-uri`. Core should additionally send `Content-Security-Policy: frame-ancestors 'self'` (or `X-Frame-Options: SAMEORIGIN`), plus `X-Content-Type-Options: nosniff`, on HTML responses. That is possible if Bun ever allows headers on HTML imports, or by serving a production static build (`bun build apps/web/index.html`) through a normal route that sets headers.
- Ideally core then sends the whole policy as a header and the meta tag stays as defence in depth.
- Every feature's web code must keep to this policy: no inline scripts or event handlers, and no `eval`. The learner and studio preview, through `LessonView`, comply.
## 6. `canView` and the global `author` role (review I1)
**Ruling applied.** Learner routes now call core `canView` with per-course authorship only (`course_authors`), exactly like core's `/api/media`. A user with only the global `author` role gets a 404 for a restricted course they are not listed on, on every route.
**Proposal for Phase 2.** Extend core `canView` so the global `author` role can view every course (the studio and AI need the same rule), for example `canView(user, course, acl, isCourseAuthor)` returning true when `user.roles.includes("author")`. The learner then needs no change: it already delegates to `canView`.
docs/contract-requests/studio.md
+89
View File
@@ -0,0 +1,89 @@
# Contract requests from WP-C (studio)
Each item says what is missing, what the studio does locally for now, and what Phase 2 should change.
## 1. Features that need the ServerContext (apps/server)
`startServer(config, features: Feature[])` builds the `ServerContext` internally, so a feature factory such as `studioFeature(ctx)` cannot be put into the feature list. The studio therefore does **not** touch `apps/server/src/main.ts`. Locally it runs through `packages/feature-studio/dev/serve.ts`, which mirrors `startServer`.
Proposal:
- `startServer(config, features: (Feature | ((ctx: ServerContext) => Feature))[])`, and main.ts adds `studioFeature`.
- An optional `stop?(): void | Promise<void>` on `Feature`, called from `stop()`. The studio uses it to stop its published-worktree watcher (an unref'd interval, so it never blocks exit).
## 2. Learners must read studio courses from the published worktree (config / main.ts)
Layout after the review (controller ruling on I4):
- `<dataDir>/studio/repos/<id>.git`: studio bare repos. They contain no `stcourse.yaml`, so discovery never loads them.
- `<dataDir>/worktrees/<id>/published`: the published branch, which is what learners read.
- `<dataDir>/worktrees/<id>/drafts/<author>`: drafts. They sit at depth 3, so the depth-2 discovery never loads them.
- `<dataDir>/courses/<id>`: the importer's plain folders (WP-B), untouched by the studio.
`createCourseRepository(config.courseDirs)` scans neither `<dataDir>/worktrees` nor `<dataDir>/courses`. The package exports `studioCourseDirs(config)`, which returns the configured dirs plus `<dataDir>/courses` and `<dataDir>/worktrees`. Tests and the dev entry use it.
Proposal: main.ts calls `createCourseRepository(studioCourseDirs(config))`, or `loadConfig` appends both dirs.
Imported courses (`source.kind` other than `authored`, with their root under `<dataDir>/courses`) are read-only in the studio for v1, with the message "Imported course: read-only in the studio for now". There is a `TODO(studio)` in `feature.ts` (`readOnlyMessage`) for converting one into a studio-managed repo. The studio refuses to create a course whose id already exists as `<dataDir>/courses/<id>`. WP-B should likewise refuse an import id that `<dataDir>/studio/repos/<id>.git` already uses.
## 3. Live preview: swap in the learner `LessonView`
The studio preview sits behind an adapter in `@stc/feature-studio/web`:
```ts
export interface LessonPreviewProps {
courseId: string;
markdown: string;
lesson?: { id: string; title: string; type: string; anchor?: string } | null;
resolveUrl: (rel: string) => string; // course-relative path -> URL of the draft file
}
export function setLessonPreview(c: React.ComponentType<LessonPreviewProps>): void;
```
v1 uses a small, safe built-in Markdown renderer. Proposal: `@stc/feature-learner` exports a `LessonView` that can render from a Markdown string plus a URL resolver, without fetching the published course. Phase 2 then calls `setLessonPreview(adapter)` in `packages/feature-studio/src/web.ts`, so drafts are previewed with the real renderer.
## 4. Web entry points (tsconfig paths)
Phase 1 packages have server code (`Bun.$`, `node:fs`) that must not enter the browser bundle, so the studio has a separate browser entry, `src/web.ts`. The `@stc/*` → `packages/*/src` path maps `@stc/feature-studio/web` to a directory that does not exist, and the workspace has no `node_modules/@stc` links. For now, `apps/web/src/features.ts` imports it with a relative path.
Proposal: add `"@stc/*/web": ["packages/*/src/web.ts"]` to the root tsconfig `paths`, and switch the line to `import { studioWeb } from "@stc/feature-studio/web"`.
## 5. Role-aware navigation (contracts/web.ts)
`WebRoute.nav` cannot be limited by role, so the header "Studio" link shows for everyone. The page itself tells learners and anonymous users that the studio is for authors, and the API returns 401/403.
Proposal: add `nav.roles?: Role[]`, and have the shell show the link only to matching users. This needs the learner package's session in the web shell.
## 6. Draft API for WP-D (AI)
Exported from `@stc/feature-studio`:
```ts
export interface DraftWriter {
writeDraftFiles(courseId: string, authorId: string, files: { path: string; content: string }[], message: string): Promise<{ branch: string; commit: string }>;
}
export function createDraftWriter(ctx: { config: Pick<AppConfig, "dataDir"> }, opts?: {
identity?: (authorId: string) => Promise<{ name: string; email: string } | null> | { name: string; email: string } | null;
}): DraftWriter;
```
- It writes and commits on `draft/<authorId>` and never publishes.
- Without `identity`, commits are authored as `<authorId> <authorId@users.stcourses.invalid>`. WP-D should pass the signed-in user's name and email.
- It throws for courses not managed by the studio, and for paths that are absolute, contain `..` or touch `.git`.
- `studioFeature(ctx).draftWriter` is the same API, already bound to `ctx`.
## 7. Authors and roles (decision to confirm)
Studio access goes to:
- `owner` and `author` roles: all studio courses, and they can create courses;
- users with a `course_authors` row: that course only.
Creating a course inserts the creator into `course_authors`. No new tables are added, so `migrationsDirs` is `[]`.
## 8. External pushes straight into the bare repo (no action needed)
The platform reloads when the published worktree's HEAD changes: an external commit in that worktree, a studio publish, or a Sync. A direct `git push` into `<dataDir>/courses/<id>` is refused by git itself ("branch is currently checked out", checked with git 2.47), so the published files cannot silently fall behind `main`. Outside tools should commit in the published worktree, or push to a configured remote that the author then Syncs.
## 9. Core media route serves SVG/HTML inline (core, Phase 2)
The studio's draft `raw` route now sends `Content-Security-Policy: …; sandbox` and `X-Content-Type-Options: nosniff`. The learner-facing `/api/media/:courseId/*` in `packages/core` serves author-supplied SVG inline from the app's origin and sets neither header.
Proposal: add the same two headers in `serveMedia`.
docs/operations.md
+100
View File
@@ -0,0 +1,100 @@
# Operating STCourses
One server process (Bun) serves the API and the web app. Course content lives in folders; the database holds user
data only (accounts, progress, notes, attempts, settings), so deleting the database loses user data but never a
course.
## Run with Docker Compose
```sh
# SQLite in the data volume (the default)
STC_SECRET="$(openssl rand -base64 48)" docker compose up -d
# Postgres instead
STC_SECRET="$(openssl rand -base64 48)" \
STC_DATABASE_URL=postgres://stc:stc@postgres:5432/stc \
docker compose --profile postgres up -d
```
The first account that signs up becomes the owner. Open `http://localhost:3000` (or `STC_HOST_PORT`).
Mounts (all course mounts are read-only; STCourses never changes your originals):
| Host | Container | What |
|---|---|---|
| volume `stc-data` | `/data` | database (SQLite), imported courses (`/data/courses`), studio repos (`/data/studio/repos`) and their published worktrees (`/data/worktrees`), caches (`/data/cache`), AI drafts for non-studio courses (`/data/drafts`) |
| `STC_COURSES_HOST` (default `./courses`) | `/courses` | course repos, one folder with a `stcourse.yaml` per course |
| `STC_IMPORT_HOST` (default `./import-inbox`) | `/import` | files the owner may import (Packt zips, video folders, PDFs, code zips) |
An overlay in `overlays/` (baked into the image) names its course repo by absolute path in `root:`. Mount that repo
read-only at the same path inside the container, for example
`/home/me/functional-programming-course:/home/me/functional-programming-course:ro`. If the repo is missing, the
course shows as broken with a diagnostic and every other course keeps working.
Back up the `stc-data` volume (and the Postgres volume `stc-pg` when you use Postgres).
## Configuration
| Variable | Default | Meaning |
|---|---|---|
| `STC_SECRET` | none (required in production) | encrypts users' own AI keys and signs sessions; use 32+ random characters |
| `STC_PUBLIC_URL` | `http://localhost:<port>` | the URL users open; sign-in checks the request origin against it |
| `STC_PORT` | `3000` | listen port |
| `STC_DATABASE_URL` | `sqlite://./data/stcourses.db` (image: `sqlite:///data/stcourses.db`) | `sqlite://<path>` or `postgres://…` |
| `STC_DATA_DIR` | `./data` (image: `/data`) | see the table above |
| `STC_COURSE_DIRS` | `./overlays:./courses` (image: `/app/overlays:/courses`) | `:`-separated folders scanned for courses; `<data>/courses` and `<data>/worktrees` are always added |
| `STC_IMPORT_ROOTS` | `<data>/import-inbox` (image: `/import`) | `:`-separated folders the importer may read |
| `STC_UPLOAD_MAX_MB` | `100` | studio upload cap |
| `STC_PYTHON` | `python3` | Python with PyMuPDF, for PDF page images and text (installed in the image) |
| `STC_TRANSCRIBE_URL`, `STC_TRANSCRIBE_KEY`, `STC_TRANSCRIBE_MODEL` | unset | Whisper-compatible endpoint for video transcripts; unset disables transcription |
| `STC_AI_ALLOWED_BASE_URLS` | unset | `,`-separated origins learners' own AI keys may use besides api.anthropic.com and api.openai.com |
| `STC_AUTH_<GITHUB/GOOGLE/MICROSOFT/GITLAB/DISCORD>_ID`, `_SECRET` | unset | social sign-in |
| `STC_AUTH_OIDC_ISSUER`, `_ID`, `_SECRET`, `_NAME`, `_SCOPES`, `_DISCOVERY_URL`, `STC_AUTH_GROUPS_CLAIM` | unset | generic OIDC sign-in; group claims drive restricted visibility |
| `STC_ALLOWED_ORIGINS` | unset | extra `,`-separated origins (e.g. `http://192.168.1.5:3000`) allowed to send writes and sign in, besides `STC_PUBLIC_URL`'s origin. `STC_TRUSTED_ORIGINS` is read as an alias |
Without Docker: `bun install`, then `STC_SECRET=… NODE_ENV=production bun apps/server/src/main.ts`. The importer needs
`ffmpeg`, `ffprobe`, `git`, `unzip` and Python with PyMuPDF on the host.
## Cross-site requests and the session cookie
- Every state-changing request (POST, PUT, PATCH, DELETE), sign-in included, must carry an `Origin` (or, without
one, a `Referer`) equal to `STC_PUBLIC_URL`'s origin or one in `STC_ALLOWED_ORIGINS`. Anything else gets 403.
Open STCourses at exactly that URL. If people reach it by several names (a LAN IP and a hostname), list the others
in `STC_ALLOWED_ORIGINS`.
- A write with a body must be `application/json` (uploads: `multipart/form-data`). HTML forms and `text/plain`
bodies, which another page can send without a CORS preflight, are refused.
- Exempt: the identity-provider callbacks (SAML ACS and SLO, OAuth/OIDC callbacks under `/api/auth`). The provider
posts those from its own origin, and Better Auth verifies the signed assertion or the OAuth state itself.
- The session cookie stays `SameSite=Lax` (Better Auth's default), not `Strict`. OAuth/OIDC sign-in returns from
the provider by a cross-site top-level navigation, and Better Auth needs its state cookie on that request. Under
`Strict` the browser withholds it, and social, OIDC and SAML sign-in break. The Origin check above covers the
same-site cases that `Lax` allows (other ports on the same host, sibling subdomains).
## Security headers
- The web app carries its Content-Security-Policy as a `<meta>` tag in `apps/web/index.html` (no inline script).
A meta tag cannot set `frame-ancestors`; if you need clickjacking protection, add
`Content-Security-Policy: frame-ancestors 'self'` (or `X-Frame-Options: SAMEORIGIN`) in your reverse proxy.
- Course media (`/api/media/…`) is sent with `X-Content-Type-Options: nosniff`, and SVG, HTML and JavaScript files
with `Content-Security-Policy: sandbox`, so author content cannot run script on the app's origin.
## Memory budget
The deployment must stay under 2 GB idle. Measured with this compose file (`--profile postgres`), the image built
from this repository, and four courses mounted read-only: the sample course, Pure Snake and Systemic Design (both
overlays, repos mounted at their own paths), and the imported Packt course "Mastering Docker Essentials" (68 videos,
9 modules, a 59-slide deck, code files; about 800 MB of media). Before idling, one owner signed up and opened the
library, a video lesson with its deck, a Pure Snake lesson and the sample exercise in headless Chrome. Then nothing
for 5 minutes, and `docker stats --no-stream`:
| Container | Memory (idle, after 5 min) | CPU |
|---|---|---|
| `stc-app-1` (Bun server, SPA bundle built and cached) | 84.7 MiB | 1.00% |
| `stc-postgres-1` (postgres:17) | 16.67 MiB | 3.52% |
| **Total** | **101.4 MiB** (about 5% of the 2 GB budget) | |
Measured 2026-10-30 on Linux 6.12 (Docker Compose v5.5.1, `oven/bun:1.3` = Bun 1.3.14). With SQLite there is
no Postgres container, so the total is the app's figure alone.
Video and PDF files are streamed from disk with Range requests; they never sit in memory. The SPA bundle is built
once, on the first page request, and cached.
docs/superpowers/plans/2026-10-30-ai.md
+178
View File
@@ -0,0 +1,178 @@
# WP-D AI Implementation Plan
> **For agentic workers:** executed inline by the WP-D agent with TDD (failing test first), one commit per task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Opt-in AI for STCourses: a course-aware learner assistant (retrieval, course tools, MCP tools, mentor mode) and a studio generator that writes house-style drafts. AI is off until an admin configures a server provider or a user saves their own key (BYOK).
**Architecture:**
- `packages/feature-ai` exports `aiFeature(ctx, opts?): Feature`, `aiWeb: WebFeature` and `migrationsDirs`.
- Providers implement the frozen `AiProvider` contract with plain `fetch` (no SDK): Anthropic Messages API (streaming SSE, tool use, default model `claude-opus-5-5`) and OpenAI-compatible chat completions (covers Ollama).
- Settings live in the existing `ai_settings` and `user_api_keys` tables (no new tables). Secrets are AES-GCM encrypted with a key derived by HKDF-SHA-256 from `ctx.config.secret`; they never leave the server.
- Retrieval is an in-process BM25 index per loaded course (course Markdown, transcripts, code, deck/transcript text under `<dataDir>/cache/<courseId>/`), plus the requesting user's notes at query time. Solution files are never indexed.
- The assistant runs a small tool loop over `search_course`, `read_lesson`, `list_notes` and admin-configured MCP servers (stdio and streamable HTTP), and streams NDJSON events to the browser.
- The generator builds house-style prompts (Pure Snake module shape, Systemic Design beats), parses `=== FILE: … ===` blocks from the model output, and hands them to a `DraftSink`. The default sink writes to `<dataDir>/drafts/<courseId>/<runId>/`; Phase 2 swaps in the studio's `createDraftWriter(ctx)`.
**Tech Stack:** Bun 1.3, TypeScript strict, WebCrypto, `Bun.SQL`, `Bun.spawn`, React 19 + React Router 6, `@stc/ui` primitives.
**Spec:** sections 9.2, 7, 2 (anti-goals). Brief: `.superpowers/sdd/2026-10-30-stcourses/phase-1-brief.md` (WP-D).
## Global Constraints
Inherited from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun only, plain CSS with tokens, both databases, AGPL, never write into the reference course repos, never commit licensed content, keyboard rules, commit trailer). Package-specific:
- **AI is off by default.** No provider configured and no BYOK key means every AI route answers `enabled: false` / 409, and the UI says how to turn it on.
- **Keys never go to the client.** API responses carry `hasApiKey: boolean`, never ciphertext or plaintext.
- **No real network in tests.** Providers take an injectable `fetch`; MCP tests use a local fake server (stdio child process and `Bun.serve` on port 0).
- **Mentor mode is on by default** and its rules go into the system prompt; when off, the UI shows "Mentor mode off: I'll give full answers".
- **Generated material is never published.** The generator only writes through a `DraftSink`.
## Review Focus
1. **Secrets:** BYOK and server keys are encrypted at rest (AES-GCM, random IV, HKDF key from `STC_SECRET`), a wrong secret fails to decrypt, and no route returns key material. Tests: Tasks 1, 2, 9.
2. **Mentor anti-goal:** mentor rules are in the system prompt unless the learner switched them off for that course; solution files (`browser.solution`, `solutions/`, `reference/`, `expansion/solution/`) are never retrieved or readable by tools. Tests: Tasks 5, 7.
3. **Generation never publishes:** drafts land only under the sink location, paths are sanitised (no `..`, no absolute paths). Test: Task 8.
4. **Access control:** assistant routes require that the user can view the course (`canView`); generator routes require owner, the `author` role or a `course_authors` row; admin routes require owner. Test: Task 9.
5. **Thinking blocks:** the Anthropic provider replays an assistant turn's raw content blocks (thinking, text, tool_use) unchanged inside one tool loop. Test: Task 3.
---
### Task 1: Package scaffold and secret box
**Files:** `packages/feature-ai/{package.json,src/index.ts,src/crypto.ts}`, test `packages/feature-ai/test/crypto.test.ts`
**Interfaces (Produces):**
- `createSecretBox(secret: string): { encrypt(plain: string): Promise<string>; decrypt(token: string): Promise<string> }`. HKDF-SHA-256 (salt `"stcourses/byok"`, info `"aes-gcm-256/v1"`) → AES-GCM-256; token `v1.<b64 iv>.<b64 ciphertext>`.
- `migrationsDirs: string[]` (empty: the package adds no tables).
- [x] Tests: round trip; two encryptions differ (random IV); a box with another secret rejects; a tampered token rejects.
- [x] Implement; commit `feat(ai): package scaffold and AES-GCM secret box`.
### Task 2: Settings repository
**Files:** `src/settings.ts`, test `test/settings.test.ts` (uses `forEachDb` from `packages/db/test/helpers.ts`)
**Interfaces:**
- `ProviderKind = "anthropic" | "openai"`; `ProviderConfig = { kind, model?, baseUrl?, effort? }`.
- `McpServerConfig = { id, name, transport: "stdio" | "http", command?, args?, env?, url?, headers?, courseIds?: string[] }` (no `courseIds` = global).
- `aiSettingsRepo(db, box)`:
- `getServer(): Promise<{ provider: (ProviderConfig & { hasApiKey }) | null; mcpServers }>`, `saveServer({ provider: (ProviderConfig & { apiKey?: string; clearApiKey?: boolean }) | null, mcpServers })`, `serverApiKey(): Promise<string | null>`.
- `getUser(userId): Promise<{ byok: (ProviderConfig & { hasApiKey }) | null; mentorOff: Record<courseId, boolean> }>`, `saveUserKey(userId, cfg & { apiKey })`, `removeUserKey(userId)`, `userApiKey(userId, kind)`, `setMentorOff(userId, courseId, off)`, `isMentorOff(userId, courseId)`.
- Storage: `ai_settings` row `('server','')` holds `{ provider, apiKeyCipher, mcpServers }`; row `('user', userId)` holds `{ byok, mentorOff }`. The per-course flag lives in `config_json` because the frozen primary key is `(scope, owner_id)`; the `mentor_off` column mirrors "off for every course" (always 0 for now). Contract request filed. Keys go in `user_api_keys` encrypted. Upserts are select-then-insert-or-update in a transaction.
- [x] Tests (both DBs): empty server config is `null` provider; saving a key stores ciphertext only (raw row does not contain the key) and `serverApiKey()` returns it; omitting `apiKey` on save keeps the old key; BYOK save/remove; mentor toggle per course is independent.
- [x] Implement; commit `feat(ai): settings repository for server config, BYOK keys and mentor toggles`.
### Task 3: SSE parsing and the Anthropic provider
**Files:** `src/providers/sse.ts`, `src/providers/anthropic.ts`, test `test/anthropic.test.ts`
**Interfaces:**
- `readSse(body: ReadableStream<Uint8Array>): AsyncIterable<{ event: string; data: string }>`.
- `createAnthropicProvider({ apiKey, model = "claude-opus-5-5", baseUrl = "https://api.anthropic.com", effort?, maxTokens = 64000, fetch? }): AiProvider`.
- POST `/v1/messages` with `stream: true`, headers `x-api-key`, `anthropic-version: 2023-06-01`. No `thinking`, `temperature` or forced `tool_choice` (Opus 5.5 rejects them). On the first-party endpoint, `fallbacks: "default"` with beta header `server-side-fallback-2026-07-01`.
- Message mapping: `system` messages join into `system`; an assistant `ChatMessage` with `toolCallId` and `name` is a tool call (content = JSON input); `tool` messages become `tool_result` blocks; consecutive same-role blocks merge.
- The provider remembers each response's full content blocks (thinking with signature, text, tool_use) by tool_use id and replays them unchanged for the matching assistant tool-call messages.
- Streams `{ text }` per `text_delta`, `{ toolCall }` per finished `tool_use` block; HTTP errors and `error` events throw with the API message; `stop_reason: "refusal"` yields a plain sentence.
- [x] Tests with a fake `fetch` that returns canned SSE: text streams in order; a tool_use with split `input_json_delta` yields one parsed toolCall; request body has model, system, tools as `input_schema`, no `thinking`/`temperature`; a second call after a tool round replays the thinking block with its signature unchanged before `tool_result`; non-2xx throws with the error message.
- [x] Implement; commit `feat(ai): Anthropic provider over fetch with streaming tool use`.
### Task 4: OpenAI-compatible provider
**Files:** `src/providers/openai.ts`, `src/providers/index.ts`, test `test/openai.test.ts`
**Interfaces:**
- `createOpenAiProvider({ baseUrl, apiKey?, model, fetch? }): AiProvider`. POST `<baseUrl>/chat/completions`, `stream: true`, tools as `{ type: "function", function: { name, description, parameters } }`. Assistant tool-call messages map to `tool_calls`; `tool` messages to `{ role: "tool", tool_call_id }`. Accumulates `delta.tool_calls` by index and emits on finish or `[DONE]`.
- `createProvider(cfg: ProviderConfig, apiKey: string | null, fetchImpl?): AiProvider`.
- [x] Tests: text deltas; fragmented tool-call arguments; no `Authorization` header without a key (Ollama); request mapping of a tool round.
- [x] Implement; commit `feat(ai): OpenAI-compatible provider (covers Ollama)`.
### Task 5: Retrieval
**Files:** `src/retrieval/{bm25.ts,course-index.ts}`, test `test/retrieval.test.ts`
**Interfaces:**
- `Bm25<T>` with `add(text, item)`, `search(query, k): { item, score }[]`.
- `Chunk = { path, kind: "lesson" | "transcript" | "markdown" | "code" | "deck" | "note", text, heading?, lessonId?, lessonTitle? }`.
- `buildCourseIndex(course: LoadedCourse, dataDir: string): Promise<CourseIndex>`; `CourseIndex.search(query, k, extra?: Chunk[]): Chunk[]`; `isSolutionPath(course, rel): boolean`.
- `getCourseIndex(course, dataDir)` caches per `LoadedCourse` object (a repository reload yields new objects) with a 5-minute TTL.
- Sources: every `.md` under the course root (depth-limited, skipping `.git`, `node_modules`), manifest code dirs and common code extensions (size cap 200 KB per file, 2 000 files per course), `.md/.txt/.vtt/.srt/.json` text under `<dataDir>/cache/<courseId>/`. Markdown splits at headings; long chunks split at ~1 500 chars. Paths are mapped to lessons through `file`, `transcript` and `hints.file` (anchor heading when it matches).
- Never indexed: `browser.solution` files and any path segment `solutions`, `solution`, `reference` under `expansion/`, or `reference/` in Systemic-Design style modules (see `isSolutionPath`).
- [x] Tests on the `sample` fixture: "sum of its arguments" hits `lessons/04-exercise.md` with lessonId `m2-text` or `m2-ex-add`; `exercises/add/solution.js` never appears; a deck text file in a temp `cache/sample/decks/d1/page-1.txt` is found with kind `deck`; note chunks passed as `extra` rank.
- [x] Implement; commit `feat(ai): BM25 course retrieval over lessons, transcripts, code, deck text and notes`.
### Task 6: MCP client
**Files:** `src/mcp/client.ts`, test `test/mcp.test.ts`, fixture `test/fixtures/fake-mcp.ts`
**Interfaces:**
- `connectMcp(cfg: McpServerConfig, opts?: { fetch?, timeoutMs? }): Promise<McpClient>`; `McpClient = { tools(): Promise<{ name, description, inputSchema }[]>; call(name, args): Promise<string>; close(): Promise<void> }`.
- JSON-RPC 2.0: `initialize` (protocolVersion `2025-06-18`), `notifications/initialized`, `tools/list`, `tools/call`. stdio = newline-delimited JSON over `Bun.spawn`; streamable HTTP = POST with `Accept: application/json, text/event-stream`, JSON or SSE replies, `Mcp-Session-Id` echoed.
- `mcpServersFor(courseId, servers)` selects global servers plus ones listing the course.
- [x] Tests: the fake server over stdio lists `echo` and `tools/call` returns its text; the same over HTTP (fake via `Bun.serve` on port 0, replying with SSE for `tools/call`) with the session id echoed; a server that exits gives a rejected promise, not a hang.
- [x] Implement; commit `feat(ai): MCP client over stdio and streamable HTTP`.
### Task 7: Assistant core
**Files:** `src/assistant/{prompt.ts,tools.ts,loop.ts}`, test `test/assistant.test.ts`
**Interfaces:**
- `buildSystemPrompt({ courseTitle, mentorText: string | null, mentorOn, sources: Chunk[], lessonTitle? }): string`. With mentor on and text present: a `## Course mentor rules` section with the file's text. With mentor off: a line saying full answers are allowed. Sources are numbered `[n] path (lesson title)` with their text; the prompt asks for citations by path.
- `courseTools({ ctx, course, userId, index })`: `ToolSpec[]` plus `run(name, input)` for `search_course`, `read_lesson` (refuses solution files) and `list_notes`.
- `runAssistant({ provider, messages, tools, signal, maxSteps = 6 }): AsyncIterable<AssistantEvent>` where `AssistantEvent = { type: "text", text } | { type: "tool", name } | { type: "error", message } | { type: "done", text }`. Tool calls become assistant messages with `toolCallId`/`name` and `tool` result messages; unknown tools and tool errors are fed back as error text.
- Citations: `citationsFor(chunks): { path, lessonId?, lessonTitle?, kind }[]` deduplicated.
- [x] Tests: mentor on puts the sample `MENTOR.md` text in the system prompt; mentor off omits it and states full answers; a fake provider that calls `search_course` then answers gets the tool result and the loop ends with text; `read_lesson` on the exercise returns markdown; solution paths are refused.
- [x] Implement; commit `feat(ai): assistant prompt, course tools and tool loop`.
### Task 8: Generator and draft sink
**Files:** `src/generator/{templates.ts,parse.ts,generate.ts,draft-sink.ts}`, test `test/generator.test.ts`
**Interfaces:**
- `DraftSink = { writeDraftFiles(courseId, authorId, files: { path, content }[], message): Promise<{ location: string; paths: string[] }> }`; `createFsDraftSink(dataDir)` writes to `<dataDir>/drafts/<courseId>/<runId>/` plus `DRAFT.json` (author, message, time). Same signature as the studio's `createDraftWriter(ctx).writeDraftFiles`.
- `GenerateRequest = { courseId, kind: "document" | "exercise" | "questions", style: "pure-snake" | "systemic-design", topic, sources: ({ kind: "lesson", lessonId } | { kind: "file", path } | { kind: "text", label, text })[], instructions? }`.
- `buildGeneratorPrompt(req, material, course)`: house-style system prompt (module shape: why, objectives, concepts, puzzle with "Think about it first" fold, everyday-programming bridge, exercises, hints as nudge / stronger hint / near-solution folds, check your understanding, in production, further reading; or Observe → Predict (+Discussion fold) → Spec → Build (Hint 1 — nudge / Hint 2 — concept / Hint 3 — pseudo-code) → Name it → Expansion Drop (+what this drop measures) → Further reading). Exercises: `README.md`, `starter.js`, `tests.js` (`export const tests` per the `ExerciseTest` contract), `solution.js`, shared fixture JSON across tracks when the course has tracks. Questions: check-your-understanding list with at least one predict and one "what would break if".
- `parseFiles(text): { files, warnings }` reads `=== FILE: <path> ===` … `=== END FILE ===`; rejects absolute and `..` paths.
- `checkHouseStyle(req, files): string[]` warnings for missing beats/tiers.
- `generateDraft({ provider, req, course, ctx, sink, authorId }): Promise<{ location, paths, warnings }>`; every Markdown file starts with a draft marker comment.
- [x] Tests: prompt for each style contains its beats; parse handles two files and rejects `../x`; fake provider output written by the fs sink lands under `<dataDir>/drafts/sample/…` and nothing changes in the course folder; missing tiers give warnings.
- [x] Implement; commit `feat(ai): house-style generator writing drafts through a DraftSink`.
### Task 9: Feature routes and jobs
**Files:** `src/feature.ts`, test `test/feature.test.ts`
**Interfaces:** `aiFeature(ctx, opts?: { fetch?; draftSink?; providerFactory?(cfg, key): AiProvider }): Feature` with:
- `GET /api/ai/status?courseId=` → `{ enabled, source: "byok" | "server" | null, mentor?: { available, on } }`.
- `GET|PUT /api/ai/admin/config` (owner), `POST /api/ai/admin/mcp/test` (owner; lists a server's tools).
- `GET /api/ai/account`, `PUT /api/ai/account/key`, `DELETE /api/ai/account/key` (signed in).
- `PUT /api/ai/courses/:courseId/mentor` `{ off }`.
- `POST /api/ai/courses/:courseId/chat` `{ message, conversationId? }` → `application/x-ndjson` stream: `{type:"meta", conversationId, mentorOn, citations}`, `text`, `tool`, `done`, `error`. Stores user and assistant turns in `ai_conversations`/`ai_messages`.
- `GET /api/ai/courses/:courseId/conversations`, `GET|DELETE /api/ai/conversations/:id`.
- `GET /api/ai/studio/sources?courseId=`, `POST /api/ai/studio/generate` (enqueues job `ai.generate`), `GET /api/ai/studio/jobs/:id`.
- Job `ai.generate` runs `generateDraft` with the requesting user's provider.
- [x] Tests through `createApp` with a fake `AuthService`: status disabled by default; admin config hides the key; non-owner gets 403; BYOK enables the assistant for that user only; chat with a fake provider streams text and `meta.citations` include `lessons/04-exercise.md`; system prompt holds mentor rules, and not after `PUT mentor {off:true}`; anonymous on a restricted course gets 404; generate as a learner gets 403; the job handler writes drafts under `<dataDir>/drafts/sample`.
- [x] Implement; commit `feat(ai): HTTP routes, chat streaming and generation job`.
### Task 10: Web UI
**Files:** `src/web/{index.ts,api.ts,AssistantPage.tsx,AdminAiPage.tsx,AccountAiPage.tsx,GeneratePage.tsx,ai.css}`, `apps/web/src/features.ts` (one line), test `test/web.test.tsx`
- `aiWeb: WebFeature` routes `/c/:courseId/assistant`, `/admin/ai` (nav admin "AI"), `/account/ai`, `/studio/generate`.
- Assistant: course name, mentor segmented toggle, the plain banner "Mentor mode off: I'll give full answers" when off, conversation rows, sources as rows with paths, composer with a primary Send pill. Disabled state explains how to enable AI.
- Admin: provider kind, model, base URL, write-only key, MCP server rows. Account: BYOK form and remove. Generate: course, kind, style, sources checklist, topic, instructions, job progress, list of draft files with location.
- `readNdjson(stream)` client helper.
- [x] Tests: `readNdjson` splits across chunk boundaries; `renderToString` of the banner component shows the exact sentence when off and nothing when on.
- [x] Implement, check in headless Chrome via `packages/feature-ai/scripts/dev.ts` on port 3464; commit `feat(ai): assistant panel, AI settings and studio generator UI`.
### Task 11: Contract requests and report
- [x] `docs/contract-requests/ai.md`: feature factories in `startServer`; tool-call fields on `ChatMessage`; per-course `ai_settings` key; studio `createDraftWriter` wiring; cache layout for deck text and transcripts; learner lesson URL.
- [x] Run `bun test`, `bun run typecheck`, the Postgres run; write `wp-ai-report.md`.
docs/superpowers/plans/2026-10-30-import.md
+56
View File
@@ -0,0 +1,56 @@
# WP-B Importer Implementation Plan
> Executed inline by the WP-B agent, task by task, TDD. Spec: `docs/superpowers/specs/2026-10-30-stcourses-design.md` sections 5, 8, 13.
**Goal:** `packages/feature-import` turns a Packt video zip, a companion code repo, a slide PDF or a folder of videos into an ordinary course folder under `<dataDir>/courses/<id>/`, as background jobs with visible progress, plus an owner-only admin UI at `/admin/import`.
**Architecture:**
- `importFeature(ctx)` returns a `Feature` with job handlers `import.packt-video`, `import.code-repo`, `import.pdf-deck`, `import.video-folder`, `import.transcribe` and owner-only `/api/import/*` routes. `ctx` is read lazily (inside handlers), because the job queue is built from the handlers.
- Courses are built in `<dataDir>/tmp/<job>` and renamed into `<dataDir>/courses/<id>` in one step. Attach jobs (code, deck) only touch courses under `<dataDir>/courses` and write the manifest last, through temp file + rename.
- Media: **copied** (`cp --reflink=auto`, so instant on CoW filesystems), or hard-linked with `options.media = "link"` (falls back to copy across filesystems). Symlinks out of the course are not an option: the media route 404s on paths that escape the course root. The originals are only ever read.
- PDFs: PyMuPDF through `py/pdf_tool.py` run as `python -I` in a subprocess (`STC_PYTHON`, default `python3`). Chosen over pdfjs/mupdf-wasm because it rasterises and extracts text natively, is already installed, is AGPL-compatible and costs no idle memory in the server. If it is missing, the deck is still attached and the job reports a warning.
- Derived data goes to `<dataDir>/cache/`: `decks/<course>/<deck>/p-001.jpg…` + `text.json`, `covers/<course>/*.jpg`, `transcripts/<course>/<lesson>.json`.
## Global Constraints
Inherited unchanged from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun conventions, AGPL, design tokens, no licensed content in the repo, commit trailer).
## Review Focus
1. Original files are never modified; attach jobs refuse courses outside `<dataDir>/courses` (overlays, authored repos).
2. Paths with spaces/non-ASCII (`S08 - Containerizing Applications`) survive unzip, copy and manifest round trip.
3. Packt parsing is structural (chapter blocks plus the section summary lists), not regexes on "Video N.M:" prefixes.
4. Failure leaves no half-built course visible; a failed PDF/ffmpeg step degrades to a warning where the course is still valid.
5. Routes: 401 without a user, 403 without the owner role.
## Tasks
### Task 1: Utilities (slug, HTML entities, natural sort, exec, manifest IO, sources)
- Files: `src/util/*.ts`, `test/util.test.ts`
- Interfaces: `slugify`, `uniqueCourseId`, `decodeEntities`, `naturalSort`, `run`, `readRawManifest`, `writeManifest` (validates with `ManifestSchema`), `openSource(path)` (zip via `unzip`, or directory).
- [ ] Failing tests for each helper; [ ] implement; [ ] commit.
### Task 2: Fixtures and media tooling
- Files: `test/fixtures.ts` (generates fake Packt zip, repo zip, PDF, video folder in a temp dir), `src/media.ts` (ffprobe duration, ffmpeg frame, copy/link), `src/pdf.ts`, `py/pdf_tool.py`.
- [ ] Failing tests (frame extraction, duration, PDF pages+text); [ ] implement; [ ] commit.
### Task 3: Packt parser and `import.packt-video`
- Files: `src/packt.ts`, `src/jobs/packt-video.ts`, `test/packt.test.ts`
- [ ] Parser test on the synthetic index (summary-only titles, missing chapter); [ ] job test (2 sections, 3 clips, restricted manifest, source filled, course loads `ok`, cover candidates); [ ] implement; [ ] commit.
### Task 4: `import.video-folder`, `import.pdf-deck`
- [ ] Tests: natural order, deck cache pages/text, attach to lesson and module, refuses unmanaged course; [ ] implement; [ ] commit.
### Task 5: `import.code-repo`
- [ ] Tests: folder `S02 - …` attaches to module 2 by number, by name when no number, PDFs become deck/attachment, git URL validation; [ ] implement; [ ] commit.
### Task 6: `import.transcribe`
- [ ] Tests with a fake Whisper server; skipped when `STC_TRANSCRIBE_URL` unset; [ ] implement; [ ] commit.
### Task 7: Routes and `importFeature`
- [ ] Tests: 401/403/owner, enqueue validation, queue integration (SQLite and Postgres), covers; [ ] implement; wire `main.ts`; [ ] commit.
### Task 8: Web UI `importWeb`
- [ ] Page with form, job list with progress, cover picker; register in `apps/web/src/features.ts`; screenshot check; [ ] commit.
### Task 9: Manual check against `/home/user/example_course_content`, report.
docs/superpowers/plans/2026-10-30-learner.md
+147
View File
@@ -0,0 +1,147 @@
# WP-A Learner App Implementation Plan
> **For agentic workers:** executed inline by the WP-A agent (no subagents), task by task, TDD, one commit per task. Steps use checkbox (`- [ ]`) syntax.
**Goal:** Build `packages/feature-learner`: sign-in (Better Auth), roles/groups admin API, the library, the lesson reader for every lesson type, notes, progress, gating, and the in-browser JS/TS exercise runner. It must render `sample`, `pure-snake` and `systemic-design`.
**Architecture:**
- Server side (`src/server/*`): `learnerFeature(ctx): Feature & { auth: AuthService }` returns the JSON API routes and the `AuthService`. It runs its own migrations (`migrations/0001_learner_auth.sql`) lazily on first use until Phase 2 wires `migrationsDirs` into `migrate()`.
- Auth: Better Auth with a **thin custom adapter over `Bun.SQL`** (`src/server/auth/adapter.ts`). One adapter serves both dialects because `@stc/db` already gives a `Bun.SQL` handle for SQLite and Postgres.
- Web side (`src/web/*`): React routes `/`, `/c/:courseId`, `/c/:courseId/:lessonId`, `/c/:courseId/notes`, `/account`, `/account/admin`. Styles are plain CSS loaded as text and injected once, so the same module graph is safe to import on the server.
- `LessonView` is a presentational lesson renderer that takes a `LessonPayload`. The studio uses it for preview.
- The browser runner (`src/web/runner/*`) runs learner code and tests as ES modules from blob URLs inside a module Web Worker, with a hard timeout (the worker is terminated). TS is transpiled by the server's `Bun.Transpiler` (`POST /api/learner/transpile`).
- Decks render client-side with pdfjs-dist. The worker script is served from `/api/learner/pdf.worker.mjs`.
**Tech Stack:** Bun 1.3, TypeScript, React 19, React Router 6, better-auth 1.7 (+ `@better-auth/sso` for SAML/OIDC SSO, `generic-oauth` for a configured OIDC provider), marked + DOMPurify, pdfjs-dist.
**Spec:** sections 4.1–4.4, 6, 7, 12, 13. Visual reference: `docs/design/mockups/platform-v1.html` minus its `[` / `]` / `/` / Ctrl+Enter shortcuts.
## Global Constraints
Inherited from `docs/superpowers/plans/2026-10-30-stcourses.md` (Bun only, plain CSS, no course content in the DB, both dialects, AGPL, tokens, keyboard rules, commit trailer). Package-specific:
- Owned paths: `packages/feature-learner/**`, `docs/superpowers/plans/2026-10-30-learner.md`, `docs/contract-requests/learner.md`. Shared touches: one line in `apps/web/src/features.ts`, the feature wiring in `apps/server/src/main.ts`.
- Frozen: `packages/contracts`, `packages/course-format`, `packages/db` (src + 0001), `packages/core`, `packages/ui`.
- No new colours: course accent (manifest) and the tokens only. Literata only for lesson titles and reading text.
## Review Focus
1. **Auth adapter correctness on both dialects** (where operators, AND/OR grouping, dates and booleans as text/int). Test: Task 2 runs the adapter suite under `forEachDb`.
2. **Owner bootstrap is race-safe**: exactly one owner even with concurrent sign-ups (unique key claim). Test: Task 3.
3. **Visibility**: lesson, outline, code and zip routes all apply `canView` and return 404 for hidden courses; gating returns `locked` without content. Test: Task 5/7.
4. **Zip download** handles spaces and non-ASCII names (UTF-8 flag) and never follows symlinks out of the root. Test: Task 7.
5. **Runner isolation**: learner code runs only inside the worker; an infinite loop is killed by the timeout. Test: Task 8.
---
### Task 1: Package scaffold, migrations runner, test helper
**Files:** `packages/feature-learner/{package.json,src/index.ts,src/server/migrate.ts,migrations/0001_learner_auth.sql,test/helpers.ts,test/migrate.test.ts}`
**Interfaces (Produces):**
- `migrationsDirs: string[]` (absolute path of `packages/feature-learner/migrations`).
- `runMigrations(db: Db, dirs?: string[]): Promise<string[]>` — same bookkeeping table and semantics as `@stc/db` `migrate()`.
- Tables: `auth_user`, `auth_session`, `auth_account`, `auth_verification`, `auth_sso_provider` (Better Auth, snake_case columns, ISO text dates, integer booleans), `learner_meta (key PK, value)`, `learner_group_claims (user_id, group_id)`.
- test helper `forEachDb(name, fn)` = `@stc/db`'s helper plus the learner migrations.
- [x] Step 1: failing test: after `runMigrations` the tables exist on both dialects and a second run applies nothing.
- [x] Step 2: implement; Step 3: green on SQLite and Postgres; commit.
### Task 2: Better Auth adapter over Bun.SQL
**Files:** `src/server/auth/adapter.ts`, `test/adapter.test.ts`
**Interfaces:** `bunSqlAdapter(db: Db)` → Better Auth `DBAdapterInstance`. Model `x` → table `auth_<snake(x)>`, field `camelCase` → `snake_case`. `supportsDates/Booleans/JSON: false`, no transactions (SQLite has one connection; a nested query in a transaction would deadlock).
- [x] Failing tests (both dialects): create/findOne/findMany (eq, ne, in, not_in, contains, starts_with, ends_with, lt/gt, insensitive mode, OR grouping, sort, limit, offset), update, updateMany, delete, deleteMany, count, consumeOne, incrementOne.
- [x] Implement, green, commit.
### Task 3: AuthService, owner bootstrap, claim groups, feature wiring
**Files:** `src/server/auth/index.ts`, `src/server/auth/groups.ts`, `src/server/feature.ts`, `test/auth.test.ts`, `apps/server/src/main.ts` (wiring), `docs/contract-requests/learner.md`
**Interfaces:**
- `createAuth(ctx, opts?)` → `{ handler(req), service: AuthService, providers(): ProviderInfo[] }`.
- Email/password always on; social providers (GitHub, Google, Microsoft, GitLab, Discord) from `STC_AUTH_<NAME>_ID/SECRET`; one generic OIDC provider from `STC_AUTH_OIDC_*`; SAML and per-domain OIDC through `@better-auth/sso` (owner-only registration).
- First user to sign up becomes `owner` (claimed via `INSERT` into `learner_meta` key `owner`, PK conflict loses); others get `learner`.
- `syncClaimGroups(db, userId, names)` creates groups by name and replaces the user's claim-sourced memberships; manual memberships are untouched.
- `learnerFeature(ctx)` returns `Feature & { auth }`; `startServer` accepts feature factories `(ctx) => Feature` and installs a feature's `auth` (contract request filed; two-line change in `main.ts`).
- [x] Failing tests: sign-up → session cookie → `userFromRequest` returns roles `[owner]`; second user `[learner]`; concurrent first sign-ups give one owner; sign-out clears; claim sync keeps manual memberships.
- [x] Implement, green, commit.
### Task 4: Admin API (roles, groups, course ACL, library curation)
**Files:** `src/server/admin.ts`, `test/admin.test.ts`
- Routes (owner only, 403 otherwise): `GET /api/admin/users`, `PUT /api/admin/users/:id/roles`, `GET|POST /api/admin/groups`, `DELETE /api/admin/groups/:id`, `PUT /api/admin/groups/:id/members`, `GET|PUT /api/admin/courses/:id/acl`, `GET|PUT /api/admin/library` (featured + collections).
- The last owner cannot drop their own owner role.
- [x] Tests first; implement; commit.
### Task 5: Library, outline and lesson API
**Files:** `src/server/catalog.ts`, `src/server/lesson.ts`, `src/server/gating.ts`, `test/catalog.test.ts`, `test/lesson.test.ts`
- `GET /api/library` → `{ featured, continue[], newest[], collections[], courses[] }` filtered by `canView`. Each course: id, title, pitch, authors, coverUrl, accent, formats (`text|video|slides|exercises`), lessonCount, minutes (video duration, text estimated at 220 wpm, exercises 15 min), collections, progress, status.
- `GET /api/library/search?q=` → courses and lessons matching title/pitch/lesson titles.
- `GET /api/courses/:id` → outline, tracks, gating, `locked` per module, progress, diagnostics (errors only for owner/authors).
- `GET /api/courses/:id/lessons/:lessonId` → `LessonPayload` (markdown, hints, decks, video, transcript, code, attachments, browser exercise, local command, prev/next) or `{ locked: true }`.
- Gating: module *i* locked when any lesson of an earlier module is not done; owners and course authors bypass.
- [x] Tests against the sample fixture plus temp courses; implement; commit.
### Task 6: Progress, notes, exercise attempts API
**Files:** `src/server/userdata.ts`, `test/userdata.test.ts`
- `PUT /api/courses/:id/progress/:lessonId` `{state, positionSeconds?}`; notes CRUD (`GET/POST /api/courses/:id/notes`, `PATCH/DELETE /api/notes/:id`); attempts `GET/PUT /api/courses/:id/lessons/:lessonId/attempt?track=`. A passing attempt marks the lesson done. Anonymous → 401. Hidden course → 404.
- [x] Tests on both dialects (acceptance: notes persist on SQLite and Postgres); implement; commit.
### Task 7: Code files and zip download
**Files:** `src/server/zip.ts`, `src/server/code.ts`, `test/zip.test.ts`, `test/code.test.ts`
- `GET /api/courses/:id/code/:moduleId` → `{ dir, command, files: {path,size}[] }` (skips `.git`, `node_modules`, symlinks escaping root; caps 2000 files).
- `GET /api/courses/:id/code/:moduleId/zip` → `application/zip`, `Content-Disposition` via `contentDisposition("<dir name>.zip")`.
- `zip(entries)`: store/deflate, CRC32, UTF-8 name flag.
- [x] Tests: zip round-trip via `unzip -l`, non-ASCII names, symlink escape skipped; commit.
### Task 8: Browser exercise runner
**Files:** `src/web/runner/{assert.ts,worker-source.ts,runner.ts}`, `src/server/transpile.ts`, `test/runner.test.ts`
- `createBrowserRunner(language, { transpile })`: `ExerciseRunner`. Worker disables `fetch`, `XMLHttpRequest`, `WebSocket`, `importScripts` before importing code. Results carry expected/actual for failed asserts.
- [x] Tests (Bun supports module workers and blob URLs): starter fails, solution passes, a syntax error is reported, `while(true)` times out, TS is transpiled.
- [x] commit.
### Task 9: Markdown pipeline
**Files:** `src/web/markdown.ts`, `test/markdown.test.ts`
- `renderLessonHtml(md, { courseId, baseFile, tracks, lessonFiles })` → sanitized HTML: heading ids, code blocks tagged with their track (`data-t`), bold track labels tagged, `<details>` turned into folds; consecutive hint folds become a tier group (`data-tier`, `data-tiers`), relative images/links rewritten to `/api/media/...` or lesson routes.
- [x] Tests with Pure Snake-style input; commit.
### Task 10: Web shell and library
**Files:** `src/web/{routes.tsx,api.ts,styles.css,shell/*,library/*}`
- Header extras (course switcher, search, Reading/Work, account) portalled into the Phase 0 header; course outline sidebar (mobile overlay, Escape closes); library: featured hero, continue row, New row, collection rows of covers, format/length/in-progress filters, search.
- [x] Component smoke tests with `react-dom/server` render; screenshot check later; commit.
### Task 11: Lesson views and `LessonView`
**Files:** `src/web/lesson/*`
- Text (track switcher, Reading/Work, folds, hint tiers), video (range-served `<video>`, resume, done near end, timestamp notes), deck viewer (pdfjs, thumbs, arrows, fullscreen, per-slide note), code files (tree, tabs, copy, zip), exercise page (editor, run, reset, per-test results, hints, local command; local-only: command + "Mark done" + optional pasted evidence), notes list, floating action bar.
- [x] Render tests for `LessonView` with each lesson type; commit.
### Task 12: Account and admin pages
**Files:** `src/web/account/*`
- Sign in / create account (email+password), configured provider buttons, SSO by email, sign out; owner admin page for users/roles, groups, featured course and collections.
- [x] Render tests; commit.
### Task 13: Wiring, verification, report
- [x] Append `learnerWeb` to `apps/web/src/features.ts`; `learnerFeature` in `main.ts`.
- [x] `bun test`, `bun run typecheck`, Postgres run; dev server on 3461 with `STC_COURSE_DIRS=./fixtures/courses:./overlays`; headless Chrome screenshots of library, text, video, deck, code, exercise, account for all three courses; fix what looks wrong.
- [x] Write `/home/user/STCourses/.superpowers/sdd/2026-10-30-stcourses/wp-learner-report.md`.
---
## Execution notes (added when the plan was finished)
- Tasks 1–9 were done by the first implementer (0b11715..99c0e49). Tasks 10–13 were done after a hand-over (89c51f0 onwards).
- Tasks 10–12 landed in one commit, because the route modules import each other (the routes list loads the lesson, library and account pages).
- Styles are a TS string (`src/web/styles.ts`) injected once, not a `.css` text import: Bun's dev server loses text imports on hot rebuilds.
- The browser entry is `@stc/feature-learner/web` (see `docs/contract-requests/learner.md` §3); the index cannot be bundled for the browser.
- Decks render client-side with pdf.js; the importer's `<dataDir>/cache/decks` is not used.
- The exercise editor is a textarea with a line gutter, as in the mockup (CodeMirror was optional).
docs/superpowers/plans/2026-10-30-studio.md
+77
View File
@@ -0,0 +1,77 @@
# STCourses WP-C Studio Implementation Plan
> **For agentic workers:** executed inline by the WP-C agent (superpowers:executing-plans). Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Browser studio for studio-managed courses: create a course, edit the manifest, lessons (Markdown with live preview), deck links, attachments and exercises in a per-author draft worktree; every save is a commit by the author; publish merges the draft into the published branch that learners read; Sync with a remote only on request; conflicts are shown and resolved by the author, never automatically.
**Architecture:**
- `packages/feature-studio/src/server/git.ts`: a thin `git` CLI wrapper over `Bun.$` (explicit identity, no hooks, no signing) plus a per-course async mutex.
- `packages/feature-studio/src/server/store.ts`: `StudioStore`, the git model of spec section 12:
- `<dataDir>/courses/<id>`: a **bare** repo (it holds no `stcourse.yaml`, so course discovery never loads it);
- `<dataDir>/worktrees/<id>/published`: worktree on the published branch (`main`); the only tree learners read;
- `<dataDir>/worktrees/<id>/drafts/<author>`: worktree on `draft/<author>` (depth 3 under `worktrees/`, so discovery, which stops at depth 2, never loads drafts).
- Publish = `git merge --no-ff draft/<author>` inside the published worktree. A conflicting merge is aborted at once, so the published tree is never left half-merged, and the conflicting paths are returned. The author then runs "Update draft" (merge of `main`, or `origin/main` for Sync, into the draft worktree), resolves each file in the studio, and publishes again.
- Save uses optimistic concurrency: each file carries the hash of the content the editor loaded; a mismatch with the file on disk (an external edit) gives 409 with both versions. Nothing is overwritten.
- `watch.ts`: polls the published HEAD of each managed course (default every 3 s, unref'd timer) and calls `ctx.courses.reload(id)` when it changed (external commit, pull, publish).
- `feature.ts`: `studioFeature(ctx)` with `/api/studio/*` routes. Access: owner, `author` role, or a `course_authors` row for that course. Courses not managed by the studio (overlays, fixture/imported folders) are listed read-only: "This course is managed outside STCourses".
- `draft-writer.ts`: `createDraftWriter(ctx)` for WP-D.
- Web (`src/web/*`): `/studio` (course list, New course) and `/studio/:courseId` (outline sidebar, editors, status bar with Publish / Update draft / Sync and the conflict view). Work look from the mockup. The lesson preview sits behind a `LessonPreview` adapter; v1 is a small safe Markdown renderer; Phase 2 swaps in the learner `LessonView` (contract request).
**Tech Stack:** Bun 1.3, TypeScript strict, git CLI 2.4x via `Bun.$`, React 19 + React Router 6, `@stc/course-format` for validation, `@stc/ui` primitives.
**Spec:** sections 5, 9.1, 12 (git model), 13. Brief: `.superpowers/sdd/2026-10-30-stcourses/phase-1-brief.md` (WP-C).
## Global Constraints
Inherits every item of the Global Constraints in `docs/superpowers/plans/2026-10-30-stcourses.md`. Additionally:
- Never write into a course whose root is outside `<dataDir>` (overlays) — the studio refuses writes for non-managed courses.
- Paths written through the studio must be relative, stay inside the worktree, and never touch `.git`.
- No new DB tables (authors use the existing `course_authors`); `migrationsDirs` is exported empty.
## Review Focus
1. A conflicting external commit to `main` is detected at publish and shown; the published tree keeps the external content (test: acceptance).
2. Drafts never reach learners: before publish the `CourseRepository` still serves the published content (test: acceptance).
3. An external edit to a draft file between load and save gives 409, not an overwrite (test: store).
4. Paths with spaces and non-ASCII characters save, commit and publish (test: store).
5. Path traversal (`..`, absolute, `.git/…`) is rejected (test: store).
## Tasks
### Task 1: git wrapper and StudioStore (create, drafts, save)
**Files:** Create `packages/feature-studio/src/server/{git.ts,store.ts,paths.ts}`; Test `packages/feature-studio/test/store.test.ts`.
**Interfaces (Produces):** `createStudioStore(dataDir): StudioStore` with `createCourse`, `isManaged`, `listManaged`, `ensureDraft`, `readFile`, `listFiles`, `saveFiles`, `status`.
- [ ] Step 1: failing tests: create gives bare repo + published worktree with a valid manifest; draft worktree on `draft/<author>`; save commits with the author's name/email; stale base hash gives a conflict; traversal rejected; non-ASCII path works.
- [ ] Step 2: run, FAIL. Step 3: implement. Step 4: PASS. Step 5: commit.
### Task 2: publish, update draft, conflict resolution, sync
**Files:** extend `store.ts`; Test `packages/feature-studio/test/publish.test.ts`.
- [ ] Step 1: failing tests: publish merges; external conflicting commit gives `{ok:false, conflicts}` and leaves `main` untouched; dirty published tree refuses publish; update draft shows ours/theirs; resolve + publish succeeds; abort; Sync with a local bare remote (pull fast-forward, push, conflict reported); no remote gives an error.
- [ ] Steps 2–5 as above.
### Task 3: watcher and draft writer
**Files:** `src/server/{watch.ts,draft-writer.ts}`; Test `test/watch.test.ts`, `test/draft-writer.test.ts`.
- [ ] Failing tests: external commit triggers `courses.reload`; `writeDraftFiles` commits on `draft/<author>` and the published tree is unchanged. Implement, pass, commit.
### Task 4: HTTP feature and acceptance
**Files:** `src/server/feature.ts`, `src/index.ts`; Test `test/feature.test.ts`, `test/acceptance.test.ts`.
- [ ] Failing tests: 401/403 for anonymous and learners; overlay course read-only; create → edit lesson → publish → learner sees it via `CourseRepository` and `/api/media`; conflicting external commit shown, not overwritten; manifest validation errors returned. Implement, pass, commit.
### Task 5: Studio web UI
**Files:** `src/web/{api.ts,markdown.ts,preview.tsx,StudioHome.tsx,CourseStudio.tsx,editors/*.tsx,studio.css}`, `src/web.ts`; Test `test/markdown.test.ts`.
- [ ] Failing test for the Markdown renderer (escaping, headings, fences, lists, links without `javascript:`). Implement UI. Commit.
### Task 6: wiring, dev server, contract requests, verification
- [ ] Append to `apps/web/src/features.ts`; dev entry `packages/feature-studio/dev/serve.ts`; `docs/contract-requests/studio.md`; run `bun test`, typecheck, Postgres run; headless Chrome screenshots of `/studio` and `/studio/:id`; report.
packages/contracts/src/index.ts
+1
View File
@@ -3,3 +3,4 @@ export * from "./exercise";
export * from "./ai";
export * from "./import";
export * from "./web";
export * from "./routes";
packages/contracts/src/routes.ts
+14
View File
@@ -0,0 +1,14 @@
// The learner's public page routes. The learner package registers these patterns; every other package (AI
// citations, studio links) builds links with the helpers, so a link always matches a real route.
// Pure functions with no dependencies, the only runtime code in @stc/contracts.
export const COURSE_ROUTE = "/c/:courseId";
export const LESSON_ROUTE = "/c/:courseId/:lessonId";
export function courseUrl(courseId: string): string {
return `/c/${encodeURIComponent(courseId)}`;
}
export function lessonUrl(courseId: string, lessonId: string): string {
return `${courseUrl(courseId)}/${encodeURIComponent(lessonId)}`;
}
packages/contracts/src/server.ts
+38 -2
View File
@@ -7,7 +7,19 @@ export interface Feature {
name: string; // "learner" | "import" | "studio" | "ai" | "core"
routes: Record<string, Partial<Record<"GET"|"POST"|"PUT"|"PATCH"|"DELETE", RouteHandler>>>; // paths start with /api/
jobs?: Record<string, JobHandler>; // job kind -> handler
auth?: AuthService; // replaces the anonymous AuthService (the learner package supplies it)
stop?(): void | Promise<void>; // called when the server stops (timers, watchers)
/** Route keys (as in `routes`) whose writes take multipart/form-data instead of JSON (uploads). */
multipartRoutes?: string[];
/**
* Path prefixes an outside identity provider posts to (SAML ACS/SLO, OAuth callbacks). Core's cross-site request
* guard skips them; the handler must verify the request itself (signed assertion, OAuth state).
*/
externalCallbacks?: string[];
}
/** Features are built after the ServerContext exists. The job queue is built from the union of their `jobs`, so a
* factory must not use `ctx.jobs` synchronously (only while a request or job runs). */
export type FeatureFactory = (ctx: ServerContext) => Feature;
export interface JobContext { reportProgress(fraction: number, message?: string): Promise<void>; signal: AbortSignal }
export type JobHandler = (payload: unknown, jc: JobContext) => Promise<unknown>;
export type JobStatus = "queued" | "running" | "done" | "failed";
@@ -20,8 +32,32 @@ export interface ServerContext {
jobs: JobQueue; auth: AuthService;
}
export interface AppConfig {
port: number; databaseUrl: string; dataDir: string; // dataDir holds cache/, media/, worktrees/
courseDirs: string[]; // folders scanned for stcourse.yaml (course repos and overlays/)
port: number; databaseUrl: string; dataDir: string; // dataDir holds cache/, courses/, studio/, worktrees/
/** Folders scanned for stcourse.yaml: STC_COURSE_DIRS, plus importedCoursesDir and studioWorktreesDir (deduplicated). */
courseDirs: string[];
secret: string; // STC_SECRET, used to encrypt BYOK keys
publicUrl: string;
/** `<dataDir>/courses`: the importer's plain course folders. */
importedCoursesDir: string;
/** `<dataDir>/studio/repos`: studio bare repos (`<id>.git`), never scanned. */
studioReposDir: string;
/** `<dataDir>/worktrees`: studio published worktrees (`<id>/published`); drafts sit one level deeper and are never scanned. */
studioWorktreesDir: string;
/** `<dataDir>/cache`: derived files (decks/, covers/, transcripts/). */
cacheDir: string;
/** STC_IMPORT_ROOTS (":"-separated), default `<dataDir>/import-inbox`: the only folders the importer reads from. */
importRoots: string[];
/** STC_UPLOAD_MAX_MB, default 100. */
uploadMaxMb: number;
/** STC_AI_ALLOWED_BASE_URLS (comma-separated origins) that learners' own AI keys may use besides the official endpoints. */
aiAllowedBaseUrls: string[];
/** STC_TRANSCRIBE_URL: Whisper-compatible endpoint; null disables transcription. */
transcribeUrl: string | null;
/** STC_PYTHON, default python3: Python with PyMuPDF, used for PDF page images and text. */
python: string;
/**
* Origins allowed to send state-changing requests: the origin of `publicUrl` plus STC_ALLOWED_ORIGINS
* (comma-separated; STC_TRUSTED_ORIGINS is read as an alias). Core refuses writes from any other origin.
*/
allowedOrigins: string[];
}
packages/contracts/src/web.ts
+3 -1
View File
@@ -1,4 +1,6 @@
import type React from "react";
import type { Role } from "./server";
export interface WebRoute { path: string; element: () => Promise<{ default: React.ComponentType }>; nav?: { label: string; area: "header" | "admin" } }
/** `nav.roles`: the shell shows the link only to signed-in users holding one of these roles (absent: everyone). */
export interface WebRoute { path: string; element: () => Promise<{ default: React.ComponentType }>; nav?: { label: string; area: "header" | "admin"; roles?: Role[] } }
export interface WebFeature { name: string; routes: WebRoute[] }
packages/contracts/test/types.test.ts
+3
View File
@@ -25,6 +25,9 @@ test("contract interfaces accept well-formed literals", async () => {
const config: AppConfig = {
port: 3000, databaseUrl: "sqlite://./data/stcourses.db", dataDir: "./data", courseDirs: ["./overlays"],
secret: "s", publicUrl: "http://localhost:3000",
importedCoursesDir: "./data/courses", studioReposDir: "./data/studio/repos", studioWorktreesDir: "./data/worktrees",
cacheDir: "./data/cache", importRoots: ["./data/import-inbox"], uploadMaxMb: 100, aiAllowedBaseUrls: [],
transcribeUrl: null, python: "python3", allowedOrigins: ["http://localhost:3000"],
};
const ctx: Pick<ServerContext, "config" | "jobs" | "auth"> = { config, jobs: queue, auth };
packages/core/src/app.ts
+55 -3
View File
@@ -6,17 +6,63 @@ import { serveMedia } from "./media";
type Method = "GET" | "POST" | "PUT" | "PATCH" | "DELETE";
export type Routes = Record<string, Partial<Record<Method, (req: Request) => Response | Promise<Response>>>>;
/**
* The one visibility rule (ruling R14). Every package calls this; none keeps a local variant.
* public: everyone; signed-in: any signed-in user; restricted: the owner, the global `author` role, a per-course
* author (`isAuthor`, a course_authors row), or an ACL grant to the user or one of their groups.
*/
export function canView(user: AuthUser | null, course: LoadedCourse, acl: { principal_type: string; principal_id: string }[], isAuthor = false): boolean {
const vis = course.manifest?.visibility ?? "restricted";
if (vis === "public") return true;
if (!user) return false;
if (vis === "signed-in") return true;
if (user.roles.includes("owner") || isAuthor) return true;
if (user.roles.includes("owner") || user.roles.includes("author") || isAuthor) return true;
return acl.some((a) => (a.principal_type === "user" && a.principal_id === user.id) || (a.principal_type === "group" && user.groups.includes(a.principal_id)));
}
const json = (body: unknown, status = 200) => Response.json(body, { status });
const WRITES = new Set(["POST", "PUT", "PATCH", "DELETE"]);
/** The request's origin from `Origin`, else from `Referer`; null when neither names one ("null" counts as none). */
function requestOrigin(req: Request): string | null {
const o = req.headers.get("origin");
if (o && o !== "null") return o.toLowerCase();
const r = req.headers.get("referer");
if (r) { try { const u = new URL(r).origin; return u === "null" ? null : u; } catch { return null; } }
return null;
}
const hasBody = (req: Request) => {
const len = req.headers.get("content-length");
if (len !== null) return Number(len) > 0;
return req.headers.has("transfer-encoding") || req.body !== null;
};
/**
* Cross-site request guard for every state-changing request (POST, PUT, PATCH, DELETE), applied once here for all
* features, Better Auth included:
* - the `Origin` (or, without it, the `Referer`) must be one of `config.allowedOrigins` (publicUrl's origin plus
* STC_ALLOWED_ORIGINS); missing or foreign gives 403;
* - a request with a body must be `application/json`, or `multipart/form-data` on the feature's `multipartRoutes`.
* Forms and text/plain bodies (what a hostile page can send without a preflight) are refused with 403.
* Paths in a feature's `externalCallbacks` (identity-provider posts) are exempt and verify themselves.
* Returns the refusal, or null to proceed.
*/
export function crossSiteRefusal(req: Request, allowedOrigins: string[], opts: { multipart: boolean; external: boolean }): Response | null {
if (!WRITES.has(req.method) || opts.external) return null;
const origin = requestOrigin(req);
if (!origin || !allowedOrigins.includes(origin)) {
return json({ error: "cross-site request refused: the Origin must be this server's public URL (STC_PUBLIC_URL or STC_ALLOWED_ORIGINS)" }, 403);
}
if (hasBody(req)) {
const type = (req.headers.get("content-type") ?? "").split(";")[0]!.trim().toLowerCase();
const ok = type === "application/json" || (opts.multipart && type === "multipart/form-data");
if (!ok) return json({ error: opts.multipart ? "send multipart/form-data or application/json" : "send application/json" }, 403);
}
return null;
}
export function createApp(ctx: ServerContext, features: Feature[]): { routes: Routes; fetch(req: Request): Promise<Response> } {
const acl = aclRepo(ctx.db);
@@ -43,19 +89,25 @@ export function createApp(ctx: ServerContext, features: Feature[]): { routes: Ro
},
};
const wrap = (h: RouteHandler) => async (req: Request & { params?: Record<string, string> }): Promise<Response> => {
// No configuration means no allowed origin: every write is refused (fail closed).
const allowed = ctx.config?.allowedOrigins ?? [];
const external = features.flatMap((f) => f.externalCallbacks ?? []);
const wrap = (h: RouteHandler, multipart: boolean) => async (req: Request & { params?: Record<string, string> }): Promise<Response> => {
const url = new URL(req.url);
const refused = crossSiteRefusal(req, allowed, { multipart, external: external.some((p) => url.pathname.startsWith(p)) });
if (refused) return refused;
const rc: RequestContext = { user: await ctx.auth.userFromRequest(req), params: req.params ?? {}, url };
return h(req, rc);
};
const routes: Routes = {};
for (const f of [core, ...features]) {
const multipart = new Set(f.multipartRoutes ?? []);
for (const [path, methods] of Object.entries(f.routes)) {
const slot = (routes[path] ??= {});
for (const [m, h] of Object.entries(methods) as [Method, RouteHandler][]) {
if (slot[m]) throw new Error(`Duplicate route ${m} ${path} (feature "${f.name}")`);
slot[m] = wrap(h);
slot[m] = wrap(h, multipart.has(path));
}
}
}
packages/core/src/compose.ts
+75
View File
@@ -0,0 +1,75 @@
import type { AppConfig, AuthService, Feature, FeatureFactory, JobHandler, JobQueue, ServerContext } from "@stc/contracts";
import { createCourseRepository } from "@stc/course-format";
import { connect, migrate, type PackageMigrations } from "@stc/db";
import { createApp } from "./app";
import { createJobQueue } from "./jobs";
/** Used until a feature supplies an `auth` service (the learner package does). */
const anonymousAuth: AuthService = { userFromRequest: async () => null };
export interface ComposeOptions {
/** Each package's migration folders, keyed by package name; run after core's, in name order. */
migrations?: PackageMigrations;
/** Start polling the job queue (default true). */
startJobs?: boolean;
}
export interface ComposedServer {
ctx: ServerContext;
features: Feature[];
app: ReturnType<typeof createApp>;
stop(): Promise<void>;
}
/**
* Builds the server context and the features (rulings R8/R14). Order:
* 1. connect and migrate (core, then packages);
* 2. load courses from `config.courseDirs`;
* 3. run each factory with the context. `ctx.jobs` already exists as a handle, but it forwards to the real queue,
* which is built in step 4 from the union of the features' `jobs`, so factories may keep it but must not
* enqueue while they run;
* 4. a feature carrying `auth` replaces the anonymous AuthService (at most one may).
*/
export async function composeServer(config: AppConfig, entries: (Feature | FeatureFactory)[], opts: ComposeOptions = {}): Promise<ComposedServer> {
const db = await connect(config.databaseUrl);
try {
await migrate(db, opts.migrations ?? {});
const courses = await createCourseRepository(config.courseDirs);
let queue: (JobQueue & { start(): void; stop(): Promise<void> }) | null = null;
const notYet = (): never => { throw new Error("the job queue is not ready while features are being built"); };
const jobs: JobQueue = {
enqueue: (kind, payload) => (queue ?? notYet()).enqueue(kind, payload),
get: (id) => (queue ?? notYet()).get(id),
list: (kind) => (queue ?? notYet()).list(kind),
};
const ctx: ServerContext = { config, db, courses, jobs, auth: anonymousAuth };
const features = entries.map((f) => (typeof f === "function" ? f(ctx) : f));
const handlers: Record<string, JobHandler> = {};
let authFrom: string | null = null;
for (const f of features) {
for (const [kind, h] of Object.entries(f.jobs ?? {})) {
if (handlers[kind]) throw new Error(`Duplicate job kind "${kind}" (feature "${f.name}")`);
handlers[kind] = h;
}
if (f.auth) {
if (authFrom) throw new Error(`Both "${authFrom}" and "${f.name}" supply an AuthService`);
authFrom = f.name;
ctx.auth = f.auth;
}
}
queue = createJobQueue(db, handlers);
const app = createApp(ctx, features);
if (opts.startJobs !== false) queue.start();
return {
ctx, features, app,
async stop() {
for (const f of features) { try { await f.stop?.(); } catch (e) { console.error(`[stc] stopping ${f.name}`, e); } }
await queue!.stop();
await db.close();
},
};
} catch (e) {
await db.close().catch(() => {});
throw e;
}
}
packages/core/src/config.ts
+89 -3
View File
@@ -1,7 +1,64 @@
import { resolve } from "node:path";
import type { AppConfig } from "@stc/contracts";
const DEV_SECRET = "stc-dev-secret-change-me";
/** The settings a deployment chooses; everything else in AppConfig is derived by `resolveConfig`. */
export type ConfigInput = Pick<AppConfig, "dataDir"> & Partial<Omit<AppConfig, "dataDir">>;
// Path lists are split on ":" only (paths may contain spaces); URL lists are trimmed.
const paths = (raw: string | undefined) => (raw ?? "").split(":").filter(Boolean);
/** `scheme://host[:port]` of a URL, lower-cased by URL parsing; null when it does not parse. */
export function originOf(u: string): string | null {
try {
const o = new URL(u).origin;
return o === "null" ? null : o;
} catch {
return null;
}
}
const urls = (raw: string | undefined) => (raw ?? "").split(",").map((s) => s.trim()).filter(Boolean);
/**
* Fills in the derived parts of the configuration. Course discovery always covers the importer's folders and the
* studio's published worktrees, after the configured dirs, without duplicates (compared as absolute paths).
* Tests and dev entries use this to build an AppConfig the same way `loadConfig` does.
*/
export function resolveConfig(input: ConfigInput): AppConfig {
const dataDir = input.dataDir;
const importedCoursesDir = input.importedCoursesDir ?? resolve(dataDir, "courses");
const studioWorktreesDir = input.studioWorktreesDir ?? resolve(dataDir, "worktrees");
const courseDirs: string[] = [];
for (const d of [...(input.courseDirs ?? []), importedCoursesDir, studioWorktreesDir]) {
if (!courseDirs.some((c) => resolve(c) === resolve(d))) courseDirs.push(d);
}
const port = input.port ?? 3000;
const publicUrl = input.publicUrl ?? `http://localhost:${port}`;
const origins: string[] = [];
for (const o of [publicUrl, ...(input.allowedOrigins ?? [])]) {
const n = originOf(o);
if (n && !origins.includes(n)) origins.push(n);
}
return {
port,
databaseUrl: input.databaseUrl ?? "sqlite://./data/stcourses.db",
dataDir,
courseDirs,
secret: input.secret ?? DEV_SECRET,
publicUrl,
allowedOrigins: origins,
importedCoursesDir,
studioReposDir: input.studioReposDir ?? resolve(dataDir, "studio", "repos"),
studioWorktreesDir,
cacheDir: input.cacheDir ?? resolve(dataDir, "cache"),
importRoots: input.importRoots ?? [resolve(dataDir, "import-inbox")],
uploadMaxMb: input.uploadMaxMb ?? 100,
aiAllowedBaseUrls: input.aiAllowedBaseUrls ?? [],
transcribeUrl: input.transcribeUrl ?? null,
python: input.python ?? "python3",
};
}
export function loadConfig(env: Record<string, string | undefined> = process.env): AppConfig {
const port = env.STC_PORT !== undefined && env.STC_PORT !== "" ? Number(env.STC_PORT) : 3000;
if (!Number.isInteger(port) || port < 0) throw new Error(`Invalid STC_PORT: ${env.STC_PORT}`);
@@ -11,12 +68,41 @@ export function loadConfig(env: Record<string, string | undefined> = process.env
console.warn("[stc] STC_SECRET is not set; using an insecure development default");
secret = DEV_SECRET;
}
return {
let uploadMaxMb = 100;
if (env.STC_UPLOAD_MAX_MB) {
uploadMaxMb = Number(env.STC_UPLOAD_MAX_MB);
if (!Number.isFinite(uploadMaxMb) || uploadMaxMb <= 0) throw new Error(`Invalid STC_UPLOAD_MAX_MB: ${env.STC_UPLOAD_MAX_MB}`);
}
const dataDir = env.STC_DATA_DIR || "./data";
const roots = paths(env.STC_IMPORT_ROOTS);
return resolveConfig({
port,
databaseUrl: env.STC_DATABASE_URL || "sqlite://./data/stcourses.db",
dataDir: env.STC_DATA_DIR || "./data",
courseDirs: (env.STC_COURSE_DIRS || "./overlays:./courses").split(":").filter(Boolean),
dataDir,
courseDirs: paths(env.STC_COURSE_DIRS || "./overlays:./courses"),
secret,
publicUrl: env.STC_PUBLIC_URL || `http://localhost:${port}`,
importRoots: roots.length ? roots.map((r) => resolve(r)) : undefined,
uploadMaxMb,
aiAllowedBaseUrls: urls(env.STC_AI_ALLOWED_BASE_URLS),
transcribeUrl: env.STC_TRANSCRIBE_URL || null,
python: env.STC_PYTHON || "python3",
allowedOrigins: [...urls(env.STC_ALLOWED_ORIGINS), ...urls(env.STC_TRUSTED_ORIGINS)],
});
}
/** The importer's derived-file layout under `cacheDir` (decks, cover candidates, transcripts), shared by every package. */
export function cachePaths(config: Pick<AppConfig, "cacheDir">) {
const root = config.cacheDir;
const seg = (s: string) => {
if (!s || s === "." || s === ".." || s.includes("/") || s.includes("\\") || s.includes("\0")) throw new Error(`invalid cache path segment "${s}"`);
return s;
};
return {
root,
deckDir: (courseId: string, deckId: string) => resolve(root, "decks", seg(courseId), seg(deckId)),
coversDir: (courseId: string) => resolve(root, "covers", seg(courseId)),
transcriptFile: (courseId: string, lessonId: string) => resolve(root, "transcripts", seg(courseId), `${seg(lessonId)}.json`),
transcriptsDir: (courseId: string) => resolve(root, "transcripts", seg(courseId)),
};
}
packages/core/src/index.ts
+4 -3
View File
@@ -1,4 +1,5 @@
export { loadConfig } from "./config";
export { loadConfig, resolveConfig, cachePaths, originOf, type ConfigInput } from "./config";
export { createJobQueue } from "./jobs";
export { serveMedia, contentType, contentDisposition } from "./media";
export { createApp, canView, type Routes } from "./app";
export { serveMedia, contentType, contentDisposition, mediaSecurityHeaders, MEDIA_SANDBOX_CSP } from "./media";
export { createApp, canView, crossSiteRefusal, type Routes } from "./app";
export { composeServer, type ComposeOptions, type ComposedServer } from "./compose";
packages/core/src/media.ts
+19 -2
View File
@@ -1,9 +1,23 @@
const TYPES: Record<string, string> = {
mp4: "video/mp4", webm: "video/webm", pdf: "application/pdf", png: "image/png", jpg: "image/jpeg", jpeg: "image/jpeg",
svg: "image/svg+xml", md: "text/markdown; charset=utf-8", txt: "text/plain; charset=utf-8", zip: "application/zip",
js: "text/javascript; charset=utf-8", ts: "text/plain; charset=utf-8",
js: "text/javascript; charset=utf-8", mjs: "text/javascript; charset=utf-8", ts: "text/plain; charset=utf-8",
html: "text/html; charset=utf-8", htm: "text/html; charset=utf-8", xhtml: "application/xhtml+xml", xml: "application/xml",
};
/** Types a browser could run script from. Course media is author content on the app origin, so these get a sandbox. */
const ACTIVE = new Set(["svg", "html", "htm", "xhtml", "xml", "js", "mjs"]);
/** Opaque origin, no script, no plugins: an svg still renders as an image, an html page as inert markup. */
export const MEDIA_SANDBOX_CSP = "sandbox; default-src 'none'; img-src 'self' data:; style-src 'unsafe-inline'; media-src 'self'";
/** Headers every media response carries; plus a sandbox CSP for script-capable types. */
export function mediaSecurityHeaders(path: string): Record<string, string> {
const ext = path.slice(path.lastIndexOf(".") + 1).toLowerCase();
const h: Record<string, string> = { "X-Content-Type-Options": "nosniff" };
if (ACTIVE.has(ext)) h["Content-Security-Policy"] = MEDIA_SANDBOX_CSP;
return h;
}
export function contentType(path: string): string {
const ext = path.slice(path.lastIndexOf(".") + 1).toLowerCase();
return TYPES[ext] ?? "application/octet-stream";
@@ -42,11 +56,14 @@ export async function serveMedia(req: Request, absPath: string, opts: { download
const headers: Record<string, string> = {
"Content-Type": contentType(absPath),
"Accept-Ranges": "bytes",
...mediaSecurityHeaders(absPath),
};
if (opts.download) headers["Content-Disposition"] = contentDisposition(name);
const range = req.headers.get("range");
const parsed = range ? parseRange(range, size) : null;
if (parsed === "invalid") return new Response(null, { status: 416, headers: { "Content-Range": `bytes */${size}`, "Accept-Ranges": "bytes" } });
if (parsed === "invalid" || (parsed && size === 0)) {
return new Response(null, { status: 416, headers: { "Content-Range": `bytes */${size}`, "Accept-Ranges": "bytes", "X-Content-Type-Options": "nosniff" } });
}
if (!parsed) {
headers["Content-Length"] = String(size);
return new Response(req.method === "HEAD" ? null : file, { status: 200, headers });
packages/core/test/app.test.ts
+8
View File
@@ -97,5 +97,13 @@ test("canView matrix", async () => {
const pub = ctx.courses.get("sample")!, priv = ctx.courses.get("secret")!;
expect(canView(null, pub, [])).toBe(true);
expect(canView(null, priv, [])).toBe(false);
const u = (roles: AuthUser["roles"], groups: string[] = []): AuthUser => ({ id: "u1", email: "u@x", name: "U", roles, groups });
expect(canView(u(["learner"]), priv, [])).toBe(false);
expect(canView(u(["owner"]), priv, [])).toBe(true);
expect(canView(u(["author"]), priv, [])).toBe(true); // R14: the global author role views every course
expect(canView(u(["learner"]), priv, [], true)).toBe(true); // per-course author
expect(canView(u(["learner"]), priv, [{ principal_type: "user", principal_id: "u1" }])).toBe(true);
expect(canView(u(["learner"], ["g"]), priv, [{ principal_type: "group", principal_id: "g" }])).toBe(true);
expect(canView(u(["learner"], ["h"]), priv, [{ principal_type: "group", principal_id: "g" }])).toBe(false);
await db.close();
});
packages/core/test/compose.test.ts
+51
View File
@@ -0,0 +1,51 @@
import { expect, test } from "bun:test";
import { mkdtempSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import type { AuthUser, Feature, ServerContext } from "@stc/contracts";
import { composeServer, resolveConfig } from "../src";
const FIXTURES = resolve(import.meta.dir, "../../../fixtures/courses");
const config = () => resolveConfig({ dataDir: mkdtempSync(join(tmpdir(), "stc-compose-")), databaseUrl: "sqlite://:memory:", courseDirs: [FIXTURES] });
test("factories get the context; the queue is built from the union of their jobs; auth and stop are honoured", async () => {
const seen: string[] = [];
let stopped = 0;
const user: AuthUser = { id: "u", email: "u@x", name: "U", roles: ["owner"], groups: [] };
let early: unknown = null;
const a = (ctx: ServerContext): Feature => {
// the handle exists while factories run, but enqueuing must wait for the real queue
try { void ctx.jobs.enqueue("b.work", {}); } catch (e) { early = e; }
return {
name: "a",
routes: { "/api/a": { GET: async (_r, rc) => Response.json({ who: rc.user?.id ?? null, courses: ctx.courses.list().length }) } },
jobs: { "a.work": async (p) => { seen.push(`a:${JSON.stringify(p)}`); return null; } },
auth: { userFromRequest: async () => user },
stop: () => { stopped++; },
};
};
const b = (ctx: ServerContext): Feature => ({
name: "b",
routes: { "/api/b": { POST: async () => Response.json({ id: await ctx.jobs.enqueue("a.work", { n: 1 }) }) } },
jobs: { "b.work": async () => null },
});
const s = await composeServer(config(), [a, b]);
try {
expect(String(early)).toMatch(/not ready/);
expect(await (await s.app.fetch(new Request("http://x/api/a"))).json()).toEqual({ who: "u", courses: 1 });
const { id } = (await (await s.app.fetch(new Request("http://x/api/b", { method: "POST", headers: { origin: "http://localhost:3000" } }))).json()) as { id: string };
for (let i = 0; i < 200 && (await s.ctx.jobs.get(id))?.status !== "done"; i++) await Bun.sleep(10);
expect((await s.ctx.jobs.get(id))?.status).toBe("done");
expect(seen).toEqual(['a:{"n":1}']);
} finally {
await s.stop();
}
expect(stopped).toBe(1);
});
test("duplicate job kinds and two auth services are refused", async () => {
const f = (name: string, extra: Partial<Feature>) => (): Feature => ({ name, routes: {}, ...extra });
await expect(composeServer(config(), [f("x", { jobs: { k: async () => null } }), f("y", { jobs: { k: async () => null } })])).rejects.toThrow(/Duplicate job kind "k"/);
const auth = { userFromRequest: async () => null };
await expect(composeServer(config(), [f("x", { auth }), f("y", { auth })])).rejects.toThrow(/AuthService/);
});
packages/core/test/config.test.ts
+43 -4
View File
@@ -1,5 +1,6 @@
import { expect, test, spyOn } from "bun:test";
import { loadConfig } from "../src";
import { resolve } from "node:path";
import { cachePaths, loadConfig, resolveConfig } from "../src";
test("defaults", () => {
const warn = spyOn(console, "warn").mockImplementation(() => {});
@@ -7,21 +8,59 @@ test("defaults", () => {
expect(c.port).toBe(3000);
expect(c.databaseUrl).toBe("sqlite://./data/stcourses.db");
expect(c.dataDir).toBe("./data");
expect(c.courseDirs).toEqual(["./overlays", "./courses"]);
// configured dirs first, then the importer's folders and the studio's published worktrees
expect(c.courseDirs).toEqual(["./overlays", "./courses", resolve("./data/courses"), resolve("./data/worktrees")]);
expect(c.importedCoursesDir).toBe(resolve("./data/courses"));
expect(c.studioReposDir).toBe(resolve("./data/studio/repos"));
expect(c.studioWorktreesDir).toBe(resolve("./data/worktrees"));
expect(c.cacheDir).toBe(resolve("./data/cache"));
expect(c.importRoots).toEqual([resolve("./data/import-inbox")]);
expect(c.uploadMaxMb).toBe(100);
expect(c.aiAllowedBaseUrls).toEqual([]);
expect(c.transcribeUrl).toBeNull();
expect(c.python).toBe("python3");
expect(c.publicUrl).toBe("http://localhost:3000");
expect(warn).toHaveBeenCalledTimes(1);
warn.mockRestore();
});
test("colon splitting and overrides", () => {
const c = loadConfig({ STC_PORT: "8080", STC_COURSE_DIRS: "/a b:/c", STC_SECRET: "s", STC_PUBLIC_URL: "https://x.test" });
const c = loadConfig({
STC_PORT: "8080", STC_COURSE_DIRS: "/a b:/c", STC_SECRET: "s", STC_PUBLIC_URL: "https://x.test", STC_DATA_DIR: "/d",
STC_IMPORT_ROOTS: "/in box:/other", STC_UPLOAD_MAX_MB: "5", STC_AI_ALLOWED_BASE_URLS: "http://gw.lan:8080, https://x.test",
STC_TRANSCRIBE_URL: "http://whisper.lan/v1/audio/transcriptions", STC_PYTHON: "/opt/py/bin/python",
});
expect(c.port).toBe(8080);
expect(c.courseDirs).toEqual(["/a b", "/c"]);
expect(c.courseDirs).toEqual(["/a b", "/c", "/d/courses", "/d/worktrees"]);
expect(c.secret).toBe("s");
expect(c.publicUrl).toBe("https://x.test");
expect(c.importRoots).toEqual(["/in box", "/other"]);
expect(c.uploadMaxMb).toBe(5);
expect(c.aiAllowedBaseUrls).toEqual(["http://gw.lan:8080", "https://x.test"]);
expect(c.transcribeUrl).toBe("http://whisper.lan/v1/audio/transcriptions");
expect(c.python).toBe("/opt/py/bin/python");
});
test("configured dirs that already name the derived dirs are not repeated", () => {
const c = loadConfig({ STC_SECRET: "s", STC_DATA_DIR: "/d", STC_COURSE_DIRS: "/d/courses:/x" });
expect(c.courseDirs).toEqual(["/d/courses", "/x", "/d/worktrees"]);
expect(resolveConfig({ dataDir: "/d", courseDirs: ["/d/worktrees/"] }).courseDirs).toEqual(["/d/worktrees/", "/d/courses"]);
});
test("invalid upload cap throws", () => {
expect(() => loadConfig({ STC_SECRET: "s", STC_UPLOAD_MAX_MB: "lots" })).toThrow(/STC_UPLOAD_MAX_MB/);
});
test("production without a secret throws", () => {
expect(() => loadConfig({ NODE_ENV: "production" })).toThrow(/STC_SECRET/);
expect(loadConfig({ NODE_ENV: "production", STC_SECRET: "x" }).secret).toBe("x");
});
test("cachePaths: the importer's layout, no escaping segments", () => {
const p = cachePaths({ cacheDir: "/d/cache" });
expect(p.deckDir("c", "deck1")).toBe("/d/cache/decks/c/deck1");
expect(p.coversDir("c")).toBe("/d/cache/covers/c");
expect(p.transcriptFile("c", "m1-l1")).toBe("/d/cache/transcripts/c/m1-l1.json");
expect(() => p.coversDir("..")).toThrow();
expect(() => p.deckDir("c", "a/b")).toThrow();
});
packages/core/test/csrf.test.ts
+84
View File
@@ -0,0 +1,84 @@
import { expect, test } from "bun:test";
import type { AuthUser, Feature, ServerContext } from "@stc/contracts";
import { createApp, resolveConfig } from "../src";
const owner: AuthUser = { id: "o", email: "o@x", name: "O", roles: ["owner"], groups: [] };
const ran: string[] = [];
const feature: Feature = {
name: "t",
routes: {
"/api/do": { POST: async () => { ran.push("do"); return Response.json({ ok: true }); }, DELETE: async () => { ran.push("del"); return Response.json({ ok: true }); }, GET: () => Response.json({ ok: true }) },
"/api/up": { POST: async () => { ran.push("up"); return Response.json({ ok: true }); } },
"/api/auth/*": { POST: async () => { ran.push("auth"); return Response.json({ ok: true }); } },
},
multipartRoutes: ["/api/up"],
externalCallbacks: ["/api/auth/sso/saml2/sp/acs"],
};
const config = resolveConfig({ dataDir: "/tmp/x", publicUrl: "https://courses.example.org", allowedOrigins: ["http://192.168.1.5:3000"] });
const app = createApp({ config, auth: { userFromRequest: async () => owner } } as unknown as ServerContext, [feature]);
const send = (path: string, init: { method?: string; headers?: Record<string, string>; body?: BodyInit } = {}) =>
app.fetch(new Request(`https://courses.example.org${path}`, { method: init.method ?? "POST", headers: init.headers, body: init.body }));
const JSONH = { "content-type": "application/json" };
test("same-origin JSON writes run; GET needs no Origin", async () => {
ran.length = 0;
expect((await send("/api/do", { headers: { origin: "https://courses.example.org", ...JSONH }, body: "{}" })).status).toBe(200);
expect((await send("/api/do", { headers: { origin: "http://192.168.1.5:3000", ...JSONH }, body: "{}" })).status).toBe(200); // STC_ALLOWED_ORIGINS
expect((await send("/api/do", { headers: { referer: "https://courses.example.org/c/x", ...JSONH }, body: "{}" })).status).toBe(200); // Referer fallback
expect((await send("/api/do", { method: "DELETE", headers: { origin: "https://courses.example.org" } })).status).toBe(200); // no body: no type needed
expect((await send("/api/do", { method: "GET" })).status).toBe(200);
expect(ran).toEqual(["do", "do", "do", "del"]);
});
test("foreign, missing or opaque Origin is 403 and the handler never runs", async () => {
ran.length = 0;
for (const headers of <Record<string, string>[]>[
{ origin: "http://evil.example", "content-type": "text/plain" },
{ origin: "http://evil.example", ...JSONH },
{ origin: "https://courses.example.org.evil.example", ...JSONH },
{ origin: "http://courses.example.org", ...JSONH }, // scheme differs
{ origin: "null", ...JSONH },
{ referer: "http://evil.example/page", ...JSONH },
{ ...JSONH }, // neither Origin nor Referer
]) {
const r = await send("/api/do", { headers, body: "{}" });
expect([headers, r.status]).toEqual([headers, 403]);
}
expect((await send("/api/do", { method: "DELETE" })).status).toBe(403);
expect(ran).toEqual([]);
});
test("same-origin bodies must be JSON (multipart only on upload routes)", async () => {
ran.length = 0;
const o = { origin: "https://courses.example.org" };
for (const type of ["text/plain", "application/x-www-form-urlencoded", "multipart/form-data; boundary=x", ""]) {
expect([type, (await send("/api/do", { headers: { ...o, ...(type ? { "content-type": type } : {}) }, body: "a=1" })).status]).toEqual([type, 403]);
}
const fd = new FormData();
fd.set("file", new File(["x"], "a.txt"));
expect((await send("/api/up", { headers: o, body: fd })).status).toBe(200);
expect((await send("/api/up", { headers: { origin: "http://evil.example" }, body: fd })).status).toBe(403);
expect((await send("/api/up", { headers: { ...o, "content-type": "text/plain" }, body: "x" })).status).toBe(403);
expect(ran).toEqual(["up"]);
});
test("Better Auth routes are guarded too, except declared identity-provider callbacks", async () => {
ran.length = 0;
expect((await send("/api/auth/sign-in/email", { headers: { origin: "http://evil.example", ...JSONH }, body: "{}" })).status).toBe(403);
expect((await send("/api/auth/sign-in/email", { headers: { origin: "https://courses.example.org", ...JSONH }, body: "{}" })).status).toBe(200);
// a SAML IdP posts a form from its own origin; Better Auth verifies the signed assertion
expect((await send("/api/auth/sso/saml2/sp/acs/idp1", { headers: { origin: "https://idp.example", "content-type": "application/x-www-form-urlencoded" }, body: "SAMLResponse=x" })).status).toBe(200);
expect(ran).toEqual(["auth", "auth"]);
});
test("no configuration means every write is refused", async () => {
const bare = createApp({ auth: { userFromRequest: async () => owner } } as unknown as ServerContext, [feature]);
const r = await bare.fetch(new Request("http://x/api/do", { method: "POST", headers: { origin: "http://x", ...JSONH }, body: "{}" }));
expect(r.status).toBe(403);
});
test("loadConfig reads STC_ALLOWED_ORIGINS (and STC_TRUSTED_ORIGINS) as origins", async () => {
const { loadConfig } = await import("../src");
const c = loadConfig({ STC_SECRET: "s", STC_PUBLIC_URL: "https://Courses.Example.org/app/", STC_ALLOWED_ORIGINS: "http://lan:3000/, junk", STC_TRUSTED_ORIGINS: "https://b.example" });
expect(c.allowedOrigins).toEqual(["https://courses.example.org", "http://lan:3000", "https://b.example"]);
});
packages/core/test/media.test.ts
+33
View File
@@ -57,3 +57,36 @@ test("non-ASCII filename gets a valid filename*", async () => {
test("missing file is 404", async () => {
expect((await serveMedia(req(), join(dir, "nope.mp4"))).status).toBe(404);
});
test("every response is nosniff; script-capable types get a sandbox CSP", async () => {
const d = mkdtempSync(join(tmpdir(), "stc-media-csp-"));
const { MEDIA_SANDBOX_CSP } = await import("../src");
for (const [name, body, sandboxed] of [
["x.svg", "<svg xmlns='http://www.w3.org/2000/svg'><script>alert(1)</script></svg>", true],
["page.html", "<script>alert(1)</script>", true],
["Page.HTM", "<p>x</p>", true],
["code.js", "alert(1)", true],
["mod.mjs", "export {}", true],
["notes.md", "# hi", false],
["clip.mp4", "x", false],
] as const) {
writeFileSync(join(d, name), body);
for (const r of [await serveMedia(req(), join(d, name)), await serveMedia(req("bytes=0-0"), join(d, name))]) {
expect(r.headers.get("x-content-type-options")).toBe("nosniff");
expect([name, r.headers.get("content-security-policy")]).toEqual([name, sandboxed ? MEDIA_SANDBOX_CSP : null]);
}
}
expect(MEDIA_SANDBOX_CSP).toMatch(/^sandbox;/);
expect(MEDIA_SANDBOX_CSP).not.toContain("allow-scripts");
});
test("a range on an empty file is 416, never a negative Content-Range", async () => {
const d = mkdtempSync(join(tmpdir(), "stc-media-empty-"));
writeFileSync(join(d, "empty.txt"), "");
for (const range of ["bytes=0-", "bytes=-5", "bytes=0-0"]) {
const r = await serveMedia(req(range), join(d, "empty.txt"));
expect(r.status).toBe(416);
expect(r.headers.get("content-range")).toBe("bytes */0");
}
expect((await serveMedia(req(), join(d, "empty.txt"))).status).toBe(200);
});
packages/course-format/package.json
+4
View File
@@ -6,5 +6,9 @@
"license": "AGPL-3.0-only",
"dependencies": {
"zod": "^4.6.7"
},
"exports": {
".": "./src/index.ts",
"./markdown": "./src/markdown.ts"
}
}
packages/db/src/index.ts
+1 -1
View File
@@ -1,5 +1,5 @@
export { connect, type Db } from "./connect";
export { migrate } from "./migrate";
export { migrate, type PackageMigrations } from "./migrate";
export { progressRepo, type LessonProgress, type ProgressState } from "./repos/progress";
export { notesRepo, type Note, type NoteInput } from "./repos/notes";
export { attemptsRepo, type Attempt } from "./repos/attempts";
packages/db/src/migrate.ts
+25 -10
View File
@@ -9,19 +9,34 @@ function statements(text: string): string[] {
return noComments.split(";").map((s) => s.trim()).filter(Boolean);
}
export async function migrate(db: Db): Promise<string[]> {
/** Migration folders contributed by feature packages, keyed by package name. */
export type PackageMigrations = Record<string, string[]>;
/**
* Applies pending migrations: core (`packages/db/migrations`) first, then each package's folders in alphabetical
* order of package name, files in name order within a folder. `schema_migrations` is keyed by file name, so file
* names must be unique across packages (each package prefixes its files, e.g. `0001_learner_auth.sql`).
*/
export async function migrate(db: Db, packages: PackageMigrations = {}): Promise<string[]> {
await db.sql.unsafe("CREATE TABLE IF NOT EXISTS schema_migrations (name TEXT PRIMARY KEY, applied_at TEXT NOT NULL)");
const done = new Set((await db.sql`SELECT name FROM schema_migrations`).map((r: { name: string }) => r.name));
const files = readdirSync(MIGRATIONS_DIR).filter((f) => f.endsWith(".sql")).sort();
const dirs = [MIGRATIONS_DIR, ...Object.keys(packages).sort().flatMap((k) => packages[k]!)];
const seen = new Map<string, string>();
const applied: string[] = [];
for (const name of files) {
if (done.has(name)) continue;
const stmts = statements(readFileSync(join(MIGRATIONS_DIR, name), "utf8"));
await db.sql.begin(async (tx) => {
for (const s of stmts) await tx.unsafe(s);
await tx`INSERT INTO schema_migrations (name, applied_at) VALUES (${name}, ${new Date().toISOString()})`;
});
applied.push(name);
for (const dir of dirs) {
for (const name of readdirSync(dir).filter((f) => f.endsWith(".sql")).sort()) {
const other = seen.get(name);
if (other && other !== dir) throw new Error(`migration ${name} exists in both ${other} and ${dir}`);
seen.set(name, dir);
if (done.has(name)) continue;
const stmts = statements(readFileSync(join(dir, name), "utf8"));
await db.sql.begin(async (tx) => {
for (const s of stmts) await tx.unsafe(s);
await tx`INSERT INTO schema_migrations (name, applied_at) VALUES (${name}, ${new Date().toISOString()})`;
});
done.add(name);
applied.push(name);
}
}
return applied;
}
packages/db/test/migrate.test.ts
+19
View File
@@ -30,3 +30,22 @@ forEachDb("all tables exist", async (db) => {
await one("collection_courses", s`INSERT INTO collection_courses VALUES ('co','c',0)`);
await one("jobs", s`INSERT INTO jobs VALUES ('j','k','queued',0,NULL,'{}',NULL,NULL,${t},${t})`);
});
forEachDb("package migrations run after core, in package-name order, once", async (db) => {
const { mkdtempSync, writeFileSync, mkdirSync } = await import("node:fs");
const { join } = await import("node:path");
const { tmpdir } = await import("node:os");
const root = mkdtempSync(join(tmpdir(), "stc-mig-"));
const a = join(root, "a"), z = join(root, "z");
mkdirSync(a); mkdirSync(z);
// z's table is referenced by a's second file only through ordering by package name: "alpha" runs before "zeta".
writeFileSync(join(z, "0001_zeta.sql"), "CREATE TABLE zeta_t (id TEXT PRIMARY KEY);");
writeFileSync(join(a, "0001_alpha.sql"), "CREATE TABLE alpha_t (id TEXT PRIMARY KEY);");
writeFileSync(join(a, "0002_alpha.sql"), "INSERT INTO alpha_t VALUES ('x');");
expect(await migrate(db, { zeta: [z], alpha: [a] })).toEqual(["0001_alpha.sql", "0002_alpha.sql", "0001_zeta.sql"]);
expect(await migrate(db, { zeta: [z], alpha: [a] })).toEqual([]);
expect((await db.sql`SELECT id FROM alpha_t`).length).toBe(1);
// the same file name in two packages is refused rather than silently skipped
writeFileSync(join(z, "0001_alpha.sql"), "SELECT 1;");
await expect(migrate(db, { zeta: [z], alpha: [a] })).rejects.toThrow(/exists in both/);
});
packages/feature-ai/package.json
+5 -1
View File
@@ -3,5 +3,9 @@
"version": "0.0.0",
"private": true,
"type": "module",
"license": "AGPL-3.0-only"
"license": "AGPL-3.0-only",
"exports": {
".": "./src/index.ts",
"./web": "./src/web/index.ts"
}
}
packages/feature-ai/scripts/dev.ts
+48
View File
@@ -0,0 +1,48 @@
// Local dev server for the AI package until Phase 2 wires feature factories into apps/server.
// STC_PORT=3464 STC_DATA_DIR=/tmp/stc-ai-data STC_COURSE_DIRS=./fixtures/courses:./overlays bun packages/feature-ai/scripts/dev.ts
// STC_AI_DEV_USER=owner|learner|none picks the signed-in user (dev only). STC_AI_DEV_FAKE=1 uses a canned provider.
import type { AiProvider, AuthService, ServerContext } from "@stc/contracts";
import { createApp, createJobQueue, loadConfig } from "@stc/core";
import { createCourseRepository } from "@stc/course-format";
import { connect, migrate } from "@stc/db";
import indexHtml from "../../../apps/web/index.html";
import { aiFeature } from "../src/feature";
const config = loadConfig();
const who = process.env.STC_AI_DEV_USER ?? "owner";
const auth: AuthService = {
userFromRequest: async () =>
who === "none" ? null : { id: `dev-${who}`, email: `${who}@localhost`, name: who, roles: who === "owner" ? ["owner"] : ["learner"], groups: [] },
};
const fake: AiProvider = {
id: "fake",
async *chat(messages) {
const sys = messages[0]?.content ?? "";
if (sys.includes("=== FILE:")) {
yield { text: "=== FILE: questions/adding-many-numbers.md ===\n# Check your understanding: adding many numbers\n\n1. Predict: what does `sum([])` return?\n2. What would break if `sum` used `a - b`?\n=== END FILE ===\n" };
return;
}
const mentor = sys.includes("## Course mentor rules");
const answer = mentor
? "Before I explain: what does the failing test give as input, and what did you expect `add` to return?\n\nThe lesson's nudge asks **which operator combines two numbers**. Try it, run the tests, and tell me what happens. [lessons/04-exercise.md · Adding numbers]"
: "Here is the whole answer.\n\n```js\nexport const add = (a, b) => a + b\n```\n\nIt returns the sum of its arguments. [lessons/04-exercise.md · Adding numbers]";
for (const part of answer.match(/.{1,12}/gs) ?? []) { await Bun.sleep(5); yield { text: part }; }
},
};
const db = await connect(config.databaseUrl);
await migrate(db);
const courses = await createCourseRepository(config.courseDirs);
const ctx = { config, db, courses, auth } as ServerContext;
const feature = aiFeature(ctx, process.env.STC_AI_DEV_FAKE ? { providerFactory: () => fake } : {});
const jobs = createJobQueue(db, feature.jobs ?? {});
ctx.jobs = jobs;
const app = createApp(ctx, [feature]);
jobs.start();
const server = Bun.serve({
port: config.port,
routes: { ...app.routes, "/api/*": (req) => app.fetch(req), "/*": indexHtml },
fetch: (req) => app.fetch(req),
});
console.log(`STCourses (AI dev) on ${server.url} as ${who}`);
packages/feature-ai/src/assistant/loop.ts
+52
View File
@@ -0,0 +1,52 @@
import type { AiProvider, ChatMessage } from "@stc/contracts";
import type { ToolSet } from "./tools";
export type AssistantEvent =
| { type: "text"; text: string }
| { type: "tool"; name: string }
| { type: "error"; message: string }
| { type: "done"; text: string };
/**
* Runs the model with tools until it answers without calling a tool, or until maxSteps model calls.
* Mutates `messages` by appending the tool round trips, so callers can inspect them.
*/
export async function* runAssistant(o: {
provider: AiProvider;
messages: ChatMessage[];
tools: ToolSet;
signal?: AbortSignal;
maxSteps?: number;
}): AsyncIterable<AssistantEvent> {
const maxSteps = o.maxSteps ?? 6;
let answer = "";
for (let step = 0; step < maxSteps; step++) {
let text = "";
const calls: { id: string; name: string; input: unknown }[] = [];
try {
for await (const ev of o.provider.chat(o.messages, { tools: o.tools.specs, signal: o.signal })) {
if (ev.text) { text += ev.text; yield { type: "text", text: ev.text }; }
if (ev.toolCall) calls.push(ev.toolCall);
}
} catch (e) {
yield { type: "error", message: (e as Error).message };
return;
}
answer += text;
if (!calls.length) {
yield { type: "done", text: answer };
return;
}
if (text) o.messages.push({ role: "assistant", content: text });
for (const c of calls) o.messages.push({ role: "assistant", content: JSON.stringify(c.input ?? {}), toolCallId: c.id, name: c.name });
for (const c of calls) {
yield { type: "tool", name: c.name };
let result: string;
try { result = await o.tools.run(c.name, c.input); } catch (e) { result = `Tool error: ${(e as Error).message}`; }
o.messages.push({ role: "tool", content: result, toolCallId: c.id, name: c.name });
}
}
const note = "\n\n(I stopped after several lookups. Ask again to continue.)";
yield { type: "text", text: note };
yield { type: "done", text: answer + note };
}
packages/feature-ai/src/assistant/prompt.ts
+72
View File
@@ -0,0 +1,72 @@
import type { CourseRepository, LoadedCourse } from "@stc/course-format";
import type { Chunk } from "../retrieval/course-index";
export interface Citation { path: string; kind: Chunk["kind"]; lessonId?: string; lessonTitle?: string }
const MAX_MENTOR = 40_000;
/** The manifest's `mentor` file (for example Pure Snake's MENTOR.md), or null when the course has none. */
export async function readMentorText(repo: CourseRepository, course: LoadedCourse): Promise<string | null> {
const rel = course.manifest?.mentor;
if (!rel) return null;
const abs = repo.resolvePath(course.id, rel);
if (!abs) return null;
try { return (await Bun.file(abs).text()).slice(0, MAX_MENTOR); } catch { return null; }
}
const label = (c: Chunk) => [c.path, c.lessonTitle && `${c.lessonTitle} (${c.lessonId})`, c.heading].filter(Boolean).join(" · ");
export function buildSystemPrompt(o: {
course: LoadedCourse;
mentorText: string | null;
mentorOn: boolean;
sources: Chunk[];
lesson?: { id: string; title: string } | null;
extraTools?: string[];
}): string {
const title = o.course.manifest?.title ?? o.course.id;
const parts: string[] = [];
parts.push(
`You are the course assistant for "${title}" on STCourses, a self-hosted course platform. You help one learner with this course: you answer questions, explain ideas, and quiz them when they ask.`,
[
"Ground your answers in the course material. Use the retrieved sources below and your tools (search_course, read_lesson, read_file, list_notes, my_progress) to look things up rather than guessing.",
"When you rely on a source, cite its file path and lesson in square brackets, for example [lessons/04-exercise.md · Adding numbers]. If the course does not cover something, say so plainly.",
"Reference solutions are for authors only and are not available to you. Do not try to reconstruct them.",
"Keep answers short and plain. Use Markdown for code.",
].join("\n"),
);
if (o.extraTools?.length) parts.push(`The administrator also connected these external tools: ${o.extraTools.join(", ")}. Use them only when the course material is not enough.`);
if (o.lesson) parts.push(`The learner is currently on the lesson "${o.lesson.title}" (id ${o.lesson.id}).`);
if (o.mentorText && o.mentorOn) {
parts.push(
`## Course mentor rules\n\nThis course ships the following rules for an AI tutor. Follow them for the whole conversation; they take priority over being maximally helpful. Where they mention files, tools or a workspace you cannot access, apply their spirit through this chat.\n\n<mentor_rules>\n${o.mentorText.trim()}\n</mentor_rules>`,
);
} else if (o.mentorText && !o.mentorOn) {
parts.push("## Mentor mode is off\n\nThe learner switched off this course's mentor rules for themselves. Give full, direct answers, including complete worked solutions when they ask for them.");
}
if (o.sources.length) {
parts.push(
`## Retrieved sources\n\nThese excerpts matched the learner's question. They may be incomplete; use the tools to read more.\n\n${o.sources
.map((c, i) => `[${i + 1}] ${label(c)}\n${c.text}`)
.join("\n\n---\n\n")}`,
);
}
return parts.join("\n\n");
}
/** Course files and lessons behind an answer, deduplicated. Notes are the learner's own and are not cited as files. */
export function citationsFor(chunks: Chunk[]): Citation[] {
const seen = new Set<string>();
const out: Citation[] = [];
for (const c of chunks) {
if (c.kind === "note") continue;
const key = `${c.path}\u0000${c.lessonId ?? ""}`;
if (seen.has(key)) continue;
seen.add(key);
const cit: Citation = { path: c.path, kind: c.kind };
if (c.lessonId) cit.lessonId = c.lessonId;
if (c.lessonTitle) cit.lessonTitle = c.lessonTitle;
out.push(cit);
}
return out;
}
packages/feature-ai/src/assistant/tools.ts
+182
View File
@@ -0,0 +1,182 @@
import { stat } from "node:fs/promises";
import type { ToolSpec } from "@stc/contracts";
import type { CourseRepository, LoadedCourse } from "@stc/course-format";
import type { McpServerConfig } from "../settings";
import { connectMcp, mcpToolName, type McpClient } from "../mcp/client";
import { canonicalCoursePath, isGuardedFile, noteChunks, type Chunk, type CourseIndex } from "../retrieval/course-index";
export interface ToolSet {
specs: ToolSpec[];
/** Runs a tool and returns text for the model. Never throws for expected failures. */
run(name: string, input: unknown): Promise<string>;
/** Course chunks the tools surfaced (for citations). */
sources: Chunk[];
close?(): Promise<void>;
}
type ProgressLike = { state: "not-started" | "in-progress" | "done"; positionSeconds: number | null };
type NoteLike = { id: string; lessonId: string; body: string; anchorKind: string; anchorValue: string | null };
const MAX_READ = 60_000;
const str = (v: unknown) => (typeof v === "string" ? v : "");
const obj = (v: unknown): Record<string, unknown> => (v && typeof v === "object" ? (v as Record<string, unknown>) : {});
export function courseTools(o: {
repo: CourseRepository;
course: LoadedCourse;
index: CourseIndex;
listNotes: (lessonId?: string) => Promise<NoteLike[]>;
/** The learner's progress in this course, keyed by lesson id (shape of `progressRepo(db).get`). */
getProgress?: () => Promise<Record<string, ProgressLike>>;
}): ToolSet {
const sources: Chunk[] = [];
const { course } = o;
const lessonChunk = (lessonId: string): Chunk | null => {
const e = course.lessonIndex.get(lessonId);
if (!e) return null;
const path = e.lesson.type === "video" ? (e.lesson.transcript ?? e.lesson.file ?? "") : (e.lesson.file ?? "");
return { path, kind: e.lesson.type === "video" ? "transcript" : "lesson", text: "", lessonId, lessonTitle: e.lesson.title };
};
const specs: ToolSpec[] = [
{
name: "search_course",
description: "Search this course's lessons, transcripts, slide text and code files. Returns the best matching excerpts with their file paths and lessons.",
inputSchema: { type: "object", properties: { query: { type: "string", description: "Words to search for" } }, required: ["query"], additionalProperties: false },
},
{
name: "read_lesson",
description: "Read the full Markdown of one lesson (or a video lesson's transcript) by lesson id.",
inputSchema: { type: "object", properties: { lessonId: { type: "string" } }, required: ["lessonId"], additionalProperties: false },
},
{
name: "read_file",
description: "Read a text or code file from the course by its course-relative path, for example a companion code file.",
inputSchema: { type: "object", properties: { path: { type: "string" } }, required: ["path"], additionalProperties: false },
},
{
name: "list_notes",
description: "List the learner's own notes in this course, optionally for one lesson.",
inputSchema: { type: "object", properties: { lessonId: { type: "string" } }, additionalProperties: false },
},
...(o.getProgress
? [{
name: "my_progress",
description: "The learner's progress in this course: which lessons are done, in progress (with the video position) or not started.",
inputSchema: { type: "object", properties: {}, additionalProperties: false },
}]
: []),
];
const lessonList = () =>
(course.manifest?.modules ?? []).map((m) => `${m.title}: ${m.lessons.map((l) => `${l.id} (${l.title})`).join(", ")}`).join("\n");
return {
specs,
sources,
async run(name, input) {
const a = obj(input);
switch (name) {
case "search_course": {
const q = str(a.query);
if (!q.trim()) return "Give a non-empty query.";
const hits = o.index.search(q, 6);
sources.push(...hits);
if (!hits.length) return "No matches in this course.";
return hits.map((c, i) => `[${i + 1}] ${[c.path, c.lessonTitle && `${c.lessonTitle} (${c.lessonId})`, c.heading].filter(Boolean).join(" · ")}\n${c.text}`).join("\n\n---\n\n");
}
case "read_lesson": {
const id = str(a.lessonId);
const ref = lessonChunk(id);
if (!ref) return `No lesson with id "${id}". Lessons:\n${lessonList()}`;
const canon = ref.path ? canonicalCoursePath(o.repo, course, ref.path) : null;
if (canon && isGuardedFile(o.repo, course, canon)) return "That file is for authors only.";
const md = await o.repo.readLessonMarkdown(course.id, id);
if (md === null) return `Lesson "${id}" has no readable text (it may be a video without a transcript).`;
sources.push(ref);
return md.slice(0, MAX_READ);
}
case "read_file": {
// Canonicalise first (decode, normalise, follow symlinks), then guard the real path.
const canon = canonicalCoursePath(o.repo, course, str(a.path));
if (!canon) return `No such file in this course: ${str(a.path)}`;
if (isGuardedFile(o.repo, course, canon)) return "That file is for authors only and is not available.";
const info = await stat(canon.abs).catch(() => null);
if (!info?.isFile()) return `No such file in this course: ${canon.rel}`;
if (info.size > 2_000_000) return "That file is too large to read here.";
const text = await Bun.file(canon.abs).text();
if (text.includes("\u0000")) return "That is a binary file.";
sources.push({ path: canon.rel, kind: /\.(md|markdown)$/i.test(canon.rel) ? "markdown" : "code", text: "" });
return text.slice(0, MAX_READ);
}
case "list_notes": {
const lessonId = str(a.lessonId) || undefined;
const notes = await o.listNotes(lessonId);
if (!notes.length) return "The learner has no notes here yet.";
return noteChunks(course, notes).map((c) => `- ${[c.lessonTitle ?? c.lessonId, c.heading].filter(Boolean).join(", ")}: ${c.text}`).join("\n");
}
case "my_progress": {
if (!o.getProgress) return `Unknown tool "${name}".`;
const p = await o.getProgress();
const lessons = (course.manifest?.modules ?? []).flatMap((m) => m.lessons);
const done = lessons.filter((l) => p[l.id]?.state === "done").length;
const clock = (s: number) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
const line = (l: { id: string; title: string }) => {
const e = p[l.id];
const state = e?.state === "done" ? "done" : e?.state === "in-progress" ? `in progress${e.positionSeconds ? `, at ${clock(e.positionSeconds)}` : ""}` : "not started";
return `- ${l.title} (${l.id}): ${state}`;
};
return [`${done} of ${lessons.length} lessons done.`, ...(course.manifest?.modules ?? []).flatMap((m) => [`${m.title}:`, ...m.lessons.map(line)])].join("\n");
}
default:
return `Unknown tool "${name}".`;
}
},
};
}
/** Tools from admin-configured MCP servers. Servers that fail to connect are skipped and reported. */
export async function mcpTools(servers: McpServerConfig[], opts: { fetch?: typeof fetch; timeoutMs?: number } = {}): Promise<ToolSet & { warnings: string[]; names: string[] }> {
const clients: McpClient[] = [];
const routes = new Map<string, { client: McpClient; tool: string }>();
const specs: ToolSpec[] = [];
const warnings: string[] = [];
await Promise.all(
servers.map(async (s) => {
try {
const client = await connectMcp(s, opts);
clients.push(client);
for (const t of await client.tools()) {
const name = mcpToolName(s.id, t.name);
routes.set(name, { client, tool: t.name });
specs.push({ name, description: `[${s.name}] ${t.description}`.slice(0, 1024), inputSchema: t.inputSchema });
}
} catch (e) {
warnings.push(`MCP server "${s.name}" is unavailable: ${(e as Error).message}`);
}
}),
);
return {
specs,
sources: [],
warnings,
names: specs.map((s) => s.name),
async run(name, input) {
const r = routes.get(name);
if (!r) return `Unknown tool "${name}".`;
try { return (await r.client.call(r.tool, input)).slice(0, MAX_READ); } catch (e) { return `Tool error: ${(e as Error).message}`; }
},
async close() { await Promise.all(clients.map((c) => c.close().catch(() => {}))); },
};
}
export function combineTools(sets: ToolSet[]): ToolSet {
const owner = new Map<string, ToolSet>();
for (const s of sets) for (const spec of s.specs) if (!owner.has(spec.name)) owner.set(spec.name, s);
return {
specs: [...owner.keys()].map((n) => owner.get(n)!.specs.find((s) => s.name === n)!),
get sources() { return sets.flatMap((s) => s.sources); },
async run(name, input) {
const s = owner.get(name);
return s ? s.run(name, input) : `Unknown tool "${name}".`;
},
async close() { await Promise.all(sets.map((s) => s.close?.())); },
};
}
packages/feature-ai/src/crypto.ts
+41
View File
@@ -0,0 +1,41 @@
/**
* Secret box for API keys at rest: AES-GCM-256 with a key derived by HKDF-SHA-256 from STC_SECRET.
* Token format: `v1.<base64 iv>.<base64 ciphertext+tag>`.
*/
export interface SecretBox {
encrypt(plain: string): Promise<string>;
decrypt(token: string): Promise<string>;
}
const enc = new TextEncoder();
const dec = new TextDecoder();
const b64 = (u: Uint8Array) => Buffer.from(u).toString("base64");
const unb64 = (s: string) => new Uint8Array(Buffer.from(s, "base64"));
export function createSecretBox(secret: string): SecretBox {
let keyP: Promise<CryptoKey> | null = null;
const key = () =>
(keyP ??= (async () => {
const ikm = await crypto.subtle.importKey("raw", enc.encode(secret), "HKDF", false, ["deriveKey"]);
return crypto.subtle.deriveKey(
{ name: "HKDF", hash: "SHA-256", salt: enc.encode("stcourses/byok"), info: enc.encode("aes-gcm-256/v1") },
ikm,
{ name: "AES-GCM", length: 256 },
false,
["encrypt", "decrypt"],
);
})());
return {
async encrypt(plain) {
const iv = crypto.getRandomValues(new Uint8Array(12));
const ct = new Uint8Array(await crypto.subtle.encrypt({ name: "AES-GCM", iv }, await key(), enc.encode(plain)));
return `v1.${b64(iv)}.${b64(ct)}`;
},
async decrypt(token) {
const [v, iv, ct] = token.split(".");
if (v !== "v1" || !iv || !ct) throw new Error("Unrecognised secret format");
const plain = await crypto.subtle.decrypt({ name: "AES-GCM", iv: unb64(iv) }, await key(), unb64(ct));
return dec.decode(plain);
},
};
}
packages/feature-ai/src/endpoints.ts
+92
View File
@@ -0,0 +1,92 @@
import type { AiProvider } from "@stc/contracts";
import type { ProviderKind } from "./settings";
/**
* Where a user's own key (BYOK) may be sent. Learners must not be able to point the server at arbitrary hosts
* (SSRF), so BYOK uses the official endpoint for its kind, or an origin the admin allowed explicitly through
* STC_AI_ALLOWED_BASE_URLS (comma-separated origins) or the server AI config's `allowedBaseUrls`.
* The admin's own server-scope provider may use any base URL (Ollama and the like).
*/
export const PRESET_BASE_URLS: Record<ProviderKind, string> = {
anthropic: "https://api.anthropic.com",
openai: "https://api.openai.com/v1",
};
const trimSlash = (s: string) => s.replace(/\/+$/, "");
/** The origin of an http(s) URL, or null. "Exact origins" means scheme, host and port must all match. */
export function originOf(u: string): string | null {
try {
const url = new URL(u.trim());
if (url.protocol !== "http:" && url.protocol !== "https:") return null;
if (url.username || url.password) return null;
return url.origin;
} catch {
return null;
}
}
/** Parses an allowlist (array or comma-separated string) into origins; entries that are not plain origins are dropped. */
export function parseAllowedOrigins(v: unknown): string[] {
const items = Array.isArray(v) ? v.map(String) : typeof v === "string" ? v.split(",") : [];
const out = new Set<string>();
for (const raw of items) {
const s = trimSlash(raw.trim());
if (!s) continue;
const o = originOf(s);
if (o && o === s) out.add(o);
}
return [...out];
}
/**
* The base URL a BYOK config may use, or null when it is not allowed.
* - empty: the official endpoint for the kind;
* - the official endpoint for the kind (with or without a trailing slash);
* - any URL whose origin is on the allowlist.
*/
export function byokBaseUrl(kind: ProviderKind, baseUrl: string | undefined, allowed: string[]): string | null {
const preset = PRESET_BASE_URLS[kind];
if (!baseUrl || !baseUrl.trim()) return preset;
const clean = trimSlash(baseUrl.trim());
if (clean === preset) return preset;
const origin = originOf(clean);
if (!origin) return null;
const url = new URL(clean);
if (url.search || url.hash) return null;
return allowed.includes(origin) ? clean : null;
}
/** fetch for user-scope calls: never follows redirects (an allowed host must not bounce the request elsewhere). */
export function noRedirectFetch(base: typeof fetch): typeof fetch {
const f = ((input: RequestInfo | URL, init?: RequestInit) => base(input, { ...init, redirect: "manual" })) as typeof fetch;
return f;
}
export const USER_PROVIDER_ERROR = "The AI provider returned an error. Check your key, model and endpoint under Account, AI.";
/** The message shown for a failed user-scope provider call: the HTTP status, if any, and a fixed sentence. */
export function publicProviderError(message: string): string {
const status = /(?:^|\s)([1-5]\d{2}):/.exec(message)?.[1];
return status ? `${status}: ${USER_PROVIDER_ERROR}` : USER_PROVIDER_ERROR;
}
/**
* Wraps a provider created with a user's key: any error is logged server-side with its full text and rethrown
* as the status plus a fixed message, so upstream response bodies never reach the browser.
*/
export function sanitizeProviderErrors(p: AiProvider, log: (msg: string) => void): AiProvider {
return {
id: p.id,
async *chat(messages, opts) {
try {
yield* p.chat(messages, opts);
} catch (e) {
const raw = (e as Error)?.message ?? String(e);
if ((e as Error)?.name === "AbortError") throw e;
log(raw);
throw new Error(publicProviderError(raw));
}
},
};
}
packages/feature-ai/src/feature.ts
+480
View File
@@ -0,0 +1,480 @@
import { readdir } from "node:fs/promises";
import { extname, join, relative, sep } from "node:path";
import type { AiProvider, AuthUser, ChatMessage, Feature, JobContext, RequestContext, ServerContext } from "@stc/contracts";
import { canView } from "@stc/core";
import type { LoadedCourse } from "@stc/course-format";
import { aclRepo, notesRepo, progressRepo } from "@stc/db";
import { buildSystemPrompt, citationsFor, readMentorText } from "./assistant/prompt";
import { combineTools, courseTools, mcpTools, type ToolSet } from "./assistant/tools";
import { runAssistant } from "./assistant/loop";
import { createSecretBox } from "./crypto";
import { createFsDraftSink, type DraftSink } from "./generator/draft-sink";
import { generateDraft, type GenerateRequest, type GenerateSource } from "./generator/generate";
import { connectMcp, mcpServersFor } from "./mcp/client";
import { createProvider } from "./providers";
import { PRESET_BASE_URLS, byokBaseUrl, noRedirectFetch, parseAllowedOrigins, publicProviderError, sanitizeProviderErrors } from "./endpoints";
import { getCourseIndex, noteChunks } from "./retrieval/course-index";
import { aiSettingsRepo, type McpServerConfig, type ProviderConfig } from "./settings";
export interface AiFeatureOptions {
/** Used by providers and HTTP MCP servers. Tests pass a fake; production uses global fetch. */
fetch?: typeof fetch;
/** Where generated drafts go. Phase 2 passes the studio's createDraftWriter(ctx); default writes under <dataDir>/drafts. */
draftSink?: DraftSink;
/** Overrides provider construction (tests use a fake AiProvider). */
providerFactory?: (cfg: ProviderConfig, apiKey: string | null) => AiProvider;
mcpTimeoutMs?: number;
/** Server-side log for provider errors that are not shown to users. Default: console.error. */
log?: (message: string) => void;
}
const json = (body: unknown, status = 200) => Response.json(body, { status });
const err = (status: number, message: string) => json({ error: message }, status);
const KINDS = new Set(["anthropic", "openai"]);
const EFFORTS = new Set(["low", "medium", "high", "xhigh", "max"]);
const HISTORY_TURNS = 20;
const MAX_MESSAGE = 8_000;
async function body(req: Request): Promise<Record<string, unknown> | null> {
try {
const b = await req.json();
return b && typeof b === "object" && !Array.isArray(b) ? (b as Record<string, unknown>) : null;
} catch { return null; }
}
const str = (v: unknown): string | undefined => (typeof v === "string" && v.trim() ? v.trim() : undefined);
function parseProvider(v: unknown, scope: "server" | "user"): (ProviderConfig & { apiKey?: string; clearApiKey?: boolean }) | null | "invalid" {
if (v === null || v === undefined) return null;
if (typeof v !== "object") return "invalid";
const p = v as Record<string, unknown>;
if (!KINDS.has(p.kind as string)) return "invalid";
if (p.effort !== undefined && p.effort !== "" && !EFFORTS.has(p.effort as string)) return "invalid";
const out: ProviderConfig & { apiKey?: string; clearApiKey?: boolean } = { kind: p.kind as ProviderConfig["kind"] };
if (str(p.model)) out.model = str(p.model);
if (str(p.baseUrl)) {
try { new URL(str(p.baseUrl)!); } catch { return "invalid"; }
out.baseUrl = str(p.baseUrl);
}
if (str(p.effort)) out.effort = str(p.effort) as ProviderConfig["effort"];
if (str(p.apiKey)) out.apiKey = str(p.apiKey);
if (p.clearApiKey === true) out.clearApiKey = true;
// Server scope: any endpoint, but OpenAI-compatible needs one. User scope: an empty base URL means the official endpoint.
if (out.kind === "openai" && (!out.model || (scope === "server" && !out.baseUrl))) return "invalid";
return out;
}
function parseMcp(v: unknown): McpServerConfig[] | "invalid" {
if (v === undefined || v === null) return [];
if (!Array.isArray(v)) return "invalid";
const out: McpServerConfig[] = [];
const ids = new Set<string>();
for (const raw of v) {
if (!raw || typeof raw !== "object") return "invalid";
const s = raw as Record<string, unknown>;
const id = str(s.id);
if (!id || !/^[a-zA-Z0-9_-]{1,40}$/.test(id) || ids.has(id)) return "invalid";
ids.add(id);
const transport = s.transport === "stdio" || s.transport === "http" ? s.transport : null;
if (!transport) return "invalid";
const cfg: McpServerConfig = { id, name: str(s.name) ?? id, transport };
if (transport === "stdio") {
if (!str(s.command)) return "invalid";
cfg.command = str(s.command);
if (Array.isArray(s.args)) cfg.args = s.args.map(String);
if (s.env && typeof s.env === "object") cfg.env = Object.fromEntries(Object.entries(s.env).map(([k, val]) => [k, String(val)]));
} else {
const url = str(s.url);
if (!url) return "invalid";
try { new URL(url); } catch { return "invalid"; }
cfg.url = url;
if (s.headers && typeof s.headers === "object") cfg.headers = Object.fromEntries(Object.entries(s.headers).map(([k, val]) => [k, String(val)]));
}
if (Array.isArray(s.courseIds)) cfg.courseIds = s.courseIds.map(String).filter(Boolean);
out.push(cfg);
}
return out;
}
function parseGenerate(b: Record<string, unknown>): GenerateRequest | null {
const courseId = str(b.courseId);
const topic = str(b.topic);
if (!courseId || !topic) return null;
if (!["document", "exercise", "questions"].includes(b.kind as string)) return null;
if (!["pure-snake", "systemic-design"].includes(b.style as string)) return null;
if (!Array.isArray(b.sources)) return null;
const sources: GenerateSource[] = [];
for (const s of b.sources) {
if (!s || typeof s !== "object") return null;
const x = s as Record<string, unknown>;
if (x.kind === "lesson" && str(x.lessonId)) sources.push({ kind: "lesson", lessonId: str(x.lessonId)! });
else if (x.kind === "file" && str(x.path)) sources.push({ kind: "file", path: str(x.path)! });
else if (x.kind === "cache" && str(x.path)) sources.push({ kind: "cache", path: str(x.path)! });
else if (x.kind === "text" && typeof x.text === "string" && x.text.trim()) sources.push({ kind: "text", label: str(x.label) ?? "Pasted text", text: x.text });
else return null;
}
return { courseId, topic: topic.slice(0, 200), kind: b.kind as GenerateRequest["kind"], style: b.style as GenerateRequest["style"], sources, instructions: typeof b.instructions === "string" ? b.instructions.slice(0, 8_000) : undefined };
}
const LISTABLE = new Set([".md", ".markdown", ".txt", ".js", ".mjs", ".ts", ".tsx", ".jsx", ".py", ".rs", ".scala", ".fs", ".go", ".java", ".sh", ".json", ".yaml", ".yml", ".sql", ".html", ".css", ".toml", ".vtt", ".srt"]);
async function listFiles(root: string, limit = 800, keep: (rel: string) => boolean = () => true): Promise<string[]> {
const out: string[] = [];
const walk = async (dir: string, depth: number) => {
if (depth > 8 || out.length >= limit) return;
let entries;
try { entries = await readdir(dir, { withFileTypes: true }); } catch { return; }
entries.sort((a, b) => a.name.localeCompare(b.name));
for (const e of entries) {
if (out.length >= limit) return;
if (e.name.startsWith(".") || e.name === "node_modules" || e.name === "target" || e.name === "dist") continue;
const abs = join(dir, e.name);
if (e.isDirectory()) await walk(abs, depth + 1);
else if (e.isFile() && LISTABLE.has(extname(e.name).toLowerCase())) {
const rel = relative(root, abs).split(sep).join("/");
if (keep(rel)) out.push(rel);
}
}
};
await walk(root, 0);
return out;
}
export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feature {
const settings = aiSettingsRepo(ctx.db, createSecretBox(ctx.config.secret));
const acl = aclRepo(ctx.db);
const notes = notesRepo(ctx.db);
const progress = progressRepo(ctx.db);
const sink = opts.draftSink ?? createFsDraftSink(ctx.config.dataDir);
const log = opts.log ?? ((m: string) => console.error(`[ai] ${m}`));
// AppConfig.aiAllowedBaseUrls (STC_AI_ALLOWED_BASE_URLS) plus the admin's list in the server AI settings.
const configOrigins = parseAllowedOrigins((ctx.config.aiAllowedBaseUrls ?? []).join(","));
const allowedOrigins = async () => [...new Set([...configOrigins, ...(await settings.getServer()).allowedBaseUrls])];
/**
* Providers for a user's own key never follow redirects, and their errors reach the user only as the HTTP
* status plus a fixed sentence (the full text is logged here). Server-scope providers are the admin's own.
*/
const makeProvider = (r: { source: "byok" | "server"; cfg: ProviderConfig; key: string | null }, userId: string): AiProvider => {
if (r.source === "server") return opts.providerFactory ? opts.providerFactory(r.cfg, r.key) : createProvider(r.cfg, r.key, opts.fetch);
const p = opts.providerFactory ? opts.providerFactory(r.cfg, r.key) : createProvider(r.cfg, r.key, noRedirectFetch(opts.fetch ?? fetch));
return sanitizeProviderErrors(p, (m) => log(`provider error for user ${userId} (own key): ${m}`));
};
/** BYOK first ("used only for their own requests"), then the server provider. */
const resolveProvider = async (userId: string | null): Promise<{ source: "byok" | "server"; cfg: ProviderConfig; key: string | null } | null> => {
if (userId) {
const u = await settings.getUser(userId);
if (u.byok) {
// Re-checked on every use: an origin the admin removed from the allowlist stops working at once.
const base = byokBaseUrl(u.byok.kind, u.byok.baseUrl, await allowedOrigins());
const key = base ? await settings.userApiKey(userId).catch(() => null) : null;
if (key && base) return { source: "byok", cfg: { ...u.byok, baseUrl: base }, key };
}
}
const s = await settings.getServer();
if (!s.provider) return null;
if (s.provider.kind === "anthropic" && !s.provider.hasApiKey) return null;
const key = s.provider.hasApiKey ? await settings.serverApiKey().catch(() => null) : null;
if (s.provider.kind === "anthropic" && !key) return null;
return { source: "server", cfg: s.provider, key };
};
const viewable = async (user: AuthUser | null, courseId: string): Promise<LoadedCourse | null> => {
const c = ctx.courses.get(courseId);
if (!c || c.status !== "ok") return null;
const ok = canView(user, c, await acl.listForCourse(c.id), user ? await acl.isAuthor(c.id, user.id) : false);
return ok ? c : null;
};
/**
* Generation and the studio file listing: the user must hold an authoring right (owner, the `author` role, or a
* `course_authors` row) AND see the course under core's `canView`, the one visibility rule (R14; it lets the
* global `author` role view every course).
*/
const canAuthor = async (user: AuthUser, course: LoadedCourse) => {
const isAuthorRow = await acl.isAuthor(course.id, user.id);
const mayAuthor = user.roles.includes("owner") || user.roles.includes("author") || isAuthorRow;
return mayAuthor && canView(user, course, await acl.listForCourse(course.id), isAuthorRow);
};
const conversationOf = async (id: string, userId: string) => {
const rows = await ctx.db.sql`SELECT id, user_id, course_id, title, created_at FROM ai_conversations WHERE id = ${id} AND user_id = ${userId}`;
return rows[0] as { id: string; user_id: string; course_id: string; title: string | null; created_at: string } | undefined;
};
// Roles: "user", "assistant", and "sources" (JSON citations for the assistant message before it).
const addMessage = async (conversationId: string, role: "user" | "assistant" | "sources", content: string) => {
await ctx.db.sql`INSERT INTO ai_messages (id, conversation_id, role, content, created_at) VALUES (${Bun.randomUUIDv7()}, ${conversationId}, ${role}, ${content}, ${new Date().toISOString()})`;
};
const messagesOf = async (conversationId: string) =>
(await ctx.db.sql`SELECT id, role, content, created_at FROM ai_messages WHERE conversation_id = ${conversationId}`)
.map((r: { id: string; role: string; content: string; created_at: string }) => ({ id: r.id, role: r.role, content: r.content, createdAt: r.created_at }))
.sort((a: { id: string }, b: { id: string }) => (a.id < b.id ? -1 : 1)) as { id: string; role: "user" | "assistant" | "sources"; content: string; createdAt: string }[];
/** Chat turns with each assistant message's stored citations attached. */
const turnsOf = async (conversationId: string) => {
const out: { id: string; role: "user" | "assistant"; content: string; createdAt: string; citations?: unknown[] }[] = [];
for (const m of await messagesOf(conversationId)) {
if (m.role === "sources") {
const last = out[out.length - 1];
if (last?.role === "assistant") { try { last.citations = JSON.parse(m.content); } catch {} }
} else out.push({ ...m, role: m.role });
}
return out;
};
const signedIn = (rc: RequestContext) => (rc.user ? null : err(401, "Sign in to use the assistant."));
const ownerOnly = (rc: RequestContext) => (!rc.user ? err(401, "Sign in first.") : rc.user.roles.includes("owner") ? null : err(403, "Only the owner can change AI settings."));
const chat = async (req: Request, rc: RequestContext): Promise<Response> => {
const deny = signedIn(rc);
if (deny) return deny;
const user = rc.user!;
const course = await viewable(user, rc.params.courseId ?? "");
if (!course) return err(404, "Course not found.");
const b = await body(req);
const message = typeof b?.message === "string" ? b.message.trim() : "";
if (!message) return err(400, "Write a message.");
if (message.length > MAX_MESSAGE) return err(400, "That message is too long.");
const resolved = await resolveProvider(user.id);
if (!resolved) return err(409, "AI is not set up. An admin can configure a server provider, or you can add your own key under Account, AI.");
let provider: AiProvider;
try { provider = makeProvider(resolved, user.id); } catch (e) { return err(409, (e as Error).message); }
let conversationId = str(b?.conversationId);
if (conversationId) {
const conv = await conversationOf(conversationId, user.id);
if (!conv || conv.course_id !== course.id) return err(404, "Conversation not found.");
} else {
conversationId = Bun.randomUUIDv7();
await ctx.db.sql`INSERT INTO ai_conversations (id, user_id, course_id, title, created_at) VALUES (${conversationId}, ${user.id}, ${course.id}, ${message.slice(0, 80)}, ${new Date().toISOString()})`;
}
const history = (await turnsOf(conversationId)).slice(-HISTORY_TURNS);
await addMessage(conversationId, "user", message);
const lessonId = str(b?.lessonId);
const lesson = lessonId ? course.lessonIndex.get(lessonId)?.lesson : undefined;
const index = await getCourseIndex(course, ctx.config.dataDir);
const userNotes = await notes.list(user.id, course.id);
const sources = index.search(lesson ? `${message} ${lesson.title}` : message, 6, noteChunks(course, userNotes));
const mentorText = await readMentorText(ctx.courses, course);
const mentorOn = !(await settings.isMentorOff(user.id, course.id));
const convId = conversationId;
const stream = new ReadableStream<Uint8Array>({
async start(controller) {
const enc = new TextEncoder();
const send = (ev: unknown) => { try { controller.enqueue(enc.encode(JSON.stringify(ev) + "\n")); } catch {} };
send({ type: "meta", conversationId: convId, mentorOn, mentorAvailable: !!mentorText, source: resolved.source, citations: citationsFor(sources) });
let tools: ToolSet | null = null;
try {
const course_ = courseTools({ repo: ctx.courses, course, index, listNotes: (lid) => notes.list(user.id, course.id, lid), getProgress: () => progress.get(user.id, course.id) });
const servers = mcpServersFor(course.id, (await settings.getServer()).mcpServers);
const mcp = servers.length ? await mcpTools(servers, { fetch: opts.fetch, timeoutMs: opts.mcpTimeoutMs }) : null;
for (const w of mcp?.warnings ?? []) send({ type: "notice", message: w });
tools = combineTools(mcp ? [course_, mcp] : [course_]);
const messages: ChatMessage[] = [
{ role: "system", content: buildSystemPrompt({ course, mentorText, mentorOn, sources, lesson: lesson ? { id: lesson.id, title: lesson.title } : null, extraTools: mcp?.names }) },
...history.map((m) => ({ role: m.role, content: m.content }) as ChatMessage),
{ role: "user", content: message },
];
let answer = "";
let cited: unknown[] = [];
for await (const ev of runAssistant({ provider, messages, tools, signal: req.signal })) {
if (ev.type === "done") {
answer = ev.text;
cited = citationsFor([...sources, ...tools.sources]);
send({ type: "done", text: ev.text, citations: cited });
} else send(ev);
}
if (answer) {
await addMessage(convId, "assistant", answer);
await addMessage(convId, "sources", JSON.stringify(cited));
}
} catch (e) {
const m = (e as Error).message;
if (resolved.source === "byok") log(`chat error for user ${user.id} (own key): ${m}`);
send({ type: "error", message: resolved.source === "byok" ? publicProviderError(m) : m });
} finally {
await tools?.close?.().catch(() => {});
try { controller.close(); } catch {}
}
},
});
return new Response(stream, { headers: { "content-type": "application/x-ndjson; charset=utf-8", "cache-control": "no-store" } });
};
const generateJob = async (payload: unknown, jc: JobContext) => {
const p = payload as { req: GenerateRequest; userId: string };
const resolved = await resolveProvider(p.userId);
if (!resolved) throw new Error("AI is not set up for this user.");
const provider = makeProvider(resolved, p.userId);
return generateDraft({ provider, repo: ctx.courses, req: p.req, sink, authorId: p.userId, dataDir: ctx.config.dataDir, signal: jc.signal, onProgress: (f, m) => jc.reportProgress(f, m) });
};
return {
name: "ai",
jobs: { "ai.generate": generateJob },
routes: {
"/api/ai/status": {
GET: async (_req, rc) => {
const resolved = rc.user ? await resolveProvider(rc.user.id) : null;
const out: Record<string, unknown> = { enabled: !!resolved, source: resolved?.source ?? null, signedIn: !!rc.user };
const courseId = rc.url.searchParams.get("courseId");
if (courseId) {
const course = await viewable(rc.user, courseId);
if (course) {
out.course = { id: course.id, title: course.manifest?.title ?? course.id };
out.mentor = { available: !!course.manifest?.mentor, on: rc.user ? !(await settings.isMentorOff(rc.user.id, course.id)) : true };
}
}
return json(out);
},
},
"/api/ai/admin/config": {
GET: async (_req, rc) => ownerOnly(rc) ?? json(await settings.getServer()),
PUT: async (req, rc) => {
const deny = ownerOnly(rc);
if (deny) return deny;
const b = await body(req);
if (!b) return err(400, "Invalid JSON.");
const provider = parseProvider(b.provider, "server");
const mcpServers = parseMcp(b.mcpServers);
if (provider === "invalid") return err(400, "Invalid provider. OpenAI-compatible endpoints need a base URL and a model.");
if (mcpServers === "invalid") return err(400, "Invalid MCP server list.");
let allowedBaseUrls: string[] | undefined;
if (b.allowedBaseUrls !== undefined) {
const list = Array.isArray(b.allowedBaseUrls) ? b.allowedBaseUrls.map(String).map((x) => x.trim()).filter(Boolean) : null;
if (!list) return err(400, "allowedBaseUrls must be a list of origins.");
allowedBaseUrls = parseAllowedOrigins(list);
if (allowedBaseUrls.length !== new Set(list.map((x) => x.replace(/\/+$/, ""))).size) return err(400, "Each allowed base URL must be an origin such as http://ollama.internal:11434.");
}
await settings.saveServer({ provider, mcpServers, allowedBaseUrls });
return json(await settings.getServer());
},
},
"/api/ai/admin/mcp/test": {
POST: async (req, rc) => {
const deny = ownerOnly(rc);
if (deny) return deny;
const b = await body(req);
const parsed = parseMcp(b ? [b] : null);
if (parsed === "invalid" || !parsed.length) return err(400, "Invalid MCP server.");
try {
const c = await connectMcp(parsed[0]!, { fetch: opts.fetch, timeoutMs: opts.mcpTimeoutMs ?? 15_000 });
try { return json({ ok: true, tools: (await c.tools()).map((t) => ({ name: t.name, description: t.description })) }); } finally { await c.close(); }
} catch (e) {
return json({ ok: false, error: (e as Error).message });
}
},
},
"/api/ai/account": {
GET: async (_req, rc) => {
if (!rc.user) return err(401, "Sign in first.");
const u = await settings.getUser(rc.user.id);
const s = await settings.getServer();
return json({ byok: u.byok, serverProvider: s.provider ? { kind: s.provider.kind, model: s.provider.model ?? null } : null });
},
},
"/api/ai/account/key": {
PUT: async (req, rc) => {
if (!rc.user) return err(401, "Sign in first.");
const b = await body(req);
const p = b ? parseProvider(b, "user") : "invalid";
if (!p || p === "invalid" || !p.apiKey) return err(400, "Choose a provider and paste your key. OpenAI-compatible endpoints also need a model.");
const base = byokBaseUrl(p.kind, p.baseUrl, await allowedOrigins());
if (!base) return err(400, "That base URL is not allowed for your own key. Use the official endpoint, or ask an admin to allow it.");
await settings.saveUserKey(rc.user.id, { ...p, baseUrl: base === PRESET_BASE_URLS[p.kind] ? undefined : base, apiKey: p.apiKey });
return json((await settings.getUser(rc.user.id)).byok);
},
DELETE: async (_req, rc) => {
if (!rc.user) return err(401, "Sign in first.");
await settings.removeUserKey(rc.user.id);
return json({ ok: true });
},
},
"/api/ai/courses/:courseId/mentor": {
PUT: async (req, rc) => {
const deny = signedIn(rc);
if (deny) return deny;
const course = await viewable(rc.user, rc.params.courseId ?? "");
if (!course) return err(404, "Course not found.");
const b = await body(req);
if (typeof b?.off !== "boolean") return err(400, "Send { off: true | false }.");
await settings.setMentorOff(rc.user!.id, course.id, b.off);
return json({ mentor: { available: !!course.manifest?.mentor, on: !b.off } });
},
},
"/api/ai/courses/:courseId/chat": { POST: chat },
"/api/ai/courses/:courseId/conversations": {
GET: async (_req, rc) => {
const deny = signedIn(rc);
if (deny) return deny;
const rows = await ctx.db.sql`SELECT id, title, created_at FROM ai_conversations WHERE user_id = ${rc.user!.id} AND course_id = ${rc.params.courseId ?? ""}`;
return json(rows.map((r: { id: string; title: string | null; created_at: string }) => ({ id: r.id, title: r.title, createdAt: r.created_at })).sort((a: { createdAt: string }, b: { createdAt: string }) => (a.createdAt < b.createdAt ? 1 : -1)));
},
},
"/api/ai/conversations/:id": {
GET: async (_req, rc) => {
const deny = signedIn(rc);
if (deny) return deny;
const conv = await conversationOf(rc.params.id ?? "", rc.user!.id);
if (!conv) return err(404, "Conversation not found.");
return json({ id: conv.id, courseId: conv.course_id, title: conv.title, messages: await turnsOf(conv.id) });
},
DELETE: async (_req, rc) => {
const deny = signedIn(rc);
if (deny) return deny;
const conv = await conversationOf(rc.params.id ?? "", rc.user!.id);
if (!conv) return err(404, "Conversation not found.");
await ctx.db.sql.begin(async (tx) => {
await tx`DELETE FROM ai_messages WHERE conversation_id = ${conv.id}`;
await tx`DELETE FROM ai_conversations WHERE id = ${conv.id}`;
});
return json({ ok: true });
},
},
"/api/ai/studio/courses": {
GET: async (_req, rc) => {
if (!rc.user) return err(401, "Sign in first.");
const out = [];
for (const c of ctx.courses.list()) {
if (c.status === "ok" && (await canAuthor(rc.user, c))) out.push({ id: c.id, title: c.manifest?.title ?? c.id, tracks: c.manifest?.tracks ?? [] });
}
return json(out);
},
},
"/api/ai/studio/sources": {
GET: async (_req, rc) => {
if (!rc.user) return err(401, "Sign in first.");
const courseId = rc.url.searchParams.get("courseId") ?? "";
const course = ctx.courses.get(courseId);
if (!course || course.status !== "ok") return err(404, "Course not found.");
if (!(await canAuthor(rc.user, course))) return err(403, "Only authors can generate drafts.");
const lessons = (course.manifest?.modules ?? []).flatMap((m) => m.lessons.map((l) => ({ id: l.id, title: l.title, type: l.type, module: m.title, hasText: l.type !== "video" || !!l.transcript })));
const files = await listFiles(course.root, 800); // authors may use any course file, solutions included
const cache = await listFiles(join(ctx.config.dataDir, "cache", course.id), 400);
return json({ lessons, files, cache });
},
},
"/api/ai/studio/generate": {
POST: async (req, rc) => {
if (!rc.user) return err(401, "Sign in first.");
const b = await body(req);
const g = b ? parseGenerate(b) : null;
if (!g) return err(400, "Choose a course, a kind (document, exercise or questions), a style and a topic.");
const course = ctx.courses.get(g.courseId);
if (!course || course.status !== "ok") return err(404, "Course not found.");
if (!(await canAuthor(rc.user, course))) return err(403, "Only authors can generate drafts.");
if (!(await resolveProvider(rc.user.id))) return err(409, "AI is not set up.");
const jobId = await ctx.jobs.enqueue("ai.generate", { req: g, userId: rc.user.id });
return json({ jobId }, 202);
},
},
"/api/ai/studio/jobs/:id": {
GET: async (_req, rc) => {
if (!rc.user) return err(401, "Sign in first.");
const job = await ctx.jobs.get(rc.params.id ?? "");
const owner = (job?.payload as { userId?: string } | null)?.userId;
if (!job || job.kind !== "ai.generate" || (owner !== rc.user.id && !rc.user.roles.includes("owner"))) return err(404, "Job not found.");
return json({ id: job.id, status: job.status, progress: job.progress, message: job.message, result: job.result, error: job.error });
},
},
},
};
}
packages/feature-ai/src/generator/draft-sink.ts
+60
View File
@@ -0,0 +1,60 @@
import { mkdir } from "node:fs/promises";
import { dirname, join, resolve, sep } from "node:path";
export interface DraftFile { path: string; content: string }
/**
* What a sink reports back. The studio's writer returns `{ branch, commit }` (a commit on `draft/<authorId>`);
* the filesystem fallback returns `{ location }` (a folder under `<dataDir>/drafts`).
*/
export interface DraftWriteResult { location?: string; branch?: string; commit?: string }
/**
* Where generated files go. Same method as the studio's `DraftWriter` (`createDraftWriter(ctx, { identity? })` in
* @stc/feature-studio), so that writer is a `DraftSink` as is, with no adapter: Phase 2 passes it in as
* `aiFeature(ctx, { draftSink: createDraftWriter(ctx) })`. Drafts are never published by the AI package.
*/
export interface DraftSink {
writeDraftFiles(courseId: string, authorId: string, files: DraftFile[], message: string): Promise<DraftWriteResult>;
}
/** Human-readable place of a draft: the folder, or `draft/<author> @ <short commit>`. */
export function draftLocation(r: DraftWriteResult): string {
if (r.location) return r.location;
if (r.branch) return r.commit ? `${r.branch} @ ${r.commit.slice(0, 7)}` : r.branch;
return "draft";
}
export function safeRelPath(p: string): string | null {
const clean = p.trim().replace(/\\/g, "/").replace(/^\.\//, "");
if (!clean || clean.startsWith("/") || /^[a-zA-Z]:/.test(clean)) return null;
const parts = clean.split("/");
if (parts.some((s) => s === ".." || s === "" || s === "." || s === ".git")) return null;
return clean;
}
/** Filesystem fallback: `<dataDir>/drafts/<courseId>/<runId>/…` plus DRAFT.json. Outside every course folder. */
export function createFsDraftSink(dataDir: string): DraftSink {
return {
async writeDraftFiles(courseId, authorId, files, message) {
if (!safeRelPath(courseId) || courseId.includes("/")) throw new Error("Invalid course id");
const runId = `${new Date().toISOString().replace(/[:.]/g, "-")}-${Bun.randomUUIDv7().slice(-8)}`;
const location = resolve(dataDir, "drafts", courseId, runId);
const paths: string[] = [];
for (const f of files) {
const rel = safeRelPath(f.path);
if (!rel) throw new Error(`Unsafe draft path: ${f.path}`);
const abs = resolve(location, rel);
if (!abs.startsWith(location + sep)) throw new Error(`Unsafe draft path: ${f.path}`);
await mkdir(dirname(abs), { recursive: true });
await Bun.write(abs, f.content);
paths.push(rel);
}
await Bun.write(
join(location, "DRAFT.json"),
JSON.stringify({ courseId, authorId, message, createdAt: new Date().toISOString(), published: false, files: paths }, null, 2),
);
return { location };
},
};
}
packages/feature-ai/src/generator/generate.ts
+176
View File
@@ -0,0 +1,176 @@
import { stat } from "node:fs/promises";
import { join } from "node:path";
import type { AiProvider } from "@stc/contracts";
import { resolveInside, type CourseRepository, type LoadedCourse } from "@stc/course-format";
import { draftLocation, safeRelPath, type DraftFile, type DraftSink } from "./draft-sink";
import { OUTPUT_FORMAT, REQUIRED, VOICE, styleTemplate, type Kind, type Style } from "./templates";
export type GenerateSource =
| { kind: "lesson"; lessonId: string }
| { kind: "file"; path: string }
| { kind: "cache"; path: string }
| { kind: "text"; label: string; text: string };
export interface GenerateRequest {
courseId: string;
kind: Kind;
style: Style;
topic: string;
sources: GenerateSource[];
instructions?: string;
}
/** `location` is a folder (filesystem sink) or `draft/<author> @ <commit>` (studio writer). */
export interface GenerateResult { location: string; branch?: string; commit?: string; paths: string[]; warnings: string[] }
const MAX_MATERIAL = 400_000;
export function slugify(s: string): string {
return s.normalize("NFKD").replace(/[̀-ͯ]/g, "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60) || "draft";
}
export function targetPaths(req: GenerateRequest): string[] {
const slug = slugify(req.topic);
if (req.kind === "exercise") return [`exercises/${slug}/README.md`, `exercises/${slug}/starter.js`, `exercises/${slug}/tests.js`, `exercises/${slug}/solution.js`, `exercises/${slug}/fixtures/${slug}.json`, `exercises/${slug}/manifest-snippet.yaml`];
if (req.kind === "questions") return [`questions/${slug}.md`];
return req.style === "systemic-design" ? [`course/${slug}/lesson.md`, `course/${slug}/scenarios.md`] : [`course/${slug}.md`];
}
/** Reads the chosen inputs. Authors may use any course file, including solutions; cache text comes from <dataDir>/cache/<courseId>/. */
export async function gatherMaterial(repo: CourseRepository, course: LoadedCourse, sources: GenerateSource[], dataDir: string): Promise<{ text: string; warnings: string[] }> {
const parts: string[] = [];
const warnings: string[] = [];
let total = 0;
const push = (label: string, body: string) => {
if (total >= MAX_MATERIAL) { warnings.push(`Left out "${label}": the material is already ${MAX_MATERIAL} characters.`); return; }
let text = body;
if (total + text.length > MAX_MATERIAL) {
text = text.slice(0, MAX_MATERIAL - total);
warnings.push(`"${label}" was shortened to fit the material limit.`);
}
total += text.length;
parts.push(`<source label="${label.replace(/"/g, "'")}">\n${text}\n</source>`);
};
const readText = async (abs: string | null, label: string) => {
const info = abs ? await stat(abs).catch(() => null) : null;
if (!abs || !info?.isFile()) { warnings.push(`Source not found: ${label}`); return; }
const text = await Bun.file(abs).text();
if (text.includes("\u0000")) { warnings.push(`Skipped binary file: ${label}`); return; }
push(label, text);
};
for (const s of sources) {
if (s.kind === "text") push(s.label || "Pasted text", s.text);
else if (s.kind === "lesson") {
const e = course.lessonIndex.get(s.lessonId);
const md = e ? await repo.readLessonMarkdown(course.id, s.lessonId) : null;
if (!e || md === null) warnings.push(`Lesson has no readable text: ${s.lessonId}`);
else push(`${e.lesson.type === "video" ? e.lesson.transcript : e.lesson.file} (lesson ${s.lessonId}: ${e.lesson.title})`, md);
} else if (s.kind === "file") await readText(repo.resolvePath(course.id, s.path), s.path);
else await readText(resolveInside(join(dataDir, "cache", course.id), s.path), `cache/${s.path}`);
}
return { text: parts.join("\n\n"), warnings };
}
function outline(course: LoadedCourse): string {
return (course.manifest?.modules ?? []).map((m, i) => `${i + 1}. ${m.title} (${m.lessons.length} lessons)`).join("\n");
}
export function buildGeneratorPrompt(req: GenerateRequest, material: string, course: LoadedCourse): { system: string; user: string } {
const what = req.kind === "document" ? "an explanatory lesson document" : req.kind === "exercise" ? "a small exercise with tests and hint tiers" : "check-your-understanding questions";
const system = [
`You draft course material for STCourses authors. You write ${what} for the course "${course.manifest?.title ?? course.id}", in the house style below, from the source material the author selected.`,
VOICE,
styleTemplate(req.kind, req.style),
OUTPUT_FORMAT,
].join("\n\n");
const tracks = course.manifest?.tracks?.map((t) => `${t.label} (${t.id})`).join(", ") || "one track (JavaScript)";
const user = [
`Course: ${course.manifest?.title ?? course.id}`,
`Language tracks: ${tracks}`,
`Existing modules:\n${outline(course) || "(none yet)"}`,
`Topic: ${req.topic}`,
req.instructions?.trim() ? `Author's instructions:\n${req.instructions.trim()}` : "",
`Write these files:\n${targetPaths(req).map((p) => `- ${p}`).join("\n")}`,
`Source material (transcripts, text, slide text and code chosen by the author):\n\n${material || "(none: work from the topic and instructions, and mark facts TODO(author): check)"}`,
].filter(Boolean).join("\n\n");
return { system, user };
}
/** Reads `=== FILE: path ===` … `=== END FILE ===` blocks. Unsafe paths are dropped with a warning. */
export function parseFiles(text: string): { files: DraftFile[]; warnings: string[] } {
const files: DraftFile[] = [];
const warnings: string[] = [];
const lines = text.split(/\r?\n/);
let cur: { path: string; lines: string[] } | null = null;
const close = () => {
if (!cur) return;
const rel = safeRelPath(cur.path);
if (!rel) warnings.push(`Dropped a file with an unsafe path: ${cur.path}`);
else files.push({ path: rel, content: cur.lines.join("\n").replace(/\n*$/, "\n") });
cur = null;
};
for (const line of lines) {
const open = /^===\s*FILE:\s*(.+?)\s*===\s*$/.exec(line);
if (open) { close(); cur = { path: open[1]!.replace(/^`|`$/g, ""), lines: [] }; continue; }
if (/^===\s*END FILE\s*===\s*$/.test(line)) { close(); continue; }
cur?.lines.push(line);
}
if (cur) { warnings.push(`The last file (${(cur as { path: string }).path}) was not closed; it may be cut off.`); close(); }
if (!files.length) warnings.push("The model returned no files.");
return { files, warnings };
}
export function checkHouseStyle(req: GenerateRequest, files: DraftFile[]): string[] {
const warnings: string[] = [];
const md = files.filter((f) => /\.(md|markdown)$/i.test(f.path)).map((f) => f.content).join("\n");
for (const s of REQUIRED[req.kind][req.style]) if (!md.includes(s)) warnings.push(`House style: missing "${s}".`);
if (req.kind === "exercise") {
for (const name of ["starter.js", "tests.js", "solution.js"]) if (!files.some((f) => f.path.endsWith(`/${name}`))) warnings.push(`Exercise: missing ${name}.`);
const tests = files.find((f) => f.path.endsWith("/tests.js"));
if (tests && !/export\s+const\s+tests\b/.test(tests.content)) warnings.push("Exercise: tests.js does not export `tests`.");
}
return warnings;
}
const marker = (req: GenerateRequest) =>
`<!-- Draft generated by STCourses AI on ${new Date().toISOString().slice(0, 10)} (${req.kind}, ${req.style} style) from author-selected sources. An author must review it before publishing. -->\n\n`;
/** Runs the model once and writes the files it returns through the DraftSink. Never publishes. */
export async function generateDraft(o: {
provider: AiProvider;
repo: CourseRepository;
req: GenerateRequest;
sink: DraftSink;
authorId: string;
dataDir: string;
signal?: AbortSignal;
onProgress?: (fraction: number, message: string) => Promise<void> | void;
}): Promise<GenerateResult> {
const course = o.repo.get(o.req.courseId);
if (!course || course.status !== "ok") throw new Error(`Course not available: ${o.req.courseId}`);
await o.onProgress?.(0.05, "Reading sources");
const material = await gatherMaterial(o.repo, course, o.req.sources, o.dataDir);
const prompt = buildGeneratorPrompt(o.req, material.text, course);
await o.onProgress?.(0.15, "Drafting");
let text = "";
let chars = 0;
for await (const ev of o.provider.chat([{ role: "system", content: prompt.system }, { role: "user", content: prompt.user }], { signal: o.signal })) {
if (ev.text) {
text += ev.text;
chars += ev.text.length;
if (chars > 4000) { chars = 0; await o.onProgress?.(Math.min(0.85, 0.15 + text.length / 60_000), "Drafting"); }
}
}
const parsed = parseFiles(text);
if (!parsed.files.length) throw new Error(parsed.warnings.join(" ") || "The model returned no files.");
const files = parsed.files.map((f) => (/\.(md|markdown)$/i.test(f.path) ? { ...f, content: marker(o.req) + f.content } : f));
await o.onProgress?.(0.9, "Writing drafts");
const written = await o.sink.writeDraftFiles(course.id, o.authorId, files, `AI draft: ${o.req.kind} "${o.req.topic}" (${o.req.style} style)`);
return {
location: draftLocation(written),
...(written.branch ? { branch: written.branch } : {}),
...(written.commit ? { commit: written.commit } : {}),
paths: files.map((f) => f.path),
warnings: [...material.warnings, ...parsed.warnings, ...checkHouseStyle(o.req, parsed.files)],
};
}
packages/feature-ai/src/generator/templates.ts
+209
View File
@@ -0,0 +1,209 @@
// House-style prompt templates for the studio generator, distilled from the two reference courses:
// - Pure Snake (functional-programming-course): module shape of course/03-pure-game-rules.md and MENTOR.md;
// - Systemic Design (systemic-design-course): the beats of course/01-handles/lesson.md and mentor/SKILL.md.
export type Style = "pure-snake" | "systemic-design";
export type Kind = "document" | "exercise" | "questions";
export const OUTPUT_FORMAT = `## Output format
Reply with one or more files and nothing that matters outside them. Write each file exactly like this:
=== FILE: <course-relative path> ===
<the whole file content>
=== END FILE ===
Use only relative paths inside the course (no leading slash, no ".."). Everything you write is a draft for an author to review; it is never published automatically.`;
export const VOICE = `## Voice and rules (both house styles)
- Write for a learner working alone at home with general programming experience. Plain, direct sentences; second person ("you"). No hype, no emoji, no headings in all caps.
- Teach by making the learner think first: a concrete situation or puzzle before the explanation, and the explanation folded so they can try first.
- Every fact about a tool, library or real system must come from the source material you are given. If the material does not say it, leave it out or mark it "TODO(author): check".
- Code in fenced blocks with a language tag. Keep examples small. Exercise solutions never appear in learner-facing text; near-solution hints are skeletons with \`___\` holes.
- When the course has several language tracks, give each track's signature and use one shared fixture file for test cases, so every track tests the same cases with the same names.`;
export const PURE_SNAKE_DOCUMENT = `## House style: Pure Snake module
Follow this module shape exactly (headings verbatim, in this order):
# Module N — <Title>
> **Time:** ~<estimate> · **You'll edit:** <files per track>
> **Tests:** <test command per track>
> **Prerequisites:** <earlier modules>
## Why this module
Two or three paragraphs: how this is usually done (stateful, the familiar way), what goes wrong, and what this module does instead. End by announcing the puzzles.
## Learning objectives
"After this module you can:" then 4-6 bullets, each starting with a bold verb (**implement**, **explain**, **reason**, **predict**, **read**).
## Concepts
### <one H3 per concept>
Explain with a tiny example on a *different* record than the exercise (say so: "(illustration on a different record)"), shown in each track.
Include at least one bridge:
> **From everyday programming:** <how the learner already meets this idea: loops, setters, callbacks, SQL, Git, unit tests, mocks>
### Puzzle 1: <a question the learner can answer on paper>
A concrete small case (a 5 x 5 board, a list of three items). Then **Before reading on, decide:** <the question>.
<details><summary>Think about it first, then open the explanation</summary>
The reasoning, the answer, and the rule it reveals. Name the fixture cases that check it.
</details>
(One or two puzzles. Each one targets a subtlety implementations commonly get wrong.)
## Exercises
### Exercise N.1 — \`functionName\`
The signature per track and a one-line example in a comment. "Fixture: \`fixtures/mNN-<function>.json\` (<case names>)."
(2-4 exercises, each small and testable.)
### Run
The test command per track.
## Hints
For EVERY exercise, three folds in this order, labelled exactly:
<details><summary>Exercise N.1 (functionName) — nudge</summary>
One question or pointer back to an earlier idea.
</details>
<details><summary>Exercise N.1 (functionName) — stronger hint</summary>
The approach in words, no code.
</details>
<details><summary>Exercise N.1 (functionName) — near-solution (try the others first)</summary>
A skeleton per track with \`___\` holes. Never the full answer.
</details>
## Check your understanding
6-8 numbered questions. Include: at least one "Predict:" question with concrete input, at least one "What would break if…", one "Explain to a friend, with a drawing…", and one design question ("Sketch a different model that…").
## In production
2-3 bullets linking the idea to real systems, each with a source from the material or marked TODO(author).
## Further reading
3-5 bullets: "[Title](url) — why it is worth reading." Only links present in the material; otherwise "TODO(author): add reading".`;
export const SYSTEMIC_DESIGN_DOCUMENT = `## House style: Systemic Design module
Every module follows the same beats, as H2 headings verbatim and in this order: Observe, Predict, Spec, Build, Name it, Expansion Drop, Further reading.
# Module N: <Title>
One paragraph: what the engine learns in this module and why later modules stand on it.
## Observe
A concrete situation from a real game or system, with the exact rule text quoted and its source cited in a link. Distinguish what the source says from the course's own reading ("That example is the course's reading of the quoted rule"). End with the tension the module resolves.
## Predict
"Write your answers down before you open the discussion, for example in \`notes/NN.md\` in your workspace."
2-3 numbered questions that make the learner commit to a design choice.
<details><summary>Discussion</summary>
Answer each question by number. Compare the tempting options and say what goes wrong with each, then name the answer.
</details>
## Spec
The contract: harness functions with signatures in each track, data formats, and how tests are named. Point to \`scenarios.md\`: each scenario heading is a test name, identical across tracks, with Setup / Action / Assertions bullets.
## Build
Where code goes (generic machinery in the engine, named content in content files; the engine must not name specific items) and what not to build yet.
<details><summary>Hint 1 — nudge</summary>
A question about who owns the data or when something happens.
</details>
<details><summary>Hint 2 — concept</summary>
The design in bolded terms (**Id**, **Entity record**, …), still no code.
</details>
<details><summary>Hint 3 — pseudo-code</summary>
\`\`\`text
language-neutral pseudo-code, never one track's real code
\`\`\`
</details>
## Name it
Name the pattern, then show where it appears in real engines or systems with links from the material, and the one idea to keep.
## Expansion Drop
New content arrives that the design must absorb without touching the engine: numbered steps (commit, copy the expansion test, implement in harness/content only, run the expansion check). State the test name and what it checks.
<details><summary>Discussion: what this drop measures</summary>
What a good design makes trivial here, and what a coupled design would have had to change.
</details>
## Further reading
Bullets with links from the material, each with one line on why.
Also write \`scenarios.md\` next to the lesson: "# Module N scenarios: <Title>", one sentence on where the tests live, then one "### test_NN_<name>" heading per scenario with Setup / Action / Assertions bullets.`;
export const EXERCISE = (style: Style) => `## House style: exercise
Write one small exercise that runs in the STCourses browser runner, plus its hint tiers.
Files (all under the exercise folder you are given):
- \`README.md\`: the task in the course's voice: one paragraph of context, the signature, two worked examples (input → output), and what "done" means. Then the hints as three folds per exercise, labelled ${style === "systemic-design" ? '"Hint 1 — nudge", "Hint 2 — concept", "Hint 3 — pseudo-code"' : '"Exercise N.M (name) — nudge", "… — stronger hint", "… — near-solution (try the others first)"'}. The near-solution tier is a skeleton with \`___\` holes (stronger hint: the approach in words; nudge: one question). The tiers in plain words are always nudge, stronger hint, near-solution.
- \`starter.js\`: an ES module exporting the function(s) with the right signature and a body that returns a wrong but type-correct value (or throws "TODO"). A doc comment states the contract.
- \`tests.js\`: an ES module \`export const tests = [ { name: "…", run(mod, assert) { assert.equal(mod.fn(…), …) } }, … ]\`. Use only \`assert.equal\`, \`assert.deepEqual\` and \`assert.ok\`. 5-8 tests including edge cases; test names read like fixture case names (\`sum-of-empty-list-is-zero\`). No imports: the runner loads this one module.
- \`solution.js\`: the reference solution (authors only; learners never see it).
- \`fixtures/<slug>.json\`: the same test cases as data (\`[{ "name", "input", "expected" }]\`), shared across tracks so other language tracks test identical cases.
- \`manifest-snippet.yaml\`: the lesson entry an author would paste into the manifest (type: exercise, browser: { language: js, starter, tests, solution }, hints: { file, match }).`;
export const QUESTIONS = `## House style: knowledge check
Write a "## Check your understanding" section as a numbered list of 6-8 questions about the material, in the reference courses' style:
- at least one "Predict:" question with concrete input the learner works out on paper;
- at least one "What would break if…" question;
- one "Explain to a friend, with a drawing…" question;
- one design question ("Sketch a different model that…");
- the rest test the main idea, not trivia. No multiple choice.
After the list, add one fold per question for authors:
<details><summary>Answer notes for question N (authors)</summary>
What a good answer contains and the misconception it probes.
</details>`;
export function styleTemplate(kind: Kind, style: Style): string {
if (kind === "exercise") return EXERCISE(style);
if (kind === "questions") return QUESTIONS;
return style === "systemic-design" ? SYSTEMIC_DESIGN_DOCUMENT : PURE_SNAKE_DOCUMENT;
}
/** Headings and labels the house-style check looks for. */
export const REQUIRED: Record<Kind, Record<Style, string[]>> = {
document: {
"pure-snake": ["## Why this module", "## Learning objectives", "## Concepts", "Think about it first", "## Exercises", "## Hints", "— nudge", "— stronger hint", "— near-solution", "## Check your understanding", "## Further reading"],
"systemic-design": ["## Observe", "## Predict", "## Spec", "## Build", "Hint 1 — nudge", "Hint 2 — concept", "Hint 3 — pseudo-code", "## Name it", "## Expansion Drop", "## Further reading"],
},
exercise: {
"pure-snake": ["nudge", "stronger hint", "near-solution"],
"systemic-design": ["Hint 1 — nudge", "Hint 2 — concept", "Hint 3 — pseudo-code"],
},
questions: {
"pure-snake": ["## Check your understanding"],
"systemic-design": ["## Check your understanding"],
},
};
packages/feature-ai/src/index.ts
+17
View File
@@ -0,0 +1,17 @@
export { createSecretBox, type SecretBox } from "./crypto";
/** Migration folders for this package. Empty: the AI package uses the ai_* tables from 0001_init.sql. */
export const migrationsDirs: string[] = [];
export * from "./settings";
export * from "./providers";
export * from "./retrieval/course-index";
export { Bm25, tokenize } from "./retrieval/bm25";
export * from "./mcp/client";
export * from "./assistant/prompt";
export * from "./assistant/tools";
export * from "./assistant/loop";
export * from "./generator/draft-sink";
export * from "./generator/generate";
export * from "./generator/templates";
export { aiFeature, type AiFeatureOptions } from "./feature";
export { aiWeb } from "./web";
packages/feature-ai/src/mcp/client.ts
+198
View File
@@ -0,0 +1,198 @@
import type { McpServerConfig } from "../settings";
import { readSse } from "../providers/sse";
export interface McpTool { name: string; description: string; inputSchema: Record<string, unknown> }
export interface McpClient {
tools(): Promise<McpTool[]>;
call(name: string, args: unknown): Promise<string>;
close(): Promise<void>;
}
const PROTOCOL_VERSION = "2025-06-18";
const CLIENT_INFO = { name: "stcourses", version: "1.0.0" };
type RpcReply = { id?: number | string; result?: any; error?: { code: number; message: string } };
interface Transport { request(method: string, params: unknown): Promise<any>; notify(method: string, params?: unknown): Promise<void>; close(): Promise<void> }
const withTimeout = <T>(p: Promise<T>, ms: number, what: string): Promise<T> =>
new Promise((resolve, reject) => {
const t = setTimeout(() => reject(new Error(`MCP ${what} timed out after ${ms} ms`)), ms);
p.then((v) => { clearTimeout(t); resolve(v); }, (e) => { clearTimeout(t); reject(e); });
});
function unwrap(r: RpcReply): any {
if (r.error) throw new Error(`MCP error ${r.error.code}: ${r.error.message}`);
return r.result;
}
/** Inherited by stdio MCP servers. Everything else (STC_SECRET, database URLs, API keys) stays with the server. */
const INHERITED_ENV = ["PATH", "HOME", "LANG"] as const;
/** The environment for a stdio MCP server: PATH, HOME and LANG from the server, plus exactly the admin-configured variables. */
export function mcpChildEnv(cfg: McpServerConfig, parent: Record<string, string | undefined> = process.env): Record<string, string> {
const env: Record<string, string> = {};
for (const k of INHERITED_ENV) if (parent[k] !== undefined) env[k] = parent[k]!;
return { ...env, ...(cfg.env ?? {}) };
}
function stdioTransport(cfg: McpServerConfig, timeoutMs: number): Transport {
if (!cfg.command) throw new Error(`MCP server "${cfg.name}" has no command`);
const proc = Bun.spawn([cfg.command, ...(cfg.args ?? [])], {
stdin: "pipe",
stdout: "pipe",
stderr: "ignore",
env: mcpChildEnv(cfg),
});
let nextId = 1;
const pending = new Map<number, { resolve(v: RpcReply): void; reject(e: Error): void }>();
let closed = false;
const failAll = (e: Error) => { for (const p of pending.values()) p.reject(e); pending.clear(); };
(async () => {
const decoder = new TextDecoder();
let buf = "";
try {
for await (const chunk of proc.stdout as unknown as AsyncIterable<Uint8Array>) {
buf += decoder.decode(chunk, { stream: true });
let nl: number;
while ((nl = buf.indexOf("\n")) !== -1) {
const line = buf.slice(0, nl).trim();
buf = buf.slice(nl + 1);
if (!line) continue;
let msg: RpcReply;
try { msg = JSON.parse(line); } catch { continue; }
if (typeof msg.id === "number" && pending.has(msg.id)) {
pending.get(msg.id)!.resolve(msg);
pending.delete(msg.id);
}
}
}
} catch {}
closed = true;
failAll(new Error(`MCP server "${cfg.name}" exited`));
})();
proc.exited.then((code) => { closed = true; failAll(new Error(`MCP server "${cfg.name}" exited with code ${code}`)); });
const send = (obj: unknown) => {
if (closed) throw new Error(`MCP server "${cfg.name}" is not running`);
proc.stdin.write(JSON.stringify(obj) + "\n");
proc.stdin.flush();
};
return {
async request(method, params) {
const id = nextId++;
const reply = new Promise<RpcReply>((resolve, reject) => pending.set(id, { resolve, reject }));
send({ jsonrpc: "2.0", id, method, params });
return unwrap(await withTimeout(reply, timeoutMs, method));
},
async notify(method, params) { send({ jsonrpc: "2.0", method, ...(params ? { params } : {}) }); },
async close() {
closed = true;
try { proc.stdin.end(); } catch {}
proc.kill();
await withTimeout(proc.exited, 2000, "shutdown").catch(() => proc.kill(9));
},
};
}
function httpTransport(cfg: McpServerConfig, timeoutMs: number, doFetch: typeof fetch): Transport {
if (!cfg.url) throw new Error(`MCP server "${cfg.name}" has no URL`);
const url = cfg.url;
let session: string | null = null;
let nextId = 1;
const post = async (body: unknown, signal: AbortSignal) => {
const headers: Record<string, string> = {
"content-type": "application/json",
accept: "application/json, text/event-stream",
"mcp-protocol-version": PROTOCOL_VERSION,
...(cfg.headers ?? {}),
};
if (session) headers["mcp-session-id"] = session;
const res = await doFetch(url, { method: "POST", headers, body: JSON.stringify(body), signal });
const sid = res.headers.get("mcp-session-id");
if (sid) session = sid;
return res;
};
return {
async request(method, params) {
const id = nextId++;
const ctl = new AbortController();
const t = setTimeout(() => ctl.abort(), timeoutMs);
try {
const res = await post({ jsonrpc: "2.0", id, method, params }, ctl.signal);
if (!res.ok) throw new Error(`MCP HTTP ${res.status} from "${cfg.name}"`);
const type = res.headers.get("content-type") ?? "";
if (type.includes("text/event-stream") && res.body) {
for await (const { data } of readSse(res.body)) {
let msg: RpcReply;
try { msg = JSON.parse(data); } catch { continue; }
if (msg.id === id) { ctl.abort(); return unwrap(msg); }
}
throw new Error(`MCP stream from "${cfg.name}" ended without a reply`);
}
const msg = await res.json();
const reply = Array.isArray(msg) ? msg.find((m: RpcReply) => m.id === id) : msg;
return unwrap(reply ?? { error: { code: -1, message: "no reply" } });
} catch (e) {
if (ctl.signal.aborted && (e as Error).name === "AbortError") throw new Error(`MCP ${method} timed out after ${timeoutMs} ms`);
throw e;
} finally { clearTimeout(t); }
},
async notify(method, params) {
const ctl = new AbortController();
const t = setTimeout(() => ctl.abort(), timeoutMs);
try { await post({ jsonrpc: "2.0", method, ...(params ? { params } : {}) }, ctl.signal); } finally { clearTimeout(t); }
},
async close() {
if (!session) return;
const headers: Record<string, string> = { "mcp-session-id": session, ...(cfg.headers ?? {}) };
await doFetch(url, { method: "DELETE", headers }).catch(() => {});
},
};
}
/** Connects and initializes an MCP server (stdio or streamable HTTP). */
export async function connectMcp(cfg: McpServerConfig, opts: { fetch?: typeof fetch; timeoutMs?: number } = {}): Promise<McpClient> {
const timeoutMs = opts.timeoutMs ?? 20_000;
const t = cfg.transport === "stdio" ? stdioTransport(cfg, timeoutMs) : httpTransport(cfg, timeoutMs, opts.fetch ?? fetch);
try {
await t.request("initialize", { protocolVersion: PROTOCOL_VERSION, capabilities: {}, clientInfo: CLIENT_INFO });
await t.notify("notifications/initialized");
} catch (e) {
await t.close().catch(() => {});
throw e;
}
return {
async tools() {
const out: McpTool[] = [];
let cursor: string | undefined;
do {
const r = await t.request("tools/list", cursor ? { cursor } : {});
for (const tool of r?.tools ?? []) out.push({ name: tool.name, description: tool.description ?? "", inputSchema: tool.inputSchema ?? { type: "object" } });
cursor = r?.nextCursor;
} while (cursor);
return out;
},
async call(name, args) {
const r = await t.request("tools/call", { name, arguments: args ?? {} });
const text = (r?.content ?? [])
.map((c: any) => (c.type === "text" ? c.text : c.type === "resource" ? (c.resource?.text ?? `[resource ${c.resource?.uri}]`) : `[${c.type} content]`))
.join("\n");
const body = text || (r?.structuredContent ? JSON.stringify(r.structuredContent) : "");
if (r?.isError) throw new Error(body || "tool failed");
return body;
},
close: () => t.close(),
};
}
/** Global servers (no courseIds) plus servers that list this course. */
export function mcpServersFor(courseId: string, servers: McpServerConfig[]): McpServerConfig[] {
return servers.filter((s) => !s.courseIds?.length || s.courseIds.includes(courseId));
}
/** Provider-safe tool name (^[a-zA-Z0-9_-]{1,64}$) for an MCP tool. */
export function mcpToolName(serverId: string, tool: string): string {
const clean = (s: string) => s.replace(/[^a-zA-Z0-9_-]/g, "_");
return `mcp_${clean(serverId).slice(0, 20)}__${clean(tool)}`.slice(0, 64);
}
packages/feature-ai/src/providers/anthropic.ts
+139
View File
@@ -0,0 +1,139 @@
import type { AiProvider, ChatMessage, ToolSpec } from "@stc/contracts";
import type { Effort } from "../settings";
import { errorMessage, readSse } from "./sse";
export const DEFAULT_ANTHROPIC_MODEL = "claude-opus-5-5";
const FIRST_PARTY = "https://api.anthropic.com";
export interface AnthropicOptions {
apiKey: string;
model?: string;
baseUrl?: string;
effort?: Effort;
maxTokens?: number;
fetch?: typeof fetch;
}
type Block = Record<string, unknown> & { type: string };
type ApiMessage = { role: "user" | "assistant"; content: Block[] };
/**
* Anthropic Messages API over plain fetch with streaming and tool use.
* - Thinking is left to the model default (adaptive on current models); no temperature or forced tool_choice.
* - The frozen ChatMessage contract has no slot for thinking blocks, so the provider remembers every
* response's raw content blocks by tool_use id and replays them unchanged when the caller sends back
* the matching assistant tool-call message (required for thinking + tool use).
* - On the first-party endpoint, server-side refusal fallbacks are on ("default" routing).
*/
export function createAnthropicProvider(o: AnthropicOptions): AiProvider {
const doFetch = o.fetch ?? fetch;
const base = (o.baseUrl || FIRST_PARTY).replace(/\/+$/, "");
const firstParty = base === FIRST_PARTY;
const rawByToolId = new Map<string, Block[]>();
const toApi = (messages: ChatMessage[]): { system: string; messages: ApiMessage[] } => {
const system = messages.filter((m) => m.role === "system").map((m) => m.content).join("\n\n");
const out: ApiMessage[] = [];
const push = (role: "user" | "assistant", block: Block) => {
const last = out[out.length - 1];
if (last && last.role === role) last.content.push(block);
else out.push({ role, content: [block] });
};
for (const m of messages) {
if (m.role === "system") continue;
if (m.role === "tool") push("user", { type: "tool_result", tool_use_id: m.toolCallId ?? "", content: m.content });
else if (m.role === "assistant" && m.toolCallId) {
let input: unknown = {};
try { input = JSON.parse(m.content || "{}"); } catch { input = { value: m.content }; }
push("assistant", { type: "tool_use", id: m.toolCallId, name: m.name ?? "", input });
} else if (m.content) push(m.role === "assistant" ? "assistant" : "user", { type: "text", text: m.content });
}
// Replace assistant turns we produced with their exact original blocks (thinking signatures intact).
for (const msg of out) {
if (msg.role !== "assistant") continue;
const id = msg.content.find((b) => b.type === "tool_use")?.id as string | undefined;
const raw = id ? rawByToolId.get(id) : undefined;
if (raw) msg.content = structuredClone(raw);
}
return { system, messages: out };
};
return {
id: "anthropic",
async *chat(messages: ChatMessage[], opts: { tools?: ToolSpec[]; signal?: AbortSignal }) {
const { system, messages: apiMessages } = toApi(messages);
const body: Record<string, unknown> = {
model: o.model || DEFAULT_ANTHROPIC_MODEL,
max_tokens: o.maxTokens ?? 64000,
stream: true,
messages: apiMessages,
};
if (system) body.system = system;
if (opts.tools?.length) body.tools = opts.tools.map((t) => ({ name: t.name, description: t.description, input_schema: t.inputSchema }));
if (o.effort) body.output_config = { effort: o.effort };
const headers: Record<string, string> = {
"content-type": "application/json",
"x-api-key": o.apiKey,
"anthropic-version": "2023-06-01",
};
if (firstParty) {
body.fallbacks = "default";
headers["anthropic-beta"] = "server-side-fallback-2026-07-01";
}
const res = await doFetch(`${base}/v1/messages`, { method: "POST", headers, body: JSON.stringify(body), signal: opts.signal });
if (!res.ok || !res.body) throw new Error(`Anthropic API error ${await errorMessage(res)}`);
const blocks: Block[] = [];
const partialJson = new Map<number, string>();
const toolIds: string[] = [];
let stopReason: string | null = null;
for await (const { data } of readSse(res.body)) {
let ev: any;
try { ev = JSON.parse(data); } catch { continue; }
switch (ev.type) {
case "content_block_start": {
const b = { ...ev.content_block } as Block;
blocks[ev.index] = b;
if (b.type === "tool_use") partialJson.set(ev.index, "");
break;
}
case "content_block_delta": {
const b = blocks[ev.index];
const d = ev.delta ?? {};
if (!b) break;
if (d.type === "text_delta") {
b.text = String(b.text ?? "") + d.text;
yield { text: d.text as string };
} else if (d.type === "input_json_delta") partialJson.set(ev.index, (partialJson.get(ev.index) ?? "") + d.partial_json);
else if (d.type === "thinking_delta") b.thinking = String(b.thinking ?? "") + d.thinking;
else if (d.type === "signature_delta") b.signature = String(b.signature ?? "") + d.signature;
else if (d.type === "citations_delta") b.citations = [...((b.citations as unknown[]) ?? []), d.citation];
break;
}
case "content_block_stop": {
const b = blocks[ev.index];
if (b?.type === "tool_use") {
const json = partialJson.get(ev.index) ?? "";
let input: unknown = {};
try { input = json ? JSON.parse(json) : (b.input ?? {}); } catch { input = { _invalidJson: json }; }
b.input = input;
toolIds.push(b.id as string);
yield { toolCall: { id: b.id as string, name: b.name as string, input } };
}
break;
}
case "message_delta":
if (ev.delta?.stop_reason) stopReason = ev.delta.stop_reason;
break;
case "error":
throw new Error(`Anthropic API error: ${ev.error?.message ?? "stream error"}`);
}
}
const content = blocks.filter(Boolean);
for (const id of toolIds) rawByToolId.set(id, content);
if (stopReason === "refusal" && !content.some((b) => b.type === "text" && b.text)) {
yield { text: "The model declined to answer this request." };
}
},
};
}
packages/feature-ai/src/providers/index.ts
+18
View File
@@ -0,0 +1,18 @@
import type { AiProvider } from "@stc/contracts";
import type { ProviderConfig } from "../settings";
import { createAnthropicProvider } from "./anthropic";
import { createOpenAiProvider } from "./openai";
export { createAnthropicProvider, DEFAULT_ANTHROPIC_MODEL } from "./anthropic";
export { createOpenAiProvider } from "./openai";
export type ProviderFactory = (cfg: ProviderConfig, apiKey: string | null) => AiProvider;
export function createProvider(cfg: ProviderConfig, apiKey: string | null, fetchImpl?: typeof fetch): AiProvider {
if (cfg.kind === "anthropic") {
if (!apiKey) throw new Error("The Anthropic provider needs an API key");
return createAnthropicProvider({ apiKey, model: cfg.model, baseUrl: cfg.baseUrl, effort: cfg.effort, fetch: fetchImpl });
}
if (!cfg.baseUrl || !cfg.model) throw new Error("An OpenAI-compatible provider needs a base URL and a model");
return createOpenAiProvider({ baseUrl: cfg.baseUrl, model: cfg.model, apiKey, fetch: fetchImpl });
}
packages/feature-ai/src/providers/openai.ts
+69
View File
@@ -0,0 +1,69 @@
import type { AiProvider, ChatMessage, ToolSpec } from "@stc/contracts";
import { errorMessage, readSse } from "./sse";
export interface OpenAiOptions { baseUrl: string; model: string; apiKey?: string | null; fetch?: typeof fetch }
type OaMessage =
| { role: "system" | "user"; content: string }
| { role: "assistant"; content: string | null; tool_calls?: { id: string; type: "function"; function: { name: string; arguments: string } }[] }
| { role: "tool"; tool_call_id: string; content: string };
function toOpenAi(messages: ChatMessage[]): OaMessage[] {
const out: OaMessage[] = [];
for (const m of messages) {
if (m.role === "assistant" && m.toolCallId) {
const call = { id: m.toolCallId, type: "function" as const, function: { name: m.name ?? "", arguments: m.content || "{}" } };
const last = out[out.length - 1];
if (last && last.role === "assistant") (last.tool_calls ??= []).push(call);
else out.push({ role: "assistant", content: null, tool_calls: [call] });
} else if (m.role === "tool") out.push({ role: "tool", tool_call_id: m.toolCallId ?? "", content: m.content });
else out.push({ role: m.role, content: m.content } as OaMessage);
}
return out;
}
/** OpenAI-compatible chat completions (OpenAI, Ollama, vLLM, LM Studio, OpenRouter …) with streaming tool calls. */
export function createOpenAiProvider(o: OpenAiOptions): AiProvider {
const doFetch = o.fetch ?? fetch;
const base = o.baseUrl.replace(/\/+$/, "");
return {
id: "openai",
async *chat(messages, opts: { tools?: ToolSpec[]; signal?: AbortSignal }) {
const body: Record<string, unknown> = { model: o.model, stream: true, messages: toOpenAi(messages) };
if (opts.tools?.length) body.tools = opts.tools.map((t) => ({ type: "function", function: { name: t.name, description: t.description, parameters: t.inputSchema } }));
const headers: Record<string, string> = { "content-type": "application/json" };
if (o.apiKey) headers.authorization = `Bearer ${o.apiKey}`;
const res = await doFetch(`${base}/chat/completions`, { method: "POST", headers, body: JSON.stringify(body), signal: opts.signal });
if (!res.ok || !res.body) throw new Error(`Chat endpoint error ${await errorMessage(res)}`);
const pending = new Map<number, { id: string; name: string; args: string }>();
const flush = function* () {
for (const [, c] of [...pending].sort((a, b) => a[0] - b[0])) {
let input: unknown = {};
try { input = c.args ? JSON.parse(c.args) : {}; } catch { input = { _invalidJson: c.args }; }
yield { toolCall: { id: c.id, name: c.name, input } };
}
pending.clear();
};
for await (const { data } of readSse(res.body)) {
if (data.trim() === "[DONE]") break;
let ev: any;
try { ev = JSON.parse(data); } catch { continue; }
if (ev.error) throw new Error(`Chat endpoint error: ${ev.error.message ?? JSON.stringify(ev.error)}`);
const choice = ev.choices?.[0];
if (!choice) continue;
const d = choice.delta ?? {};
if (typeof d.content === "string" && d.content) yield { text: d.content };
for (const tc of d.tool_calls ?? []) {
const idx = tc.index ?? 0;
const cur = pending.get(idx) ?? { id: "", name: "", args: "" };
if (tc.id) cur.id = tc.id;
if (tc.function?.name) cur.name += tc.function.name;
if (tc.function?.arguments) cur.args += tc.function.arguments;
pending.set(idx, cur);
}
if (choice.finish_reason) yield* flush();
}
yield* flush();
},
};
}
packages/feature-ai/src/providers/sse.ts
+44
View File
@@ -0,0 +1,44 @@
/** Minimal Server-Sent Events reader: yields one {event, data} per blank-line-terminated record. */
export async function* readSse(body: ReadableStream<Uint8Array>): AsyncIterable<{ event: string; data: string }> {
const decoder = new TextDecoder();
let buf = "";
let event = "message";
let data: string[] = [];
const flushLine = function* (line: string): Generator<{ event: string; data: string }> {
if (line === "") {
if (data.length) yield { event, data: data.join("\n") };
event = "message";
data = [];
return;
}
if (line.startsWith(":")) return;
const i = line.indexOf(":");
const field = i === -1 ? line : line.slice(0, i);
let value = i === -1 ? "" : line.slice(i + 1);
if (value.startsWith(" ")) value = value.slice(1);
if (field === "event") event = value;
else if (field === "data") data.push(value);
};
for await (const chunk of body as unknown as AsyncIterable<Uint8Array>) {
buf += decoder.decode(chunk, { stream: true });
let nl: number;
while ((nl = buf.search(/\r\n|\n|\r/)) !== -1) {
const line = buf.slice(0, nl);
buf = buf.slice(nl + (buf.startsWith("\r\n", nl) ? 2 : 1));
yield* flushLine(line);
}
}
buf += decoder.decode();
if (buf) yield* flushLine(buf);
yield* flushLine("");
}
export async function errorMessage(res: Response): Promise<string> {
const text = await res.text().catch(() => "");
try {
const j = JSON.parse(text);
const m = j?.error?.message ?? j?.message ?? j?.error;
if (typeof m === "string") return `${res.status}: ${m}`;
} catch {}
return `${res.status}: ${text.slice(0, 300) || res.statusText}`;
}
packages/feature-ai/src/retrieval/bm25.ts
+79
View File
@@ -0,0 +1,79 @@
const STOP = new Set(
"a an and are as at be but by for from has have i if in into is it its of on or so that the their then there these this to was were what when which who why will with you your".split(" "),
);
/** Light plural folding so "apples" matches "apple". */
const stem = (w: string): string =>
w.length > 4 && w.endsWith("ies") ? `${w.slice(0, -3)}y` : w.length > 3 && w.endsWith("s") && !w.endsWith("ss") ? w.slice(0, -1) : w;
/** Lowercased word tokens without stopwords; camelCase and snake_case identifiers also add their parts. */
export function tokenize(text: string): string[] {
const out: string[] = [];
for (const raw of text.match(/[\p{L}\p{N}_]+/gu) ?? []) {
const lower = raw.toLowerCase();
if (STOP.has(lower) || lower.length > 64) continue;
out.push(stem(lower));
const parts = raw.split(/_|(?<=[\p{Ll}\p{N}])(?=\p{Lu})/u).filter(Boolean);
if (parts.length > 1) for (const p of parts) if (p.length > 1) out.push(stem(p.toLowerCase()));
}
return out;
}
type Doc<T> = { item: T; tf: Map<string, number>; len: number };
const termFreq = (tokens: string[]) => {
const tf = new Map<string, number>();
for (const t of tokens) tf.set(t, (tf.get(t) ?? 0) + 1);
return tf;
};
/** Okapi BM25 (k1 = 1.2, b = 0.75) over an in-memory document list. */
export class Bm25<T> {
private docs: Doc<T>[] = [];
private df = new Map<string, number>();
private totalLen = 0;
add(text: string, item: T): void {
const tokens = tokenize(text);
const tf = termFreq(tokens);
for (const t of tf.keys()) this.df.set(t, (this.df.get(t) ?? 0) + 1);
this.docs.push({ item, tf, len: tokens.length });
this.totalLen += tokens.length;
}
get size(): number { return this.docs.length; }
private idf(term: string): number {
const n = this.docs.length;
const df = this.df.get(term) ?? 0;
return Math.log(1 + (n - df + 0.5) / (df + 0.5));
}
private score(q: string[], tf: Map<string, number>, len: number): number {
const avg = this.docs.length ? this.totalLen / this.docs.length : Math.max(1, len);
let s = 0;
for (const term of q) {
const f = tf.get(term);
if (!f) continue;
s += this.idf(term) * ((f * 2.2) / (f + 1.2 * (0.25 + (0.75 * len) / avg)));
}
return s;
}
/** Top-k matches. `extra` documents are scored with this index's statistics without being added to it. */
search(query: string, k: number, extra: { text: string; item: T }[] = []): { item: T; score: number }[] {
const q = [...new Set(tokenize(query))];
if (!q.length) return [];
const results: { item: T; score: number }[] = [];
for (const d of this.docs) {
const s = this.score(q, d.tf, d.len);
if (s > 0) results.push({ item: d.item, score: s });
}
for (const e of extra) {
const tokens = tokenize(e.text);
const s = this.score(q, termFreq(tokens), tokens.length);
if (s > 0) results.push({ item: e.item, score: s });
}
return results.sort((a, b) => b.score - a.score).slice(0, k);
}
}
packages/feature-ai/src/retrieval/course-index.ts
+349
View File
@@ -0,0 +1,349 @@
import { realpathSync } from "node:fs";
import { readdir, stat } from "node:fs/promises";
import { basename, extname, join, posix as posixPath, relative, sep } from "node:path";
import { slugify, type CourseRepository, type LoadedCourse } from "@stc/course-format";
import { Bm25 } from "./bm25";
export type ChunkKind = "lesson" | "transcript" | "markdown" | "code" | "deck" | "note";
export interface Chunk {
path: string; // course-relative path (for cache text: the deck or lesson it belongs to)
kind: ChunkKind;
text: string;
heading?: string;
lessonId?: string;
lessonTitle?: string;
}
export interface CourseIndex { size: number; search(query: string, k: number, extra?: Chunk[]): Chunk[] }
const SKIP_DIRS = new Set([".git", "node_modules", "dist", "build", "target", ".astro", ".next", "coverage", "bin", "obj", ".venv", "__pycache__"]);
const MD = new Set([".md", ".markdown"]);
const CODE = new Set([".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx", ".py", ".rs", ".scala", ".sc", ".fs", ".fsx", ".go", ".java", ".kt", ".rb", ".php", ".c", ".h", ".cpp", ".cs", ".sh", ".bash", ".ps1", ".sql", ".yaml", ".yml", ".toml", ".json", ".tf", ".html", ".css"]);
const CODE_NAMES = new Set(["Dockerfile", "Makefile", "docker-compose.yml", "Containerfile"]);
const CACHE_TEXT = new Set([".txt", ".md", ".vtt", ".srt"]);
// Lock files, the manifest, and instructions written for AI agents (the mentor file goes into the system
// prompt only while mentor mode is on, so it must not leak back in through retrieval).
const SKIP_FILES = new Set(["package-lock.json", "bun.lock", "yarn.lock", "pnpm-lock.yaml", "Cargo.lock", "stcourse.yaml", "AGENTS.md", "CLAUDE.md", "GEMINI.md"]);
const MAX_FILE = 200_000;
const MAX_FILES = 2_000;
const MAX_CHUNK = 1_500;
const SOLUTION_SEGMENTS = new Set(["solution", "solutions", "reference"]);
const posix = (p: string) => p.split(sep).join("/");
/**
* Files only authors may see: manifest `browser.solution` files and anything under a `solution`, `solutions`
* or `reference` folder (Pure Snake `js/solutions/`, Systemic Design `reference/` and `expansion/solution/`).
* The assistant never retrieves or reads them. `rel` is normalised first; for paths that come from a user or a
* model, use `canonicalCoursePath` + `isGuardedFile`, which also follow symlinks.
*/
export function isSolutionPath(course: LoadedCourse, rel: string): boolean {
const p = normalizeRel(rel);
if (p === null) return true; // not a course-relative path at all: refuse
if (p.split("/").slice(0, -1).some((s) => SOLUTION_SEGMENTS.has(s.toLowerCase()))) return true;
if (/(^|\/)solution\.[a-z]+$/i.test(p)) return true;
for (const m of course.manifest?.modules ?? []) for (const l of m.lessons) if (l.browser?.solution && normalizeRel(l.browser.solution) === p) return true;
return false;
}
/** Posix-normalised course-relative path, or null when it is absolute or climbs out with `..`. */
export function normalizeRel(raw: string): string | null {
if (raw.includes("\u0000")) return null;
const p = posixPath.normalize(raw.replace(/\\/g, "/")).replace(/^(\.\/)+/, "");
if (!p || p === "." || p.startsWith("/") || /^[a-zA-Z]:/.test(p)) return null;
if (p.split("/").includes("..")) return null;
return p.replace(/\/+$/, "");
}
/** Undoes URL encoding (up to three layers). A malformed escape leaves the string as it is. */
function decodeLayers(raw: string): string {
let s = raw;
for (let i = 0; i < 3 && /%[0-9a-fA-F]{2}/.test(s); i++) {
try { s = decodeURIComponent(s); } catch { break; }
}
return s;
}
/**
* Canonical form of a path a user or the model asked for: URL-decoded, backslashes as slashes, posix-normalised,
* leading `./` and `/` stripped, `..` rejected, then resolved through `CourseRepository.resolvePath` (real path,
* symlinks followed, must stay inside the course root). `rel` is the real path relative to the real root.
* Null when the path is invalid, escapes the course or does not exist.
*/
export function canonicalCoursePath(repo: CourseRepository, course: LoadedCourse, raw: string): { rel: string; abs: string } | null {
const norm = normalizeRel(decodeLayers(raw.trim()).replace(/\\/g, "/").replace(/^\/+/, ""));
if (!norm) return null;
const abs = repo.resolvePath(course.id, norm);
if (!abs) return null;
let realRoot: string;
try { realRoot = realpathSync(course.root); } catch { return null; }
const rel = relative(realRoot, abs).split(sep).join("/");
if (!rel || rel.startsWith("..")) return null;
return { rel, abs };
}
/**
* The guard for tool reads, on canonical paths only: solution files (by the folder and name rules on the real
* relative path, and by comparing real paths with every manifest `browser.solution`) and hidden files or folders.
*/
export function isGuardedFile(repo: CourseRepository, course: LoadedCourse, canonical: { rel: string; abs: string }): boolean {
if (canonical.rel.split("/").some((s) => s.startsWith("."))) return true;
if (isSolutionPath(course, canonical.rel)) return true;
for (const m of course.manifest?.modules ?? []) {
for (const l of m.lessons) {
if (!l.browser?.solution) continue;
const sol = repo.resolvePath(course.id, l.browser.solution);
if (sol && sol === canonical.abs) return true;
}
}
return false;
}
async function walk(root: string, onFile: (abs: string, rel: string) => Promise<boolean>, depth = 0, base = root): Promise<boolean> {
if (depth > 8) return true;
let entries;
try { entries = await readdir(root, { withFileTypes: true }); } catch { return true; }
entries.sort((a, b) => a.name.localeCompare(b.name));
for (const e of entries) {
if (e.name.startsWith(".") && e.name !== ".") continue;
const abs = join(root, e.name);
if (e.isDirectory()) {
if (SKIP_DIRS.has(e.name)) continue;
if (!(await walk(abs, onFile, depth + 1, base))) return false;
} else if (e.isFile()) {
if (!(await onFile(abs, posix(relative(base, abs))))) return false;
}
}
return true;
}
/** Splits Markdown at headings (outside code fences), then long sections at paragraph breaks. */
export function splitMarkdown(text: string): { heading?: string; text: string }[] {
const sections: { heading?: string; lines: string[] }[] = [{ lines: [] }];
let fence = false;
for (const line of text.split("\n")) {
if (/^\s*(```|~~~)/.test(line)) fence = !fence;
const h = !fence && /^(#{1,6})\s+(.*)$/.exec(line);
if (h) sections.push({ heading: h[2]!.replace(/#+\s*$/, "").trim(), lines: [line] });
else sections[sections.length - 1]!.lines.push(line);
}
const out: { heading?: string; text: string }[] = [];
for (const s of sections) {
const body = s.lines.join("\n").trim();
if (!body) continue;
out.push(...splitLong(body).map((t) => ({ heading: s.heading, text: t })));
}
return out;
}
function splitLong(text: string): string[] {
if (text.length <= MAX_CHUNK) return [text];
const out: string[] = [];
let cur = "";
for (const para of text.split(/\n{2,}/)) {
if (cur && cur.length + para.length > MAX_CHUNK) { out.push(cur); cur = ""; }
if (para.length > MAX_CHUNK) {
for (let i = 0; i < para.length; i += MAX_CHUNK) out.push(para.slice(i, i + MAX_CHUNK));
} else cur = cur ? `${cur}\n\n${para}` : para;
}
if (cur) out.push(cur);
return out;
}
type LessonRef = { lessonId: string; lessonTitle: string; anchor?: string; transcript: boolean };
function lessonRefs(course: LoadedCourse): Map<string, LessonRef[]> {
const refs = new Map<string, LessonRef[]>();
const add = (p: string | undefined, r: LessonRef) => {
if (!p) return;
const key = posix(p).replace(/^\.\//, "");
refs.set(key, [...(refs.get(key) ?? []), r]);
};
for (const m of course.manifest?.modules ?? []) {
for (const l of m.lessons) {
const base = { lessonId: l.id, lessonTitle: l.title };
if (l.type !== "video") add(l.file, { ...base, anchor: l.anchor, transcript: false });
add(l.transcript, { ...base, transcript: true });
if (l.hints?.file && l.hints.file !== l.file) add(l.hints.file, { ...base, transcript: false });
for (const t of Object.values(l.tracks ?? {})) add(t.file, { ...base, transcript: false });
}
}
return refs;
}
function pickLesson(refs: LessonRef[] | undefined, heading?: string): LessonRef | undefined {
if (!refs?.length) return undefined;
if (heading) {
const slug = slugify(heading);
const hit = refs.find((r) => r.anchor && r.anchor === slug);
if (hit) return hit;
}
return refs.find((r) => !r.anchor) ?? refs[0];
}
function lessonTitleOf(course: LoadedCourse, lessonId: string | undefined): string | undefined {
return lessonId ? course.lessonIndex.get(lessonId)?.lesson.title : undefined;
}
async function readJson<T>(path: string): Promise<T | null> {
try {
const f = Bun.file(path);
if (!(await f.exists()) || f.size > MAX_FILE * 25) return null;
return (await f.json()) as T;
} catch {
return null;
}
}
const clock = (s: number) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
export async function buildCourseIndex(course: LoadedCourse, dataDir: string): Promise<CourseIndex> {
const bm = new Bm25<Chunk>();
const seen = new Set<string>();
const refs = lessonRefs(course);
const mentor = course.manifest?.mentor ? posix(course.manifest.mentor).replace(/^\.\//, "") : null;
const codeDirs = (course.manifest?.modules ?? []).flatMap((m) => (m.code ? [posix(m.code.dir).replace(/\/$/, "") + "/"] : []));
const add = (c: Chunk) => {
const key = `${c.kind}\u0000${c.text}`;
if (seen.has(key)) return;
seen.add(key);
bm.add([c.heading ?? "", c.lessonTitle ?? "", c.path, c.text].join("\n"), c);
};
let files = 0;
if (course.status === "ok") {
await walk(course.root, async (abs, rel) => {
if (files >= MAX_FILES) return false;
const ext = extname(rel).toLowerCase();
const name = basename(rel);
const isMd = MD.has(ext);
const isCode = !isMd && (CODE.has(ext) || CODE_NAMES.has(name) || codeDirs.some((d) => rel.startsWith(d)));
if ((!isMd && !isCode) || SKIP_FILES.has(name) || rel === mentor || isSolutionPath(course, rel)) return true;
let size = 0;
try { size = (await stat(abs)).size; } catch { return true; }
if (size > MAX_FILE || size === 0) return true;
const text = await Bun.file(abs).text();
if (isCode && text.includes("\u0000")) return true;
files++;
const r = refs.get(rel);
if (isMd) {
for (const part of splitMarkdown(text)) {
const ref = pickLesson(r, part.heading);
add({
path: rel,
kind: ref ? (ref.transcript ? "transcript" : "lesson") : "markdown",
text: part.text,
heading: part.heading,
lessonId: ref?.lessonId,
lessonTitle: ref?.lessonTitle,
});
}
} else {
const lines = text.split("\n");
for (let i = 0; i < lines.length; i += 60) {
add({ path: rel, kind: "code", text: lines.slice(i, i + 60).join("\n"), heading: lines.length > 60 ? `lines ${i + 1}-${Math.min(lines.length, i + 60)}` : undefined });
}
}
return true;
});
}
const decks = new Map((course.manifest?.decks ?? []).map((d) => [d.id, d]));
// The importer's deck cache: <dataDir>/cache/decks/<courseId>/<deckId>/text.json = [{ page, text }].
for (const deck of decks.values()) {
const pages = await readJson<{ page?: number; text?: string }[]>(join(dataDir, "cache", "decks", course.id, deck.id, "text.json"));
if (!Array.isArray(pages)) continue;
const lessonId = course.lessonIndex.has(deck.attachTo) ? deck.attachTo : undefined;
for (const p of pages) {
const text = typeof p?.text === "string" ? p.text.trim() : "";
if (!text) continue;
for (const part of splitLong(text)) {
add({ path: posix(deck.file), kind: "deck", text: part, heading: p.page ? `Slide ${p.page}` : deck.title, lessonId, lessonTitle: lessonTitleOf(course, lessonId) });
}
}
}
// The importer's transcription cache: <dataDir>/cache/transcripts/<courseId>/<lessonId>.json =
// { lessonId, text, segments: [{ start, end, text }] }. Skipped when the lesson has a transcript file in the
// course, which the transcription job writes alongside and which is indexed above.
const trDir = join(dataDir, "cache", "transcripts", course.id);
for (const name of await readdir(trDir).catch(() => [] as string[])) {
if (extname(name) !== ".json") continue;
const lessonId = name.slice(0, -5);
const entry = course.lessonIndex.get(lessonId);
if (!entry || entry.lesson.transcript) continue;
const tr = await readJson<{ text?: string; segments?: { start?: number; text?: string }[] }>(join(trDir, name));
if (!tr) continue;
const segs = Array.isArray(tr.segments) ? tr.segments.filter((s) => typeof s?.text === "string" && s.text.trim()) : [];
const blocks: { at: number; text: string }[] = [];
if (segs.length) {
for (const s of segs) {
const last = blocks[blocks.length - 1];
if (last && last.text.length + s.text!.length < MAX_CHUNK) last.text += ` ${s.text!.trim()}`;
else blocks.push({ at: Number(s.start) || 0, text: s.text!.trim() });
}
} else if (typeof tr.text === "string" && tr.text.trim()) {
blocks.push(...splitLong(tr.text.trim()).map((t) => ({ at: -1, text: t })));
}
for (const b of blocks) {
add({ path: entry.lesson.file ? posix(entry.lesson.file) : `transcripts/${lessonId}`, kind: "transcript", text: b.text, heading: b.at >= 0 ? `at ${clock(b.at)}` : undefined, lessonId, lessonTitle: entry.lesson.title });
}
}
// Older layout: plain text under <dataDir>/cache/<courseId>/**.
const cacheRoot = join(dataDir, "cache", course.id);
await walk(cacheRoot, async (abs, rel) => {
if (!CACHE_TEXT.has(extname(rel).toLowerCase())) return true;
const size = (await stat(abs).catch(() => null))?.size ?? 0;
if (!size || size > MAX_FILE * 5) return true;
const text = await Bun.file(abs).text();
const segs = rel.split("/");
const deck = segs.map((s) => decks.get(s)).find(Boolean);
const lessonSeg = segs.map((s) => s.replace(/\.[^.]+$/, "")).find((s) => course.lessonIndex.has(s));
const page = /(?:page|slide)[-_ ]?(\d+)/i.exec(basename(rel))?.[1];
if (deck) {
const lessonId = course.lessonIndex.has(deck.attachTo) ? deck.attachTo : undefined;
for (const part of splitLong(text.trim())) {
add({ path: posix(deck.file), kind: "deck", text: part, heading: page ? `Slide ${Number(page)}` : deck.title, lessonId, lessonTitle: lessonTitleOf(course, lessonId) });
}
} else {
const kind: ChunkKind = /transcript|\.vtt$|\.srt$/i.test(rel) || lessonSeg ? "transcript" : "deck";
for (const part of splitLong(text.trim())) {
add({ path: `cache/${rel}`, kind, text: part, lessonId: lessonSeg, lessonTitle: lessonTitleOf(course, lessonSeg) });
}
}
return true;
});
return {
get size() { return bm.size; },
search(query, k, extra = []) {
return bm.search(query, k, extra.map((c) => ({ text: [c.heading ?? "", c.lessonTitle ?? "", c.text].join("\n"), item: c }))).map((r) => r.item);
},
};
}
/** The user's notes as chunks, scored at query time and never cached. */
export function noteChunks(course: LoadedCourse, notes: { id: string; lessonId: string; body: string; anchorKind: string; anchorValue: string | null }[]): Chunk[] {
return notes.map((n) => ({
path: "notes",
kind: "note" as const,
text: n.body,
heading: n.anchorKind === "video" && n.anchorValue ? `at ${n.anchorValue}s` : n.anchorKind === "slide" && n.anchorValue ? `slide ${n.anchorValue}` : undefined,
lessonId: n.lessonId,
lessonTitle: lessonTitleOf(course, n.lessonId),
}));
}
const TTL_MS = 5 * 60_000;
const cache = new WeakMap<LoadedCourse, { at: number; index: Promise<CourseIndex> }>();
/** Cached per LoadedCourse object: a repository reload yields new objects, and entries expire after 5 minutes. */
export function getCourseIndex(course: LoadedCourse, dataDir: string): Promise<CourseIndex> {
const hit = cache.get(course);
if (hit && Date.now() - hit.at < TTL_MS) return hit.index;
const index = buildCourseIndex(course, dataDir);
cache.set(course, { at: Date.now(), index });
index.catch(() => cache.delete(course));
return index;
}
packages/feature-ai/src/settings.ts
+135
View File
@@ -0,0 +1,135 @@
import type { Db } from "@stc/db";
import type { SecretBox } from "./crypto";
export type ProviderKind = "anthropic" | "openai";
export type Effort = "low" | "medium" | "high" | "xhigh" | "max";
export interface ProviderConfig { kind: ProviderKind; model?: string; baseUrl?: string; effort?: Effort }
export type PublicProviderConfig = ProviderConfig & { hasApiKey: boolean };
export interface McpServerConfig {
id: string;
name: string;
transport: "stdio" | "http";
command?: string;
args?: string[];
env?: Record<string, string>;
url?: string;
headers?: Record<string, string>;
/** Courses this server is offered in. Absent or empty: every course (global). */
courseIds?: string[];
}
/** `allowedBaseUrls`: origins learners' own keys (BYOK) may be sent to, besides the official endpoints. */
export interface ServerSettings { provider: PublicProviderConfig | null; mcpServers: McpServerConfig[]; allowedBaseUrls: string[] }
export interface ServerSettingsInput {
provider: (ProviderConfig & { apiKey?: string; clearApiKey?: boolean }) | null;
mcpServers: McpServerConfig[];
/** Omitted: keep the saved list. */
allowedBaseUrls?: string[];
}
export interface UserSettings { byok: PublicProviderConfig | null; mentorOff: Record<string, boolean> }
type ServerRow = { provider: ProviderConfig | null; apiKeyCipher: string | null; mcpServers: McpServerConfig[]; allowedBaseUrls?: string[] };
type UserRow = { byok: ProviderConfig | null; mentorOff: Record<string, boolean> };
const pickProvider = (p: ProviderConfig): ProviderConfig => {
const out: ProviderConfig = { kind: p.kind };
if (p.model) out.model = p.model;
if (p.baseUrl) out.baseUrl = p.baseUrl;
if (p.effort) out.effort = p.effort;
return out;
};
const parse = <T>(s: string | undefined, fallback: T): T => {
if (!s) return fallback;
try { return { ...fallback, ...JSON.parse(s) }; } catch { return fallback; }
};
/**
* AI settings in the ai_settings and user_api_keys tables of 0001_init.sql.
* - ('server', '') holds the admin provider (key encrypted) and MCP servers.
* - ('user', <userId>) holds the BYOK provider choice and per-course mentor toggles. The frozen primary key
* is (scope, owner_id), so the per-course flag lives in config_json; the mentor_off column means
* "off for every course" and stays 0 (see docs/contract-requests/ai.md).
* Keys never leave this module except through serverApiKey/userApiKey, which only server code calls.
*/
export function aiSettingsRepo(db: Db, box: SecretBox) {
const read = async (scope: "server" | "user", owner: string): Promise<string | undefined> => {
const rows = await db.sql`SELECT config_json FROM ai_settings WHERE scope = ${scope} AND owner_id = ${owner}`;
return rows[0]?.config_json;
};
const write = async (scope: "server" | "user", owner: string, value: unknown) => {
const json = JSON.stringify(value);
const now = new Date().toISOString();
await db.sql.begin(async (tx) => {
const exists = await tx`SELECT 1 AS x FROM ai_settings WHERE scope = ${scope} AND owner_id = ${owner}`;
if (exists.length) await tx`UPDATE ai_settings SET config_json = ${json}, updated_at = ${now} WHERE scope = ${scope} AND owner_id = ${owner}`;
else await tx`INSERT INTO ai_settings (scope, owner_id, mentor_off, config_json, updated_at) VALUES (${scope}, ${owner}, 0, ${json}, ${now})`;
});
};
const serverRow = async (): Promise<ServerRow> => parse(await read("server", ""), { provider: null, apiKeyCipher: null, mcpServers: [] });
const userRow = async (userId: string): Promise<UserRow> => parse(await read("user", userId), { byok: null, mentorOff: {} });
const keyRow = async (userId: string) =>
(await db.sql`SELECT provider, ciphertext FROM user_api_keys WHERE user_id = ${userId}`)[0] as { provider: string; ciphertext: string } | undefined;
return {
async getServer(): Promise<ServerSettings> {
const r = await serverRow();
return { provider: r.provider ? { ...r.provider, hasApiKey: !!r.apiKeyCipher } : null, mcpServers: r.mcpServers ?? [], allowedBaseUrls: r.allowedBaseUrls ?? [] };
},
async saveServer(input: ServerSettingsInput): Promise<void> {
const prev = await serverRow();
let apiKeyCipher: string | null = null;
if (input.provider) {
if (input.provider.apiKey) apiKeyCipher = await box.encrypt(input.provider.apiKey);
else if (!input.provider.clearApiKey) apiKeyCipher = prev.apiKeyCipher;
}
await write("server", "", {
provider: input.provider ? pickProvider(input.provider) : null,
apiKeyCipher,
mcpServers: input.mcpServers ?? [],
allowedBaseUrls: input.allowedBaseUrls ?? prev.allowedBaseUrls ?? [],
});
},
async serverApiKey(): Promise<string | null> {
const r = await serverRow();
return r.provider && r.apiKeyCipher ? box.decrypt(r.apiKeyCipher) : null;
},
async getUser(userId: string): Promise<UserSettings> {
const r = await userRow(userId);
const k = await keyRow(userId);
return { byok: r.byok && k ? { ...r.byok, hasApiKey: true } : null, mentorOff: r.mentorOff ?? {} };
},
async saveUserKey(userId: string, cfg: ProviderConfig & { apiKey: string }): Promise<void> {
const ciphertext = await box.encrypt(cfg.apiKey);
const now = new Date().toISOString();
await db.sql.begin(async (tx) => {
await tx`DELETE FROM user_api_keys WHERE user_id = ${userId}`;
await tx`INSERT INTO user_api_keys (user_id, provider, ciphertext, created_at) VALUES (${userId}, ${cfg.kind}, ${ciphertext}, ${now})`;
});
const r = await userRow(userId);
await write("user", userId, { ...r, byok: pickProvider(cfg) });
},
async removeUserKey(userId: string): Promise<void> {
await db.sql`DELETE FROM user_api_keys WHERE user_id = ${userId}`;
const r = await userRow(userId);
await write("user", userId, { ...r, byok: null });
},
async userApiKey(userId: string): Promise<string | null> {
const k = await keyRow(userId);
return k ? box.decrypt(k.ciphertext) : null;
},
async setMentorOff(userId: string, courseId: string, off: boolean): Promise<void> {
const r = await userRow(userId);
const mentorOff = { ...r.mentorOff };
if (off) mentorOff[courseId] = true;
else delete mentorOff[courseId];
await write("user", userId, { ...r, mentorOff });
},
async isMentorOff(userId: string, courseId: string): Promise<boolean> {
return (await userRow(userId)).mentorOff?.[courseId] === true;
},
};
}
export type AiSettingsRepo = ReturnType<typeof aiSettingsRepo>;
packages/feature-ai/src/web/AccountAiPage.tsx
+84
View File
@@ -0,0 +1,84 @@
import { useEffect, useState } from "react";
import { Pill } from "@stc/ui";
import { api, ApiError } from "./api";
import { Field, PageHead } from "./components";
interface Account {
byok: { kind: "anthropic" | "openai"; model?: string; baseUrl?: string; hasApiKey: boolean } | null;
serverProvider: { kind: string; model: string | null } | null;
}
export default function AccountAiPage() {
const [acct, setAcct] = useState<Account | null>(null);
const [denied, setDenied] = useState<string | null>(null);
const [kind, setKind] = useState<"anthropic" | "openai">("anthropic");
const [apiKey, setApiKey] = useState("");
const [model, setModel] = useState("");
const [baseUrl, setBaseUrl] = useState("");
const [msg, setMsg] = useState<{ ok: boolean; text: string } | null>(null);
const load = () => api<Account>("/api/ai/account").then((a) => {
setAcct(a);
if (a.byok) { setKind(a.byok.kind); setModel(a.byok.model ?? ""); setBaseUrl(a.byok.baseUrl ?? ""); }
}, (e: ApiError) => setDenied(e.status === 401 ? "Sign in to manage your own AI key." : e.message));
useEffect(() => { load(); }, []);
const save = async () => {
setMsg(null);
try {
await api("/api/ai/account/key", { method: "PUT", body: { kind, apiKey, model, baseUrl } });
setApiKey("");
await load();
setMsg({ ok: true, text: "Saved. Your questions now use your own key." });
} catch (e) { setMsg({ ok: false, text: (e as Error).message }); }
};
const remove = async () => {
await api("/api/ai/account/key", { method: "DELETE" });
setApiKey("");
await load();
setMsg({ ok: true, text: "Your key is removed." });
};
if (denied) return <div className="ai-page"><PageHead chap="Account" title="Your AI key" /><p className="ai-note">{denied}</p></div>;
if (!acct) return <div className="ai-page"><PageHead chap="Account" title="Your AI key" /></div>;
return (
<div className="ai-page">
<PageHead chap="Account" title="Your AI key">
<p>
Bring your own key to use the course assistant and the studio generator with your own account. It is used only for your requests, stored encrypted, and never shown again.{" "}
{acct.serverProvider ? "This server also has a shared provider; your key takes priority while it is saved." : "This server has no shared provider, so AI is off for you until you add a key."}
</p>
</PageHead>
{acct.byok ? <p className="ai-banner" role="status">A {acct.byok.kind === "anthropic" ? "Anthropic" : "OpenAI-compatible"} key is saved{acct.byok.model ? ` (model ${acct.byok.model})` : ""}.</p> : null}
<form onSubmit={(e) => { e.preventDefault(); save(); }}>
<div className="ai-grid">
<Field label="Provider">
<select className="ai-select" value={kind} onChange={(e) => setKind(e.target.value as "anthropic" | "openai")}>
<option value="anthropic">Anthropic</option>
<option value="openai">OpenAI-compatible</option>
</select>
</Field>
<Field label="Model" hint={kind === "anthropic" ? "Leave empty for claude-opus-5-5." : undefined}>
<input className="ai-input" value={model} onChange={(e) => setModel(e.target.value)} placeholder={kind === "anthropic" ? "claude-opus-5-5" : "model name"} />
</Field>
</div>
{kind === "openai" ? (
<Field label="Base URL (optional)" hint="Leave empty for api.openai.com. Other endpoints work only if an admin allowed them.">
<input className="ai-input ai-mono" value={baseUrl} onChange={(e) => setBaseUrl(e.target.value)} placeholder="https://api.openai.com/v1" />
</Field>
) : null}
<Field label="API key" hint={acct.byok ? "Paste a new key to replace the saved one." : undefined}>
<input className="ai-input ai-mono" type="password" autoComplete="off" value={apiKey} onChange={(e) => setApiKey(e.target.value)} placeholder="paste key" />
</Field>
<div className="ai-footer">
<Pill variant="primary" type="submit" disabled={!apiKey.trim()}>Save key</Pill>
{acct.byok ? <Pill onClick={remove}>Remove my key</Pill> : null}
{msg ? <span className={msg.ok ? "ai-hint ai-ok" : "ai-error"} role="status" style={{ margin: 0 }}>{msg.text}</span> : null}
</div>
</form>
</div>
);
}
packages/feature-ai/src/web/AdminAiPage.tsx
+184
View File
@@ -0,0 +1,184 @@
import { useEffect, useState } from "react";
import { Pill } from "@stc/ui";
import { api, ApiError } from "./api";
import { Field, PageHead } from "./components";
type Kind = "" | "anthropic" | "openai";
interface ProviderOut { kind: "anthropic" | "openai"; model?: string; baseUrl?: string; effort?: string; hasApiKey: boolean }
interface McpServer { id: string; name: string; transport: "stdio" | "http"; command?: string; args?: string[]; env?: Record<string, string>; url?: string; headers?: Record<string, string>; courseIds?: string[] }
interface Config { provider: ProviderOut | null; mcpServers: McpServer[]; allowedBaseUrls: string[] }
type ServerDraft = { id: string; name: string; transport: "stdio" | "http"; command: string; args: string; env: string; url: string; headers?: Record<string, string>; courses: string; test?: string };
const toDraft = (s: McpServer): ServerDraft => ({
id: s.id, name: s.name, transport: s.transport, command: s.command ?? "", args: (s.args ?? []).join(" "),
env: Object.entries(s.env ?? {}).map(([k, v]) => `${k}=${v}`).join("\n"), url: s.url ?? "", headers: s.headers, courses: (s.courseIds ?? []).join(", "),
});
const parseEnv = (text: string): Record<string, string> =>
Object.fromEntries(text.split("\n").map((l) => l.trim()).filter((l) => l.includes("=")).map((l) => [l.slice(0, l.indexOf("=")).trim(), l.slice(l.indexOf("=") + 1)]));
const fromDraft = (d: ServerDraft): McpServer => ({
id: d.id, name: d.name || d.id, transport: d.transport,
...(d.transport === "stdio" ? { command: d.command, args: d.args.trim() ? d.args.trim().split(/\s+/) : [], env: parseEnv(d.env) } : { url: d.url, ...(d.headers ? { headers: d.headers } : {}) }),
courseIds: d.courses.split(",").map((s) => s.trim()).filter(Boolean),
});
export default function AdminAiPage() {
const [loaded, setLoaded] = useState<Config | null>(null);
const [denied, setDenied] = useState<string | null>(null);
const [kind, setKind] = useState<Kind>("");
const [model, setModel] = useState("");
const [baseUrl, setBaseUrl] = useState("");
const [effort, setEffort] = useState("");
const [apiKey, setApiKey] = useState("");
const [clearKey, setClearKey] = useState(false);
const [servers, setServers] = useState<ServerDraft[]>([]);
const [allowed, setAllowed] = useState("");
const [msg, setMsg] = useState<{ ok: boolean; text: string } | null>(null);
const apply = (c: Config) => {
setLoaded(c);
setKind(c.provider?.kind ?? "");
setModel(c.provider?.model ?? "");
setBaseUrl(c.provider?.baseUrl ?? "");
setEffort(c.provider?.effort ?? "");
setApiKey("");
setClearKey(false);
setServers(c.mcpServers.map(toDraft));
setAllowed((c.allowedBaseUrls ?? []).join("\n"));
};
useEffect(() => {
api<Config>("/api/ai/admin/config").then(apply, (e: ApiError) => setDenied(e.status === 401 ? "Sign in as the owner to change AI settings." : e.message));
}, []);
const save = async () => {
setMsg(null);
try {
const provider = kind ? { kind, model, baseUrl, effort, apiKey: apiKey || undefined, clearApiKey: clearKey } : null;
apply(await api<Config>("/api/ai/admin/config", { method: "PUT", body: { provider, mcpServers: servers.map(fromDraft), allowedBaseUrls: allowed.split(/[\s,]+/).filter(Boolean) } }));
setMsg({ ok: true, text: kind ? "Saved. The assistant is available to learners." : "Saved. AI is off for everyone without their own key." });
} catch (e) {
setMsg({ ok: false, text: (e as Error).message });
}
};
const setServer = (i: number, patch: Partial<ServerDraft>) => setServers((s) => s.map((x, j) => (j === i ? { ...x, ...patch } : x)));
const testServer = async (i: number) => {
setServer(i, { test: "Connecting…" });
try {
const r = await api<{ ok: boolean; tools?: { name: string }[]; error?: string }>("/api/ai/admin/mcp/test", { method: "POST", body: fromDraft(servers[i]!) });
setServer(i, { test: r.ok ? `Connected. Tools: ${r.tools!.map((t) => t.name).join(", ") || "none"}` : `Failed: ${r.error}` });
} catch (e) {
setServer(i, { test: `Failed: ${(e as Error).message}` });
}
};
if (denied) return <div className="ai-page"><PageHead title="AI settings" /><p className="ai-note">{denied}</p></div>;
if (!loaded) return <div className="ai-page"><PageHead title="AI settings" /></div>;
return (
<div className="ai-page">
<PageHead title="AI settings">
<p>AI is off until you choose a provider here. Learners and authors can also add their own key, which is used only for their own requests. Keys are encrypted on the server and never shown again.</p>
</PageHead>
<form onSubmit={(e) => { e.preventDefault(); save(); }}>
<section className="ai-section">
<h2>Server provider</h2>
<span className="ai-hint">Used for everyone who has not added their own key.</span>
<div className="ai-grid">
<Field label="Provider">
<select className="ai-select" value={kind} onChange={(e) => setKind(e.target.value as Kind)}>
<option value="">None (AI off)</option>
<option value="anthropic">Anthropic</option>
<option value="openai">OpenAI-compatible (OpenAI, Ollama, vLLM …)</option>
</select>
</Field>
{kind ? (
<Field label="Model" hint={kind === "anthropic" ? "Leave empty for claude-opus-5-5." : "For example llama3.1 on Ollama."}>
<input className="ai-input" value={model} onChange={(e) => setModel(e.target.value)} placeholder={kind === "anthropic" ? "claude-opus-5-5" : "model name"} />
</Field>
) : null}
</div>
{kind ? (
<>
<Field label={kind === "anthropic" ? "Base URL (optional)" : "Base URL"} hint={kind === "openai" ? "The address that ends before /chat/completions, for example http://ollama:11434/v1." : "Leave empty for api.anthropic.com."}>
<input className="ai-input ai-mono" value={baseUrl} onChange={(e) => setBaseUrl(e.target.value)} placeholder={kind === "openai" ? "http://localhost:11434/v1" : "https://api.anthropic.com"} />
</Field>
<div className="ai-grid">
<Field label="API key" hint={loaded.provider?.hasApiKey ? "A key is saved. Leave empty to keep it." : kind === "openai" ? "Optional for a local Ollama." : undefined}>
<input className="ai-input ai-mono" type="password" autoComplete="off" value={apiKey} onChange={(e) => setApiKey(e.target.value)} placeholder={loaded.provider?.hasApiKey ? "••••••••" : "paste key"} />
</Field>
{kind === "anthropic" ? (
<Field label="Effort" hint="How hard the model thinks. Empty uses the model's default.">
<select className="ai-select" value={effort} onChange={(e) => setEffort(e.target.value)}>
<option value="">Default</option>
<option value="low">Low</option>
<option value="medium">Medium</option>
<option value="high">High</option>
</select>
</Field>
) : null}
</div>
{loaded.provider?.hasApiKey ? (
<label className="ai-check"><input type="checkbox" checked={clearKey} onChange={(e) => setClearKey(e.target.checked)} /> Remove the saved key</label>
) : null}
</>
) : null}
</section>
<section className="ai-section">
<h2>Endpoints for learners' own keys</h2>
<span className="ai-hint">Own keys go to api.anthropic.com or api.openai.com only. To let people use another endpoint, such as a shared Ollama, list its origin here (one per line, for example http://ollama.internal:11434). STC_AI_ALLOWED_BASE_URLS adds more.</span>
<Field label="Allowed origins">
<textarea className="ai-input ai-mono" rows={2} value={allowed} onChange={(e) => setAllowed(e.target.value)} placeholder="http://ollama.internal:11434" />
</Field>
</section>
<section className="ai-section">
<h2>MCP servers</h2>
<span className="ai-hint">Extra tools for the assistant. Leave "Courses" empty to offer a server in every course.</span>
{servers.map((s, i) => (
<div className="ai-server" key={i}>
<div className="ai-grid">
<Field label="Id"><input className="ai-input ai-mono" value={s.id} onChange={(e) => setServer(i, { id: e.target.value })} /></Field>
<Field label="Name"><input className="ai-input" value={s.name} onChange={(e) => setServer(i, { name: e.target.value })} /></Field>
<Field label="Transport">
<select className="ai-select" value={s.transport} onChange={(e) => setServer(i, { transport: e.target.value as "stdio" | "http" })}>
<option value="http">Streamable HTTP</option>
<option value="stdio">Local command (stdio)</option>
</select>
</Field>
</div>
{s.transport === "http" ? (
<Field label="URL"><input className="ai-input ai-mono" value={s.url} onChange={(e) => setServer(i, { url: e.target.value })} placeholder="https://mcp.example.com/mcp" /></Field>
) : (
<div className="ai-grid">
<Field label="Command"><input className="ai-input ai-mono" value={s.command} onChange={(e) => setServer(i, { command: e.target.value })} placeholder="bunx" /></Field>
<Field label="Arguments"><input className="ai-input ai-mono" value={s.args} onChange={(e) => setServer(i, { args: e.target.value })} placeholder="some-mcp-server --flag" /></Field>
</div>
)}
{s.transport === "stdio" ? (
<Field label="Environment" hint="One NAME=value per line. The command gets only PATH, HOME, LANG and these; server secrets are never passed on.">
<textarea className="ai-input ai-mono" rows={2} value={s.env} onChange={(e) => setServer(i, { env: e.target.value })} placeholder="API_TOKEN=…" />
</Field>
) : null}
<Field label="Courses" hint="Course ids, separated by commas."><input className="ai-input ai-mono" value={s.courses} onChange={(e) => setServer(i, { courses: e.target.value })} /></Field>
<div className="ai-footer" style={{ margin: "0 0 10px" }}>
<Pill onClick={() => testServer(i)}>Test connection</Pill>
<Pill variant="ghost" onClick={() => setServers((x) => x.filter((_, j) => j !== i))}>Remove</Pill>
{s.test ? <span className="ai-hint">{s.test}</span> : null}
</div>
</div>
))}
<div className="ai-footer">
<Pill onClick={() => setServers((x) => [...x, { id: `server-${x.length + 1}`, name: "", transport: "http", command: "", args: "", env: "", url: "", courses: "" }])}>Add MCP server</Pill>
</div>
</section>
<div className="ai-footer">
<Pill variant="primary" type="submit">Save AI settings</Pill>
{msg ? <span className={msg.ok ? "ai-hint ai-ok" : "ai-error"} role="status" style={{ margin: 0 }}>{msg.text}</span> : null}
</div>
</form>
</div>
);
}
packages/feature-ai/src/web/AssistantPage.tsx
+205
View File
@@ -0,0 +1,205 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { Link, useParams } from "react-router-dom";
import { Pill, SegmentedToggle } from "@stc/ui";
import { api, readNdjson, type ChatEvent, type Citation, type Status } from "./api";
import { courseUrl, lessonUrl } from "./urls";
import { Markdown, MentorBanner, PageHead } from "./components";
type Turn = { role: "user" | "assistant"; text: string; citations?: Citation[]; tools?: string[]; error?: string; notices?: string[] };
type ConvSummary = { id: string; title: string | null; createdAt: string };
const TOOL_LABEL: Record<string, string> = { search_course: "searched the course", read_lesson: "read a lesson", read_file: "read a file", list_notes: "read your notes" };
const toolLabel = (n: string) => TOOL_LABEL[n] ?? `used ${n.replace(/^mcp_[^_]+__/, "")}`;
function Sources({ courseId, citations }: { courseId: string; citations: Citation[] }) {
// One row per file; a file shared by several lessons links to the first one.
const rows = [...new Map(citations.map((c) => [c.path, c] as const)).values()];
if (!rows.length) return null;
return (
<div className="ai-sources">
<span>Sources</span>
<ul className="ai-rows">
{rows.map((c) => (
<li key={c.path}>
<span className="ai-path">{c.path}</span>
{c.lessonId ? <Link className="ai-meta" to={lessonUrl(courseId, c.lessonId)}>{c.lessonTitle ?? c.lessonId}</Link> : <span className="ai-meta">{c.kind}</span>}
</li>
))}
</ul>
</div>
);
}
export default function AssistantPage() {
const { courseId = "" } = useParams();
const [status, setStatus] = useState<Status | null>(null);
const [loadError, setLoadError] = useState<string | null>(null);
const [turns, setTurns] = useState<Turn[]>([]);
const [conversationId, setConversationId] = useState<string | null>(null);
const [history, setHistory] = useState<ConvSummary[]>([]);
const [draft, setDraft] = useState("");
const [busy, setBusy] = useState(false);
const abortRef = useRef<AbortController | null>(null);
const lessonId = new URLSearchParams(typeof location === "undefined" ? "" : location.search).get("lesson") ?? undefined;
const loadHistory = useCallback(async () => {
try { setHistory(await api<ConvSummary[]>(`/api/ai/courses/${encodeURIComponent(courseId)}/conversations`)); } catch { setHistory([]); }
}, [courseId]);
useEffect(() => {
api<Status>(`/api/ai/status?courseId=${encodeURIComponent(courseId)}`).then(setStatus, (e) => setLoadError(e.message));
loadHistory();
const linked = new URLSearchParams(location.search).get("c");
if (linked) openConversation(linked).catch(() => {});
return () => abortRef.current?.abort();
}, [courseId, loadHistory]);
const mentor = status?.mentor ?? { available: false, on: true };
const setMentor = async (on: boolean) => {
if (!status) return;
setStatus({ ...status, mentor: { ...mentor, on } });
try {
const r = await api<{ mentor: Status["mentor"] }>(`/api/ai/courses/${encodeURIComponent(courseId)}/mentor`, { method: "PUT", body: { off: !on } });
setStatus((s) => (s ? { ...s, mentor: r.mentor } : s));
} catch (e) {
setStatus({ ...status, mentor });
setLoadError((e as Error).message);
}
};
async function openConversation(id: string) {
const c = await api<{ id: string; messages: { role: "user" | "assistant"; content: string; citations?: Citation[] }[] }>(`/api/ai/conversations/${id}`);
setConversationId(c.id);
setTurns(c.messages.map((m) => ({ role: m.role, text: m.content, citations: m.citations })));
}
const deleteConversation = async (id: string) => {
await api(`/api/ai/conversations/${id}`, { method: "DELETE" });
if (id === conversationId) { setConversationId(null); setTurns([]); }
loadHistory();
};
const send = async () => {
const message = draft.trim();
if (!message || busy) return;
setDraft("");
setBusy(true);
const update = (fn: (t: Turn) => Turn) => setTurns((ts) => [...ts.slice(0, -1), fn(ts[ts.length - 1]!)]);
setTurns((ts) => [...ts, { role: "user", text: message }, { role: "assistant", text: "", tools: [] }]);
const ctl = new AbortController();
abortRef.current = ctl;
try {
const res = await fetch(`/api/ai/courses/${encodeURIComponent(courseId)}/chat`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ message, conversationId, lessonId }),
signal: ctl.signal,
});
if (!res.ok || !res.body) {
let msg = res.statusText;
try { msg = (await res.json()).error ?? msg; } catch {}
update((t) => ({ ...t, error: msg }));
return;
}
for await (const ev of readNdjson<ChatEvent>(res.body)) {
if (ev.type === "meta") { setConversationId(ev.conversationId); update((t) => ({ ...t, citations: ev.citations })); }
else if (ev.type === "text") update((t) => ({ ...t, text: t.text + ev.text }));
else if (ev.type === "tool") update((t) => ({ ...t, tools: [...(t.tools ?? []), ev.name] }));
else if (ev.type === "notice") update((t) => ({ ...t, notices: [...(t.notices ?? []), ev.message] }));
else if (ev.type === "error") update((t) => ({ ...t, error: ev.message }));
else if (ev.type === "done") update((t) => ({ ...t, text: ev.text || t.text, citations: ev.citations }));
}
loadHistory();
} catch (e) {
if ((e as Error).name !== "AbortError") update((t) => ({ ...t, error: (e as Error).message }));
} finally {
setBusy(false);
abortRef.current = null;
}
};
const newConversation = () => { abortRef.current?.abort(); setConversationId(null); setTurns([]); };
const courseTitle = status?.course?.title ?? courseId;
const chap = <Link to={courseUrl(courseId)}>{courseTitle}</Link>;
if (loadError && !status) return <div className="ai-page"><PageHead chap={chap} title="Course assistant" /><p className="ai-error">{loadError}</p></div>;
if (!status) return <div className="ai-page"><PageHead chap={chap} title="Course assistant" /></div>;
if (!status.course) return <div className="ai-page"><PageHead title="Course assistant" /><p className="ai-note">This course was not found.</p></div>;
if (!status.signedIn) {
return (
<div className="ai-page">
<PageHead chap={chap} title="Course assistant" />
<p className="ai-note">Sign in to ask the assistant about this course.</p>
</div>
);
}
if (!status.enabled) {
return (
<div className="ai-page">
<PageHead chap={chap} title="Course assistant" />
<p className="ai-note">
The assistant is off. AI stays off until an admin sets up a server provider, or you add your own API key in <Link to="/account/ai">Account, AI</Link>. Your key is used only for your own questions.
</p>
</div>
);
}
return (
<div className="ai-page">
<PageHead chap={chap} title="Course assistant">
<p>Answers come from this course's lessons, transcripts, slides, code and your notes, with the sources listed.{status.source === "byok" ? " Using your own key." : ""}</p>
</PageHead>
{mentor.available ? (
<div className="ai-bar">
<SegmentedToggle label="Mentor mode" value={mentor.on ? "on" : "off"} onChange={(v) => setMentor(v === "on")} options={[{ value: "on", label: "Mentor on" }, { value: "off", label: "Mentor off" }]} />
<span className="ai-hint">{mentor.on ? "Follows the course's tutoring rules: questions and hints first, no full solutions." : "The course's tutoring rules are off for you in this course."}</span>
</div>
) : null}
<MentorBanner on={mentor.on} available={mentor.available} />
{loadError ? <p className="ai-error">{loadError}</p> : null}
{turns.length ? (
<ol className="ai-thread" aria-live="polite">
{turns.map((t, i) => (
<li key={i} className={`ai-turn ai-turn--${t.role}`}>
<span className="ai-who">{t.role === "user" ? "You" : "Assistant"}</span>
{t.text ? <Markdown text={t.text} /> : t.role === "assistant" && busy && i === turns.length - 1 && !t.error ? <p className="ai-activity">Thinking…</p> : null}
{t.tools?.length ? <p className="ai-activity">{[...new Set(t.tools.map(toolLabel))].join(", ").replace(/^./, (c) => c.toUpperCase())}.</p> : null}
{t.notices?.map((n) => <p key={n} className="ai-activity">{n}</p>)}
{t.error ? <p className="ai-error">{t.error}</p> : null}
{t.role === "assistant" && t.citations ? <Sources courseId={courseId} citations={t.citations} /> : null}
</li>
))}
</ol>
) : (
<p className="ai-note">Ask about anything in the course: an idea you are unsure of, a failing test, or "quiz me on this module".</p>
)}
<form className="ai-composer" onSubmit={(e) => { e.preventDefault(); send(); }}>
<label className="sr" htmlFor="ai-message">Your question</label>
<textarea id="ai-message" className="ai-textarea" value={draft} onChange={(e) => setDraft(e.target.value)} placeholder="Ask a question about this course" rows={4} />
<div className="ai-actions">
{turns.length ? <Pill variant="ghost" onClick={newConversation}>New conversation</Pill> : null}
{busy ? <Pill onClick={() => abortRef.current?.abort()}>Stop</Pill> : null}
<Pill variant="primary" type="submit" disabled={busy || !draft.trim()}>Send</Pill>
</div>
</form>
{history.length ? (
<section className="ai-history">
<h2>Earlier conversations</h2>
<ul className="ai-rows">
{history.map((h) => (
<li key={h.id}>
<button type="button" className="ai-linkbtn" aria-current={h.id === conversationId} onClick={() => openConversation(h.id)}>{h.title || "Untitled"}</button>
<Pill variant="ghost" onClick={() => deleteConversation(h.id)} aria-label={`Delete conversation ${h.title ?? ""}`}>Delete</Pill>
</li>
))}
</ul>
</section>
) : null}
</div>
);
}
packages/feature-ai/src/web/GeneratePage.tsx
+199
View File
@@ -0,0 +1,199 @@
import { useEffect, useMemo, useState } from "react";
import { Pill, ProgressBar, SegmentedToggle } from "@stc/ui";
import { api, ApiError, type Status } from "./api";
import { Field, PageHead } from "./components";
interface CourseOpt { id: string; title: string }
interface Sources { lessons: { id: string; title: string; type: string; module: string; hasText: boolean }[]; files: string[]; cache: string[] }
interface Job { id: string; status: "queued" | "running" | "done" | "failed"; progress: number; message: string | null; error: string | null; result: { location: string; branch?: string; commit?: string; paths: string[]; warnings: string[] } | null }
const KINDS = [
{ value: "document", label: "Document" },
{ value: "exercise", label: "Exercise" },
{ value: "questions", label: "Questions" },
];
const STYLES = [
{ value: "pure-snake", label: "Module shape" },
{ value: "systemic-design", label: "Observe to Expansion Drop" },
];
const STYLE_HINT: Record<string, string> = {
"pure-snake": "Pure Snake's shape: why, objectives, concepts, a puzzle before the explanation, exercises with nudge, stronger hint and near-solution, and check questions.",
"systemic-design": "Systemic Design's beats: Observe, Predict, Spec, Build, Name it, Expansion Drop, Further reading.",
};
export default function GeneratePage() {
const [courses, setCourses] = useState<CourseOpt[] | null>(null);
const [denied, setDenied] = useState<string | null>(null);
const [enabled, setEnabled] = useState(true);
const [courseId, setCourseId] = useState("");
const [src, setSrc] = useState<Sources | null>(null);
const [kind, setKind] = useState("document");
const [style, setStyle] = useState("pure-snake");
const [topic, setTopic] = useState("");
const [instructions, setInstructions] = useState("");
const [lessons, setLessons] = useState<Set<string>>(new Set());
const [files, setFiles] = useState<Set<string>>(new Set());
const [cache, setCache] = useState<Set<string>>(new Set());
const [pasted, setPasted] = useState("");
const [filter, setFilter] = useState("");
const [job, setJob] = useState<Job | null>(null);
const [error, setError] = useState<string | null>(null);
useEffect(() => {
api<CourseOpt[]>("/api/ai/studio/courses").then((c) => {
setCourses(c);
if (c[0]) setCourseId(c[0].id);
}, (e: ApiError) => setDenied(e.status === 401 ? "Sign in as an author to generate drafts." : e.message));
api<Status>("/api/ai/status").then((s) => setEnabled(s.enabled), () => {});
}, []);
useEffect(() => {
if (!courseId) return;
setSrc(null);
setLessons(new Set()); setFiles(new Set()); setCache(new Set());
api<Sources>(`/api/ai/studio/sources?courseId=${encodeURIComponent(courseId)}`).then(setSrc, (e) => setError(e.message));
}, [courseId]);
useEffect(() => {
if (!job || job.status === "done" || job.status === "failed") return;
const t = setTimeout(() => api<Job>(`/api/ai/studio/jobs/${job.id}`).then(setJob, (e) => setError(e.message)), 1000);
return () => clearTimeout(t);
}, [job]);
const toggle = (set: Set<string>, setter: (s: Set<string>) => void, v: string) => {
const n = new Set(set);
if (n.has(v)) n.delete(v); else n.add(v);
setter(n);
};
const byModule = useMemo(() => {
const m = new Map<string, Sources["lessons"]>();
for (const l of src?.lessons ?? []) m.set(l.module, [...(m.get(l.module) ?? []), l]);
return [...m];
}, [src]);
const shownFiles = (src?.files ?? []).filter((f) => !filter || f.toLowerCase().includes(filter.toLowerCase())).slice(0, 200);
const sourceCount = lessons.size + files.size + cache.size + (pasted.trim() ? 1 : 0);
const generate = async () => {
setError(null);
setJob(null);
const sources = [
...[...lessons].map((lessonId) => ({ kind: "lesson", lessonId })),
...[...files].map((path) => ({ kind: "file", path })),
...[...cache].map((path) => ({ kind: "cache", path })),
...(pasted.trim() ? [{ kind: "text", label: "Pasted text", text: pasted }] : []),
];
try {
const { jobId } = await api<{ jobId: string }>("/api/ai/studio/generate", { method: "POST", body: { courseId, kind, style, topic, instructions, sources } });
setJob({ id: jobId, status: "queued", progress: 0, message: "Queued", error: null, result: null });
} catch (e) { setError((e as Error).message); }
};
if (denied) return <div className="ai-page"><PageHead chap="Studio" title="Generate a draft" /><p className="ai-note">{denied}</p></div>;
if (!courses) return <div className="ai-page"><PageHead chap="Studio" title="Generate a draft" /></div>;
if (!courses.length) return <div className="ai-page"><PageHead chap="Studio" title="Generate a draft" /><p className="ai-note">You are not an author of any course yet.</p></div>;
const running = job && (job.status === "queued" || job.status === "running");
return (
<div className="ai-page">
<PageHead chap="Studio" title="Generate a draft">
<p>Drafts a document, an exercise or knowledge-check questions from material you choose. The result is written as draft files for you to review and edit. Nothing is published.</p>
</PageHead>
{!enabled ? <p className="ai-banner">AI is not set up. Ask the owner to configure a provider, or add your own key in Account, AI.</p> : null}
<form onSubmit={(e) => { e.preventDefault(); generate(); }}>
<div className="ai-grid">
<Field label="Course">
<select className="ai-select" value={courseId} onChange={(e) => setCourseId(e.target.value)}>
{courses.map((c) => <option key={c.id} value={c.id}>{c.title}</option>)}
</select>
</Field>
<Field label="Topic">
<input className="ai-input" value={topic} onChange={(e) => setTopic(e.target.value)} placeholder="For example: Folding time" required />
</Field>
</div>
<div className="ai-bar">
<SegmentedToggle label="What to draft" value={kind} onChange={setKind} options={KINDS} />
<SegmentedToggle label="House style" value={style} onChange={setStyle} options={STYLES} />
</div>
<p className="ai-hint" style={{ margin: "-6px 0 18px" }}>{STYLE_HINT[style]}</p>
<section className="ai-section">
<h2>Material</h2>
<span className="ai-hint">Lessons and transcripts, code and text files, slide text from imports, or text you paste. {sourceCount ? `${sourceCount} selected.` : ""}</span>
{!src ? <p className="ai-hint">Loading…</p> : (
<>
<div className="ai-checklist" role="group" aria-label="Lessons">
{byModule.map(([mod, ls]) => (
<div key={mod}>
<div className="ai-group">{mod}</div>
{ls.map((l) => (
<label className="ai-check" key={l.id}>
<input type="checkbox" disabled={!l.hasText} checked={lessons.has(l.id)} onChange={() => toggle(lessons, setLessons, l.id)} />
<span>{l.title} <small>{l.type}{l.hasText ? "" : ", no transcript"}</small></span>
</label>
))}
</div>
))}
</div>
{src.files.length ? (
<>
<Field label="Files">
<input className="ai-input" value={filter} onChange={(e) => setFilter(e.target.value)} placeholder="Filter files" />
</Field>
<div className="ai-checklist" role="group" aria-label="Files">
{shownFiles.map((f) => (
<label className="ai-check" key={f}><input type="checkbox" checked={files.has(f)} onChange={() => toggle(files, setFiles, f)} /><span className="ai-mono">{f}</span></label>
))}
</div>
</>
) : null}
{src.cache.length ? (
<div className="ai-checklist" role="group" aria-label="Slide text and transcripts">
<div className="ai-group">Slide text and transcripts</div>
{src.cache.map((f) => (
<label className="ai-check" key={f}><input type="checkbox" checked={cache.has(f)} onChange={() => toggle(cache, setCache, f)} /><span className="ai-mono">{f}</span></label>
))}
</div>
) : null}
</>
)}
<Field label="Pasted text" hint="A transcript, notes or anything else to draw on.">
<textarea className="ai-textarea" value={pasted} onChange={(e) => setPasted(e.target.value)} />
</Field>
<Field label="Instructions" hint="Optional. For example: aim at Module 5, two exercises, JavaScript only.">
<textarea className="ai-textarea" value={instructions} onChange={(e) => setInstructions(e.target.value)} rows={3} />
</Field>
</section>
<div className="ai-footer">
<Pill variant="primary" type="submit" disabled={!enabled || !topic.trim() || !!running}>Generate draft</Pill>
{error ? <span className="ai-error" style={{ margin: 0 }}>{error}</span> : null}
</div>
</form>
{job ? (
<section className="ai-section" aria-live="polite">
<h2>{job.status === "done" ? "Draft written" : job.status === "failed" ? "Generation failed" : "Generating"}</h2>
{running ? <div className="ai-progress"><ProgressBar value={job.progress} label="Generation progress" /><span>{job.message ?? "Queued"}</span></div> : null}
{job.status === "failed" ? <p className="ai-error">{job.error}</p> : null}
{job.result ? (
<>
{job.result.branch ? (
<span className="ai-hint">Committed to your draft branch <span className="ai-mono">{job.result.location}</span>. Review it in the studio before publishing.</span>
) : (
<span className="ai-hint">This course is not edited in the studio, so the draft was saved as files on the server at <span className="ai-mono">{job.result.location}</span>. Nothing in the course changed; copy what you want into it yourself.</span>
)}
<ul className="ai-rows">{job.result.paths.map((p) => <li key={p}><span className="ai-path">{p}</span><span className="ai-meta">draft</span></li>)}</ul>
{job.result.warnings.length ? (
<div className="ai-section">
<h2>Check these</h2>
<ul className="ai-rows">{job.result.warnings.map((w) => <li key={w}><span>{w}</span><span /></li>)}</ul>
</div>
) : null}
</>
) : null}
</section>
) : null}
</div>
);
}
packages/feature-ai/src/web/ai.css
+77
View File
@@ -0,0 +1,77 @@
/* AI feature: assistant panel, settings and generator. Tokens only (packages/ui/src/tokens.css). */
.ai-page { max-width: 760px; margin: 0 auto 64px; font-size: 14px; }
.ai-head { margin: 0 0 22px; padding-bottom: 20px; border-bottom: 1px solid var(--line); }
.ai-chap { font-size: 13px; font-weight: 500; color: var(--muted); margin: 0 0 10px; }
.ai-chap a { text-decoration: none; }
.ai-chap a:hover { color: var(--head); }
.ai-title { font: 600 28px/1.15 var(--sans); color: var(--head); margin: 0; letter-spacing: -.01em; }
.ai-lede { margin: 10px 0 0; color: var(--muted); font-size: 14px; line-height: 1.55; }
.ai-lede p { margin: 0; }
.ai-bar { display: flex; flex-wrap: wrap; align-items: center; gap: 10px 14px; margin: 0 0 18px; }
.ai-bar .ai-hint { flex: 1; min-width: 200px; }
.ai-banner { margin: 0 0 18px; padding: 10px 14px; border: 1px solid var(--line2); border-radius: 10px; background: var(--side); color: var(--head); font-size: 13.5px; }
.ai-note { margin: 0 0 18px; color: var(--muted); font-size: 13.5px; line-height: 1.55; }
.ai-note a { color: var(--text); }
.ai-error { margin: 0 0 14px; color: var(--bad); font-size: 13.5px; }
.ai-ok { color: var(--ok); }
/* conversation */
.ai-thread { list-style: none; margin: 0 0 18px; padding: 0; border-top: 1px solid var(--line); }
.ai-turn { padding: 16px 2px; border-bottom: 1px solid var(--line); }
.ai-who { display: block; font-size: 12px; color: var(--faint); margin: 0 0 6px; }
.ai-turn--user .ai-md { color: var(--head); }
.ai-md { font-size: 15px; line-height: 1.6; color: var(--text); overflow-wrap: anywhere; }
.ai-md p, .ai-md ul, .ai-md ol { margin: 0 0 .8em; }
.ai-md > :last-child { margin-bottom: 0; }
.ai-md li { margin-bottom: .25em; }
.ai-md .ai-md-h { font-weight: 600; color: var(--head); }
.ai-md :not(pre) > code { font-size: .85em; padding: .08em .32em; background: color-mix(in srgb, var(--raised) 75%, transparent); border-radius: 4px; }
.ai-md pre.code, pre.code { font-size: 13px; line-height: 1.6; background: var(--side); border: 1px solid var(--line); border-radius: 10px; padding: 12px 14px; margin: 0 0 .8em; overflow-x: auto; white-space: pre; }
.ai-activity { margin: 8px 0 0; font-size: 12.5px; color: var(--faint); }
.ai-sources { margin: 12px 0 0; }
.ai-sources > span { display: block; font-size: 12px; color: var(--faint); margin: 0 0 4px; }
/* rows (same rhythm as the mockup's .rows) */
.ai-rows { list-style: none; margin: 0; padding: 0; border-top: 1px solid var(--line); }
.ai-rows li { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: 10px; align-items: center; padding: 9px 2px; border-bottom: 1px solid var(--line); font-size: 13.5px; }
.ai-rows .ai-path { font: 12.5px/1.4 var(--mono); color: var(--text); overflow-wrap: anywhere; }
.ai-rows .ai-meta { font-size: 12px; color: var(--faint); white-space: nowrap; }
.ai-rows a { text-decoration: none; }
.ai-rows a:hover { text-decoration: underline; text-underline-offset: 3px; }
/* composer and forms */
.ai-composer { display: grid; gap: 10px; }
.ai-composer .ai-actions { display: flex; gap: 8px; justify-content: flex-end; }
.ai-input, .ai-select, .ai-textarea { width: 100%; font-family: inherit; background: var(--side); border: 1px solid var(--line2); border-radius: 10px; padding: 9px 12px; color: var(--text); font-size: 14px; line-height: 1.5; }
.ai-textarea { min-height: 96px; resize: vertical; }
.ai-input:focus, .ai-select:focus, .ai-textarea:focus { outline: 0; border-color: var(--faint); }
.ai-input:focus-visible, .ai-select:focus-visible, .ai-textarea:focus-visible { outline: 2px solid var(--accent); outline-offset: 1px; }
.ai-mono { font-family: var(--mono); font-size: 13px; }
.ai-field { display: grid; gap: 5px; margin: 0 0 14px; align-content: start; }
.ai-label { font-size: 12.5px; font-weight: 500; color: var(--muted); }
.ai-hint { font-size: 12.5px; color: var(--faint); line-height: 1.45; }
.ai-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: 0 16px; align-items: start; }
.ai-section { margin: 30px 0 0; }
.ai-section > h2 { font: 600 15px/1.3 var(--sans); color: var(--head); margin: 0 0 4px; }
.ai-section > .ai-hint { display: block; margin: 0 0 12px; }
.ai-check { display: flex; gap: 10px; align-items: baseline; padding: 6px 2px; font-size: 13.5px; color: var(--text); cursor: pointer; }
.ai-check input { accent-color: var(--accent); }
.ai-check small { color: var(--faint); font-size: 12px; }
.ai-checklist { max-height: 300px; overflow-y: auto; border-top: 1px solid var(--line); border-bottom: 1px solid var(--line); padding: 4px 0; margin: 0 0 14px; }
.ai-group { font-size: 12px; color: var(--faint); padding: 8px 2px 2px; }
.ai-server { border-top: 1px solid var(--line); padding: 14px 0 4px; }
.ai-server:last-of-type { border-bottom: 1px solid var(--line); }
.ai-footer { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; margin: 22px 0 0; }
.ai-progress { display: flex; align-items: center; gap: 12px; margin: 18px 0 0; font-size: 13px; color: var(--muted); }
.ai-progress .stc-bar { max-width: 260px; }
.ai-history { margin: 34px 0 0; }
.ai-history h2 { font: 600 15px/1.3 var(--sans); color: var(--head); margin: 0 0 10px; }
.ai-linkbtn { background: none; border: 0; padding: 0; color: var(--text); text-align: left; font-size: 13.5px; }
.ai-linkbtn:hover { text-decoration: underline; text-underline-offset: 3px; }
.ai-linkbtn[aria-current="true"] { color: var(--head); font-weight: 500; }
@media (max-width: 640px) {
.ai-title { font-size: 24px; }
.ai-rows li { grid-template-columns: minmax(0, 1fr); }
}
packages/feature-ai/src/web/api.ts
+51
View File
@@ -0,0 +1,51 @@
export class ApiError extends Error {
constructor(public status: number, message: string) { super(message); }
}
export async function api<T>(path: string, init: { method?: string; body?: unknown } = {}): Promise<T> {
const res = await fetch(path, {
method: init.method ?? "GET",
headers: init.body === undefined ? undefined : { "content-type": "application/json" },
body: init.body === undefined ? undefined : JSON.stringify(init.body),
credentials: "same-origin",
});
if (!res.ok) {
let msg = res.statusText;
try { msg = (await res.json()).error ?? msg; } catch {}
throw new ApiError(res.status, msg);
}
return res.json() as Promise<T>;
}
/** Parses a newline-delimited JSON stream into objects. */
export async function* readNdjson<T = unknown>(body: ReadableStream<Uint8Array>): AsyncIterable<T> {
const reader = body.getReader();
const decoder = new TextDecoder();
let buf = "";
for (;;) {
const { value, done } = await reader.read();
if (value) buf += decoder.decode(value, { stream: true });
let nl: number;
while ((nl = buf.indexOf("\n")) !== -1) {
const line = buf.slice(0, nl).trim();
buf = buf.slice(nl + 1);
if (line) yield JSON.parse(line) as T;
}
if (done) break;
}
buf += decoder.decode();
if (buf.trim()) yield JSON.parse(buf) as T;
}
export interface Citation { path: string; kind: string; lessonId?: string; lessonTitle?: string }
export interface Status {
enabled: boolean; source: "byok" | "server" | null; signedIn: boolean;
course?: { id: string; title: string }; mentor?: { available: boolean; on: boolean };
}
export type ChatEvent =
| { type: "meta"; conversationId: string; mentorOn: boolean; citations: Citation[] }
| { type: "text"; text: string }
| { type: "tool"; name: string }
| { type: "notice"; message: string }
| { type: "error"; message: string }
| { type: "done"; text: string; citations: Citation[] };
packages/feature-ai/src/web/components.tsx
+77
View File
@@ -0,0 +1,77 @@
import { Fragment, type ReactNode } from "react";
import "./ai.css";
/** Shown whenever the learner has switched the course's mentor rules off. */
export function MentorBanner({ on, available }: { on: boolean; available: boolean }) {
if (on || !available) return null;
return <p className="ai-banner" role="status">Mentor mode off: I'll give full answers</p>;
}
export function Field({ label, hint, children }: { label: string; hint?: string; children: ReactNode }) {
return (
<label className="ai-field">
<span className="ai-label">{label}</span>
{children}
{hint ? <span className="ai-hint">{hint}</span> : null}
</label>
);
}
export function PageHead({ chap, title, children }: { chap?: ReactNode; title: string; children?: ReactNode }) {
return (
<header className="ai-head">
{chap ? <p className="ai-chap">{chap}</p> : null}
<h1 className="ai-title">{title}</h1>
{children ? <div className="ai-lede">{children}</div> : null}
</header>
);
}
function inline(text: string, key: string): ReactNode[] {
const out: ReactNode[] = [];
const re = /(`[^`]+`|\*\*[^*]+\*\*)/g;
let last = 0;
let m: RegExpExecArray | null;
let i = 0;
while ((m = re.exec(text))) {
if (m.index > last) out.push(text.slice(last, m.index));
const tok = m[0];
out.push(tok.startsWith("`") ? <code key={`${key}-${i++}`}>{tok.slice(1, -1)}</code> : <strong key={`${key}-${i++}`}>{tok.slice(2, -2)}</strong>);
last = m.index + tok.length;
}
if (last < text.length) out.push(text.slice(last));
return out;
}
/** A small, safe Markdown subset for assistant answers: paragraphs, lists, headings, fenced code, `code`, **bold**. */
export function Markdown({ text }: { text: string }) {
const blocks: ReactNode[] = [];
const lines = text.replace(/\r\n/g, "\n").split("\n");
let i = 0;
let k = 0;
while (i < lines.length) {
const line = lines[i]!;
if (/^\s*```/.test(line)) {
const code: string[] = [];
i++;
while (i < lines.length && !/^\s*```/.test(lines[i]!)) code.push(lines[i++]!);
i++;
blocks.push(<pre className="code" key={k++}>{code.join("\n")}</pre>);
} else if (/^\s*([-*]|\d+\.)\s+/.test(line)) {
const ordered = /^\s*\d+\./.test(line);
const items: string[] = [];
while (i < lines.length && /^\s*([-*]|\d+\.)\s+/.test(lines[i]!)) items.push(lines[i++]!.replace(/^\s*([-*]|\d+\.)\s+/, ""));
const kids = items.map((it, j) => <li key={j}>{inline(it, `${k}-${j}`)}</li>);
blocks.push(ordered ? <ol key={k++}>{kids}</ol> : <ul key={k++}>{kids}</ul>);
} else if (/^#{1,6}\s/.test(line)) {
blocks.push(<p className="ai-md-h" key={k++}>{inline(line.replace(/^#+\s*/, ""), String(k))}</p>);
i++;
} else if (!line.trim()) i++;
else {
const para: string[] = [];
while (i < lines.length && lines[i]!.trim() && !/^\s*(```|[-*]\s|\d+\.\s|#{1,6}\s)/.test(lines[i]!)) para.push(lines[i++]!);
blocks.push(<p key={k++}>{para.map((p, j) => <Fragment key={j}>{j ? <br /> : null}{inline(p, `${k}-${j}`)}</Fragment>)}</p>);
}
}
return <div className="ai-md">{blocks}</div>;
}
packages/feature-ai/src/web/index.ts
+12
View File
@@ -0,0 +1,12 @@
import type { WebFeature } from "@stc/contracts";
export const aiWeb: WebFeature = {
name: "ai",
routes: [
{ path: "/c/:courseId/assistant", element: () => import("./AssistantPage") },
{ path: "/admin/ai", element: () => import("./AdminAiPage"), nav: { label: "AI", area: "admin", roles: ["owner"] } },
{ path: "/account/ai", element: () => import("./AccountAiPage") },
{ path: "/studio/generate", element: () => import("./GeneratePage") },
],
};
export { courseUrl, lessonUrl } from "./urls";
packages/feature-ai/src/web/urls.ts
+2
View File
@@ -0,0 +1,2 @@
// Learner URLs the AI pages link to; the learner owns the routes (shared helpers in @stc/contracts, request 7).
export { courseUrl, lessonUrl } from "@stc/contracts";
packages/feature-ai/test/anthropic.test.ts
+116
View File
@@ -0,0 +1,116 @@
import { expect, test } from "bun:test";
import type { ChatMessage } from "@stc/contracts";
import { createAnthropicProvider } from "../src/providers/anthropic";
import { collect, fakeFetch, sseBody } from "./helpers";
const ok = (events: [string | null, unknown][]) => () => new Response(sseBody(events), { headers: { "content-type": "text/event-stream" } });
const textTurn: [string, unknown][] = [
["message_start", { type: "message_start", message: { id: "m1", content: [] } }],
["content_block_start", { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }],
["ping", { type: "ping" }],
["content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "Hello " } }],
["content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "world" } }],
["content_block_stop", { type: "content_block_stop", index: 0 }],
["message_delta", { type: "message_delta", delta: { stop_reason: "end_turn" } }],
["message_stop", { type: "message_stop" }],
];
const toolTurn: [string, unknown][] = [
["message_start", { type: "message_start", message: { id: "m2", content: [] } }],
["content_block_start", { type: "content_block_start", index: 0, content_block: { type: "thinking", thinking: "", signature: "" } }],
["content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "thinking_delta", thinking: "" } }],
["content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "signature_delta", signature: "SIG-abc" } }],
["content_block_stop", { type: "content_block_stop", index: 0 }],
["content_block_start", { type: "content_block_start", index: 1, content_block: { type: "text", text: "" } }],
["content_block_delta", { type: "content_block_delta", index: 1, delta: { type: "text_delta", text: "Let me look." } }],
["content_block_stop", { type: "content_block_stop", index: 1 }],
["content_block_start", { type: "content_block_start", index: 2, content_block: { type: "tool_use", id: "toolu_1", name: "search_course", input: {} } }],
["content_block_delta", { type: "content_block_delta", index: 2, delta: { type: "input_json_delta", partial_json: "{\"que" } }],
["content_block_delta", { type: "content_block_delta", index: 2, delta: { type: "input_json_delta", partial_json: "ry\": \"add\"}" } }],
["content_block_stop", { type: "content_block_stop", index: 2 }],
["message_delta", { type: "message_delta", delta: { stop_reason: "tool_use" } }],
["message_stop", { type: "message_stop" }],
];
test("streams text and sends a well-formed request", async () => {
const ff = fakeFetch([ok(textTurn)]);
const p = createAnthropicProvider({ apiKey: "sk-test", fetch: ff.fetch });
const msgs: ChatMessage[] = [{ role: "system", content: "You are kind." }, { role: "user", content: "Hi" }];
const r = await collect(p.chat(msgs, { tools: [{ name: "search_course", description: "Search", inputSchema: { type: "object", properties: {} } }] }));
expect(r.text).toBe("Hello world");
const c = ff.calls[0]!;
expect(c.url).toBe("https://api.anthropic.com/v1/messages");
const h = new Headers(c.init.headers);
expect(h.get("x-api-key")).toBe("sk-test");
expect(h.get("anthropic-version")).toBe("2023-06-01");
expect(c.body.model).toBe("claude-opus-5-5");
expect(c.body.stream).toBe(true);
expect(c.body.system).toBe("You are kind.");
expect(c.body.messages).toEqual([{ role: "user", content: [{ type: "text", text: "Hi" }] }]);
expect(c.body.tools).toEqual([{ name: "search_course", description: "Search", input_schema: { type: "object", properties: {} } }]);
expect(c.body.thinking).toBeUndefined();
expect(c.body.temperature).toBeUndefined();
expect(c.body.tool_choice).toBeUndefined();
expect(c.body.fallbacks).toBe("default");
expect(h.get("anthropic-beta")).toBe("server-side-fallback-2026-07-01");
});
test("assembles tool input and replays thinking blocks unchanged on the next call", async () => {
const ff = fakeFetch([ok(toolTurn), ok(textTurn)]);
const p = createAnthropicProvider({ apiKey: "k", fetch: ff.fetch });
const msgs: ChatMessage[] = [{ role: "user", content: "How do I add?" }];
const first = await collect(p.chat(msgs, {}));
expect(first.text).toBe("Let me look.");
expect(first.toolCalls).toEqual([{ id: "toolu_1", name: "search_course", input: { query: "add" } }]);
msgs.push({ role: "assistant", content: "Let me look." });
msgs.push({ role: "assistant", content: JSON.stringify({ query: "add" }), toolCallId: "toolu_1", name: "search_course" });
msgs.push({ role: "tool", content: "lessons/04-exercise.md: ...", toolCallId: "toolu_1", name: "search_course" });
await collect(p.chat(msgs, {}));
const sent = ff.calls[1]!.body.messages;
expect(sent).toHaveLength(3);
expect(sent[1]).toEqual({
role: "assistant",
content: [
{ type: "thinking", thinking: "", signature: "SIG-abc" },
{ type: "text", text: "Let me look." },
{ type: "tool_use", id: "toolu_1", name: "search_course", input: { query: "add" } },
],
});
expect(sent[2]).toEqual({ role: "user", content: [{ type: "tool_result", tool_use_id: "toolu_1", content: "lessons/04-exercise.md: ..." }] });
});
test("tool calls it did not produce are mapped from the message fields", async () => {
const ff = fakeFetch([ok(textTurn)]);
const p = createAnthropicProvider({ apiKey: "k", fetch: ff.fetch });
await collect(p.chat([
{ role: "user", content: "q" },
{ role: "assistant", content: "{\"x\":1}", toolCallId: "t9", name: "list_notes" },
{ role: "tool", content: "none", toolCallId: "t9" },
], {}));
expect(ff.calls[0]!.body.messages[1]).toEqual({ role: "assistant", content: [{ type: "tool_use", id: "t9", name: "list_notes", input: { x: 1 } }] });
});
test("errors carry the API message", async () => {
const ff = fakeFetch([() => Response.json({ type: "error", error: { type: "authentication_error", message: "invalid x-api-key" } }, { status: 401 })]);
const p = createAnthropicProvider({ apiKey: "bad", fetch: ff.fetch });
await expect(collect(p.chat([{ role: "user", content: "x" }], {}))).rejects.toThrow(/invalid x-api-key/);
const ff2 = fakeFetch([ok([["error", { type: "error", error: { type: "overloaded_error", message: "Overloaded" } }]])]);
await expect(collect(createAnthropicProvider({ apiKey: "k", fetch: ff2.fetch }).chat([{ role: "user", content: "x" }], {}))).rejects.toThrow(/Overloaded/);
});
test("a refusal yields a plain sentence", async () => {
const ff = fakeFetch([ok([["message_start", { type: "message_start", message: { content: [] } }], ["message_delta", { type: "message_delta", delta: { stop_reason: "refusal" } }], ["message_stop", { type: "message_stop" }]])]);
const r = await collect(createAnthropicProvider({ apiKey: "k", fetch: ff.fetch }).chat([{ role: "user", content: "x" }], {}));
expect(r.text).toContain("declined");
});
test("custom base URL, model and effort", async () => {
const ff = fakeFetch([ok(textTurn)]);
await collect(createAnthropicProvider({ apiKey: "k", baseUrl: "http://proxy.local/", model: "claude-sonnet-5-5", effort: "high", fetch: ff.fetch }).chat([{ role: "user", content: "x" }], {}));
expect(ff.calls[0]!.url).toBe("http://proxy.local/v1/messages");
expect(ff.calls[0]!.body.model).toBe("claude-sonnet-5-5");
expect(ff.calls[0]!.body.output_config).toEqual({ effort: "high" });
expect(ff.calls[0]!.body.fallbacks).toBeUndefined();
});
packages/feature-ai/test/assistant.test.ts
+171
View File
@@ -0,0 +1,171 @@
import { beforeAll, expect, test } from "bun:test";
import { join } from "node:path";
import type { ChatMessage } from "@stc/contracts";
import { createCourseRepository, type CourseRepository, type LoadedCourse } from "@stc/course-format";
import { buildSystemPrompt, citationsFor, readMentorText } from "../src/assistant/prompt";
import { courseTools, combineTools } from "../src/assistant/tools";
import { runAssistant, type AssistantEvent } from "../src/assistant/loop";
import { buildCourseIndex, type CourseIndex } from "../src/retrieval/course-index";
import { fakeProvider } from "./helpers";
const FIXTURES = join(import.meta.dir, "../../../fixtures/courses");
let repo: CourseRepository;
let course: LoadedCourse;
let index: CourseIndex;
beforeAll(async () => {
repo = await createCourseRepository([FIXTURES]);
course = repo.get("sample")!;
index = await buildCourseIndex(course, "/nonexistent");
});
test("mentor rules are in the system prompt when mentor mode is on", async () => {
const mentorText = await readMentorText(repo, course);
expect(mentorText).toContain("never full solutions");
const on = buildSystemPrompt({ course, mentorText, mentorOn: true, sources: index.search("add", 2) });
expect(on).toContain("## Course mentor rules");
expect(on).toContain("Give hints, never full solutions.");
expect(on).toContain("[1] lessons/04-exercise.md");
const off = buildSystemPrompt({ course, mentorText, mentorOn: false, sources: [] });
expect(off).not.toContain("never full solutions");
expect(off).toContain("Mentor mode is off");
});
test("citations are deduplicated by path and lesson", () => {
const c = citationsFor([
{ path: "a.md", kind: "lesson", text: "x", lessonId: "l1", lessonTitle: "L1" },
{ path: "a.md", kind: "lesson", text: "y", lessonId: "l1", lessonTitle: "L1" },
{ path: "notes", kind: "note", text: "z", lessonId: "l1" },
{ path: "b.js", kind: "code", text: "z" },
]);
expect(c).toEqual([
{ path: "a.md", kind: "lesson", lessonId: "l1", lessonTitle: "L1" },
{ path: "b.js", kind: "code" },
]);
});
test("course tools search, read lessons and files, list notes, and refuse solutions", async () => {
const t = courseTools({ repo, course, index, listNotes: async () => [{ id: "n", lessonId: "m1-intro", body: "my note", anchorKind: "none", anchorValue: null }] });
expect(t.specs.map((s) => s.name).sort()).toEqual(["list_notes", "read_file", "read_lesson", "search_course"]);
expect(await t.run("search_course", { query: "sum of its arguments" })).toContain("lessons/04-exercise.md");
expect(t.sources.some((s) => s.path === "lessons/04-exercise.md")).toBe(true);
expect(await t.run("read_lesson", { lessonId: "m2-ex-add" })).toContain("Implement `add(a, b)`");
expect(await t.run("read_lesson", { lessonId: "nope" })).toMatch(/No lesson/);
expect(await t.run("read_file", { path: "exercises/add/starter.js" })).toContain("add");
expect(await t.run("read_file", { path: "exercises/add/solution.js" })).toMatch(/authors only/);
expect(await t.run("read_file", { path: "../../../etc/passwd" })).toMatch(/No such file/);
expect(await t.run("list_notes", {})).toContain("my note");
});
test("the loop runs tools and feeds results back until the model answers", async () => {
const t = courseTools({ repo, course, index, listNotes: async () => [] });
const { provider, seen } = fakeProvider([
() => ({ text: "Searching.", toolCalls: [{ id: "c1", name: "search_course", input: { query: "add arguments" } }] }),
(msgs) => ({ text: `Found it in lessons/04-exercise.md. (${msgs.filter((m) => m.role === "tool").length} tool result)` }),
]);
const messages: ChatMessage[] = [{ role: "system", content: "sys" }, { role: "user", content: "How do I add?" }];
const events: AssistantEvent[] = [];
for await (const e of runAssistant({ provider, messages, tools: combineTools([t]) })) events.push(e);
expect(events.filter((e) => e.type === "tool")).toEqual([{ type: "tool", name: "search_course" }]);
const done = events.find((e) => e.type === "done") as { type: "done"; text: string };
expect(done.text).toBe("Searching.Found it in lessons/04-exercise.md. (1 tool result)");
const second = seen[1]!.messages;
expect(second.at(-2)).toMatchObject({ role: "assistant", toolCallId: "c1", name: "search_course" });
expect(second.at(-1)!.role).toBe("tool");
expect(second.at(-1)!.content).toContain("lessons/04-exercise.md");
expect(seen[0]!.tools!.map((x) => x.name)).toContain("search_course");
});
test("unknown tools and provider errors are reported, and the step limit holds", async () => {
const t = courseTools({ repo, course, index, listNotes: async () => [] });
const looping = fakeProvider([() => ({ toolCalls: [{ id: crypto.randomUUID(), name: "nope", input: {} }] })]);
const events: AssistantEvent[] = [];
for await (const e of runAssistant({ provider: looping.provider, messages: [{ role: "user", content: "x" }], tools: combineTools([t]), maxSteps: 3 })) events.push(e);
expect(looping.seen).toHaveLength(3);
expect(looping.seen[1]!.messages.at(-1)!.content).toMatch(/Unknown tool/);
expect(events.at(-1)!.type).toBe("done");
const broken = { id: "b", async *chat() { throw new Error("boom"); } };
const ev2: AssistantEvent[] = [];
for await (const e of runAssistant({ provider: broken, messages: [{ role: "user", content: "x" }], tools: combineTools([t]) })) ev2.push(e);
expect(ev2).toEqual([{ type: "error", message: "boom" }]);
});
test("my_progress reports the learner's progress per lesson", async () => {
const t = courseTools({
repo, course, index, listNotes: async () => [],
getProgress: async () => ({ "m1-intro": { state: "done", positionSeconds: null, updatedAt: "2026-10-30" }, "m1-video": { state: "in-progress", positionSeconds: 75, updatedAt: "2026-10-30" } }),
});
expect(t.specs.map((s) => s.name)).toContain("my_progress");
const out = await t.run("my_progress", {});
expect(out).toContain("Introduction (m1-intro): done");
expect(out).toContain("Watch the clip (m1-video): in progress, at 1:15");
expect(out).toContain("Adding numbers (m2-text): not started");
expect(out).toMatch(/1 of \d+ lessons done/);
});
// The solution guard must hold for every spelling of a path: it checks the canonical real path.
test("read_file refuses solutions however the path is written, including through symlinks", async () => {
const { cpSync, mkdtempSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } = await import("node:fs");
const { tmpdir } = await import("node:os");
const dir = mkdtempSync(join(tmpdir(), "stc-ai-guard-"));
try {
const root = join(dir, "courses/sample");
cpSync(join(FIXTURES, "sample"), root, { recursive: true });
const yaml = join(root, "stcourse.yaml");
writeFileSync(yaml, readFileSync(yaml, "utf8").replace("solution: exercises/add/solution.js", "solution: exercises/add/answer.js"));
writeFileSync(join(root, "exercises/add/answer.js"), "export const add = (a, b) => a + b; // ANSWER\n");
mkdirSync(join(root, "solutions"));
writeFileSync(join(root, "solutions/ref.js"), "// REFERENCE SOLUTION\n");
writeFileSync(join(root, ".env"), "TOKEN=hidden\n");
symlinkSync("exercises/add/answer.js", join(root, "peek.js"));
symlinkSync("solutions", join(root, "material"));
symlinkSync("../solutions/ref.js", join(root, "lessons/extra.js"));
const r = await createCourseRepository([join(dir, "courses")]);
const c = r.get("sample")!;
expect(c.status).toBe("ok");
const t = courseTools({ repo: r, course: c, index: await buildCourseIndex(c, "/nonexistent"), listNotes: async () => [] });
const read = (path: string) => t.run("read_file", { path });
const refused = [
"exercises/add/answer.js",
"exercises/add/../add/answer.js",
"./exercises/add/answer.js",
"exercises//add/answer.js",
"./exercises/./add/answer.js",
"/exercises/add/answer.js",
"exercises\\add\\answer.js",
"exercises%2Fadd%2Fanswer.js",
"exercises/add/%2E%2E/add/answer.js",
"exercises%252Fadd%252Fanswer.js",
"%2E/exercises/add/answer.js",
"peek.js",
"material/ref.js",
"lessons/extra.js",
"lessons/../solutions/ref.js",
"exercises/add/solution.js",
".env",
"lessons/../.env",
];
for (const p of refused) {
const out = await read(p);
expect({ p, out }).toEqual({ p, out: expect.stringMatching(/authors only|not available|No such file/) });
expect(out).not.toContain("ANSWER");
expect(out).not.toContain("REFERENCE");
expect(out).not.toContain("hidden");
}
expect(await read("exercises/add/answer.js")).toMatch(/authors only/);
expect(await read("peek.js")).toMatch(/authors only/);
expect(await read("../../etc/passwd")).toMatch(/No such file/);
// Ordinary files still read, in any spelling.
expect(await read("exercises/add/starter.js")).toContain("add");
expect(await read("./exercises/./add/../add/starter.js")).toContain("add");
expect(await read("exercises%2Fadd%2Fstarter.js")).toContain("add");
expect(t.sources.at(-1)?.path).toBe("exercises/add/starter.js");
// The index never holds them either.
const idx = await buildCourseIndex(c, "/nonexistent");
const hits = idx.search("ANSWER REFERENCE SOLUTION add", 50);
expect(hits.filter((x) => /answer|ref\.js|peek|material|extra/.test(x.path) || x.text.includes("ANSWER") || x.text.includes("REFERENCE"))).toEqual([]);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
packages/feature-ai/test/crypto.test.ts
+27
View File
@@ -0,0 +1,27 @@
import { expect, test } from "bun:test";
import { createSecretBox } from "../src/crypto";
test("round trips and uses a fresh IV each time", async () => {
const box = createSecretBox("s3cret");
const a = await box.encrypt("sk-ant-123");
const b = await box.encrypt("sk-ant-123");
expect(a).not.toBe(b);
expect(a).not.toContain("sk-ant-123");
expect(a.startsWith("v1.")).toBe(true);
expect(await box.decrypt(a)).toBe("sk-ant-123");
expect(await box.decrypt(b)).toBe("sk-ant-123");
});
test("another secret cannot decrypt", async () => {
const token = await createSecretBox("one").encrypt("key");
await expect(createSecretBox("two").decrypt(token)).rejects.toThrow();
});
test("a tampered token is rejected", async () => {
const token = await createSecretBox("one").encrypt("key");
const parts = token.split(".");
const ct = Buffer.from(parts[2]!, "base64");
ct[0] = ct[0]! ^ 1;
await expect(createSecretBox("one").decrypt(`${parts[0]}.${parts[1]}.${ct.toString("base64")}`)).rejects.toThrow();
await expect(createSecretBox("one").decrypt("garbage")).rejects.toThrow();
});
packages/feature-ai/test/feature.test.ts
+242
View File
@@ -0,0 +1,242 @@
import { expect } from "bun:test";
import { cpSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import type { AppConfig, AuthUser, JobRecord, ServerContext } from "@stc/contracts";
import { createApp, createJobQueue, resolveConfig } from "@stc/core";
import { createCourseRepository } from "@stc/course-format";
import type { Db } from "@stc/db";
import { forEachDb } from "../../db/test/helpers";
import { aiFeature, type AiFeatureOptions } from "../src/feature";
import { fakeProvider } from "./helpers";
const SAMPLE = resolve(import.meta.dir, "../../../fixtures/courses/sample");
const USERS: Record<string, AuthUser> = {
owner: { id: "owner", email: "o@x", name: "Owner", roles: ["owner"], groups: [] },
ann: { id: "ann", email: "a@x", name: "Ann", roles: ["learner"], groups: [] },
bob: { id: "bob", email: "b@x", name: "Bob", roles: ["learner"], groups: [] },
auth: { id: "auth", email: "w@x", name: "Writer", roles: ["author"], groups: [] },
};
async function setup(db: Db, turns: Parameters<typeof fakeProvider>[0] = [() => ({ text: "ok" })], extra: Partial<AiFeatureOptions> = {}, configOverrides: Partial<AppConfig> = {}) {
const dir = mkdtempSync(join(tmpdir(), "stc-ai-feat-"));
cpSync(SAMPLE, join(dir, "courses/sample"), { recursive: true });
cpSync(SAMPLE, join(dir, "courses/secret"), { recursive: true });
const m = readFileSync(join(SAMPLE, "stcourse.yaml"), "utf8");
writeFileSync(join(dir, "courses/secret/stcourse.yaml"), m.replace("id: sample", "id: secret").replace("visibility: public", "visibility: restricted"));
const courses = await createCourseRepository([join(dir, "courses")]);
const fake = fakeProvider(turns);
const keys: (string | null)[] = [];
const ctx = {
db, courses,
config: resolveConfig({ port: 0, databaseUrl: "", dataDir: join(dir, "data"), courseDirs: [], secret: "test-secret", publicUrl: "http://x", ...configOverrides }),
auth: { userFromRequest: async (req: Request) => USERS[req.headers.get("x-user") ?? ""] ?? null },
} as unknown as ServerContext;
const feature = aiFeature(ctx, { providerFactory: (_cfg, key) => { keys.push(key); return fake.provider; }, ...extra });
const jobs = createJobQueue(db, feature.jobs!);
ctx.jobs = jobs;
const app = createApp(ctx, [feature]);
const call = async (method: string, path: string, user?: string, body?: unknown) => {
const headers: Record<string, string> = { origin: "http://x" };
if (user) headers["x-user"] = user;
if (body !== undefined) headers["content-type"] = "application/json";
return app.fetch(new Request(`http://x${path}`, { method, headers, body: body === undefined ? undefined : JSON.stringify(body) }));
};
return { ctx, dir, call, fake, keys, feature };
}
const ndjson = async (r: Response) => (await r.text()).trim().split("\n").map((l) => JSON.parse(l));
forEachDb("AI is off by default and admin config is owner-only and hides keys", async (db) => {
const { call } = await setup(db);
expect(await (await call("GET", "/api/ai/status?courseId=sample", "ann")).json()).toMatchObject({ enabled: false, source: null });
expect((await call("POST", "/api/ai/courses/sample/chat", "ann", { message: "hi" })).status).toBe(409);
expect((await call("GET", "/api/ai/admin/config", "ann")).status).toBe(403);
expect((await call("GET", "/api/ai/admin/config")).status).toBe(401);
const put = await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "anthropic", apiKey: "sk-ant-SERVER" }, mcpServers: [] });
expect(put.status).toBe(200);
const cfg = await (await call("GET", "/api/ai/admin/config", "owner")).text();
expect(cfg).not.toContain("SERVER");
expect(JSON.parse(cfg).provider).toEqual({ kind: "anthropic", hasApiKey: true });
expect((await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "nope" }, mcpServers: [] })).status).toBe(400);
expect(await (await call("GET", "/api/ai/status?courseId=sample", "ann")).json()).toMatchObject({ enabled: true, source: "server", mentor: { available: true, on: true } });
});
forEachDb("BYOK enables AI for that user only and is used before the server key", async (db) => {
const { call, keys } = await setup(db);
expect((await call("PUT", "/api/ai/account/key", "ann", { kind: "anthropic", apiKey: "sk-ann" })).status).toBe(200);
const acct = await (await call("GET", "/api/ai/account", "ann")).text();
expect(acct).not.toContain("sk-ann");
expect(JSON.parse(acct).byok).toEqual({ kind: "anthropic", hasApiKey: true });
expect(await (await call("GET", "/api/ai/status?courseId=sample", "ann")).json()).toMatchObject({ enabled: true, source: "byok" });
expect(await (await call("GET", "/api/ai/status?courseId=sample", "bob")).json()).toMatchObject({ enabled: false });
await call("POST", "/api/ai/courses/sample/chat", "ann", { message: "hello" }).then((r) => r.text());
expect(keys).toEqual(["sk-ann"]);
expect((await call("DELETE", "/api/ai/account/key", "ann")).status).toBe(200);
expect(await (await call("GET", "/api/ai/status?courseId=sample", "ann")).json()).toMatchObject({ enabled: false });
expect((await call("PUT", "/api/ai/account/key", "ann", { kind: "anthropic" })).status).toBe(400);
});
forEachDb("chat cites course files, applies mentor rules, and stores the conversation", async (db) => {
const { call, fake } = await setup(db, [
() => ({ toolCalls: [{ id: "t1", name: "read_lesson", input: { lessonId: "m2-ex-add" } }] }),
() => ({ text: "Which operator combines two numbers? See [lessons/04-exercise.md · Adding numbers]." }),
]);
await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "anthropic", apiKey: "k" }, mcpServers: [] });
const r = await call("POST", "/api/ai/courses/sample/chat", "ann", { message: "How do I return the sum of its arguments?" });
expect(r.status).toBe(200);
expect(r.headers.get("content-type")).toContain("application/x-ndjson");
const events = await ndjson(r);
expect(events[0]).toMatchObject({ type: "meta", mentorOn: true });
expect(events[0].citations.map((c: { path: string }) => c.path)).toContain("lessons/04-exercise.md");
expect(events.some((e) => e.type === "tool" && e.name === "read_lesson")).toBe(true);
const done = events.find((e) => e.type === "done");
expect(done.text).toContain("Which operator");
expect(done.citations.some((c: { path: string; lessonId?: string }) => c.path === "lessons/04-exercise.md")).toBe(true);
const system = fake.seen[0]!.messages[0]!;
expect(system.role).toBe("system");
expect(system.content).toContain("Give hints, never full solutions.");
expect(fake.seen[0]!.tools!.map((t) => t.name)).toEqual(expect.arrayContaining(["search_course", "read_lesson", "list_notes", "my_progress"]));
const convs = await (await call("GET", "/api/ai/courses/sample/conversations", "ann")).json();
expect(convs).toHaveLength(1);
const conv = await (await call("GET", `/api/ai/conversations/${convs[0].id}`, "ann")).json();
expect(conv.messages.map((m: { role: string }) => m.role)).toEqual(["user", "assistant"]);
expect(conv.messages[1].citations.some((c: { path: string }) => c.path === "lessons/04-exercise.md")).toBe(true);
expect((await call("GET", `/api/ai/conversations/${convs[0].id}`, "bob")).status).toBe(404);
// Learner switches mentor mode off for this course: rules leave the prompt.
expect((await call("PUT", "/api/ai/courses/sample/mentor", "ann", { off: true })).status).toBe(200);
expect(await (await call("GET", "/api/ai/status?courseId=sample", "ann")).json()).toMatchObject({ mentor: { available: true, on: false } });
const r2 = await ndjson(await call("POST", "/api/ai/courses/sample/chat", "ann", { message: "Show me the full solution", conversationId: convs[0].id }));
expect(r2[0]).toMatchObject({ type: "meta", mentorOn: false, conversationId: convs[0].id });
const sys2 = fake.seen.at(-1)!.messages[0]!.content;
expect(sys2).not.toContain("never full solutions");
expect(sys2).toContain("Mentor mode is off");
// history is replayed
expect(fake.seen.at(-1)!.messages.some((m) => m.role === "assistant" && m.content.includes("Which operator"))).toBe(true);
// Bob still has mentor mode on
expect(await (await call("GET", "/api/ai/status?courseId=sample", "bob")).json()).toMatchObject({ mentor: { on: true } });
expect((await call("DELETE", `/api/ai/conversations/${convs[0].id}`, "ann")).status).toBe(200);
expect(await (await call("GET", "/api/ai/courses/sample/conversations", "ann")).json()).toEqual([]);
});
forEachDb("course visibility and sign-in are enforced", async (db) => {
const { call } = await setup(db);
await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "anthropic", apiKey: "k" }, mcpServers: [] });
expect((await call("POST", "/api/ai/courses/secret/chat", "ann", { message: "hi" })).status).toBe(404);
expect((await call("POST", "/api/ai/courses/sample/chat", undefined, { message: "hi" })).status).toBe(401);
expect((await call("POST", "/api/ai/courses/nope/chat", "ann", { message: "hi" })).status).toBe(404);
expect((await call("POST", "/api/ai/courses/secret/chat", "owner", { message: "hi" })).status).toBe(200);
});
forEachDb("generation is for authors and writes drafts, not course files", async (db) => {
const doc = "=== FILE: questions/adding.md ===\n## Check your understanding\n1. Predict: add(2, 3)?\n=== END FILE ===";
const { call, ctx, dir } = await setup(db, [() => ({ text: doc })]);
await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "anthropic", apiKey: "k" }, mcpServers: [] });
const body = { courseId: "sample", kind: "questions", style: "pure-snake", topic: "Adding", sources: [{ kind: "lesson", lessonId: "m2-text" }] };
expect((await call("POST", "/api/ai/studio/generate", "ann", body)).status).toBe(403);
expect((await call("GET", "/api/ai/studio/sources?courseId=sample", "ann")).status).toBe(403);
const src = await (await call("GET", "/api/ai/studio/sources?courseId=sample", "auth")).json();
expect(src.lessons.map((l: { id: string }) => l.id)).toContain("m2-text");
expect(src.files).toContain("code/S01 - First steps/hello.sh");
expect((await call("POST", "/api/ai/studio/generate", "auth", { ...body, kind: "poem" })).status).toBe(400);
const g = await call("POST", "/api/ai/studio/generate", "auth", body);
expect(g.status).toBe(202);
const { jobId } = await g.json();
// run the queued job by hand (the worker is not started in tests)
const job = (await ctx.jobs.get(jobId)) as JobRecord;
expect(job.kind).toBe("ai.generate");
const feature = aiFeature(ctx, { providerFactory: () => fakeProvider([() => ({ text: doc })]).provider });
const result = (await feature.jobs!["ai.generate"]!(job.payload, { reportProgress: async () => {}, signal: new AbortController().signal })) as { location: string; paths: string[] };
expect(result.paths).toEqual(["questions/adding.md"]);
expect(result.location.startsWith(join(dir, "data", "drafts", "sample"))).toBe(true);
expect(await Bun.file(join(dir, "courses/sample/questions/adding.md")).exists()).toBe(false);
expect((await call("GET", `/api/ai/studio/jobs/${jobId}`, "auth")).status).toBe(200);
expect((await call("GET", `/api/ai/studio/jobs/${jobId}`, "bob")).status).toBe(404);
});
// SSRF guard: a learner's own key (BYOK) may only go to the official endpoints or admin-allowed origins.
forEachDb("BYOK base URLs are limited to official endpoints and allowed origins", async (db) => {
const { call } = await setup(db, undefined, {}, { aiAllowedBaseUrls: ["http://ollama.internal:11434", "https://llm.example.org"] });
const put = (body: unknown) => call("PUT", "/api/ai/account/key", "ann", body);
// Internal or arbitrary hosts are refused for either kind.
for (const baseUrl of ["http://127.0.0.1:8080/v1", "http://169.254.169.254/latest", "http://localhost:11434/v1", "https://api.anthropic.com.evil.test", "http://ollama.internal:11435/v1", "https://ollama.internal:11434/v1", "file:///etc/passwd"]) {
expect((await put({ kind: "openai", model: "m", apiKey: "k", baseUrl })).status).toBe(400);
}
expect((await put({ kind: "anthropic", apiKey: "k", baseUrl: "http://127.0.0.1:9/" })).status).toBe(400);
expect((await call("GET", "/api/ai/account", "ann")).status).toBe(200);
expect((await (await call("GET", "/api/ai/account", "ann")).json()).byok).toBeNull();
// Official endpoints and allowlisted origins (any path on them) are accepted.
expect((await put({ kind: "anthropic", apiKey: "k" })).status).toBe(200);
expect((await put({ kind: "anthropic", apiKey: "k", baseUrl: "https://api.anthropic.com/" })).status).toBe(200);
expect((await put({ kind: "openai", model: "m", apiKey: "k" })).status).toBe(200);
expect(await (await call("GET", "/api/ai/account", "ann")).json()).toMatchObject({ byok: { kind: "openai", model: "m", hasApiKey: true } }); // official endpoint: stored as "no base URL"
expect((await put({ kind: "openai", model: "m", apiKey: "k", baseUrl: "http://ollama.internal:11434/v1" })).status).toBe(200);
expect((await put({ kind: "openai", model: "m", apiKey: "k", baseUrl: "https://llm.example.org/v1/" })).status).toBe(200);
// The admin's server-scope provider may use any endpoint (Ollama on localhost).
expect((await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "openai", model: "llama", baseUrl: "http://127.0.0.1:11434/v1" }, mcpServers: [] })).status).toBe(200);
});
forEachDb("the admin allowlist in the server config enables an origin for BYOK, and removing it disables the key", async (db) => {
const { call } = await setup(db);
const put = () => call("PUT", "/api/ai/account/key", "ann", { kind: "openai", model: "m", apiKey: "k", baseUrl: "http://gpu-box:8000/v1" });
expect((await put()).status).toBe(400);
expect((await call("PUT", "/api/ai/admin/config", "owner", { provider: null, mcpServers: [], allowedBaseUrls: ["http://gpu-box:8000/v1"] })).status).toBe(400); // origins only
expect((await call("PUT", "/api/ai/admin/config", "ann", { provider: null, mcpServers: [], allowedBaseUrls: ["http://gpu-box:8000"] })).status).toBe(403);
expect((await call("PUT", "/api/ai/admin/config", "owner", { provider: null, mcpServers: [], allowedBaseUrls: ["http://gpu-box:8000"] })).status).toBe(200);
expect((await (await call("GET", "/api/ai/admin/config", "owner")).json()).allowedBaseUrls).toEqual(["http://gpu-box:8000"]);
expect((await put()).status).toBe(200);
expect(await (await call("GET", "/api/ai/status?courseId=sample", "ann")).json()).toMatchObject({ enabled: true, source: "byok" });
// Saving the config without the field keeps the list; an empty list removes it, and the saved key stops being used.
await call("PUT", "/api/ai/admin/config", "owner", { provider: null, mcpServers: [] });
expect((await (await call("GET", "/api/ai/admin/config", "owner")).json()).allowedBaseUrls).toEqual(["http://gpu-box:8000"]);
await call("PUT", "/api/ai/admin/config", "owner", { provider: null, mcpServers: [], allowedBaseUrls: [] });
expect(await (await call("GET", "/api/ai/status?courseId=sample", "ann")).json()).toMatchObject({ enabled: false });
});
forEachDb("provider errors on a user's own key show the status and a fixed message, never the upstream body", async (db) => {
const logs: string[] = [];
const seen: { url: string; redirect?: RequestRedirect }[] = [];
const fakeFetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
seen.push({ url: String(input), redirect: init?.redirect });
return new Response(JSON.stringify({ error: { message: "INTERNAL-SECRET-BODY from upstream" } }), { status: 502, headers: { "content-type": "application/json" } });
}) as typeof fetch;
const { call } = await setup(db, undefined, { providerFactory: undefined, fetch: fakeFetch, log: (m) => logs.push(m) }, { aiAllowedBaseUrls: ["http://ollama.internal:11434"] });
expect((await call("PUT", "/api/ai/account/key", "ann", { kind: "openai", model: "m", apiKey: "k", baseUrl: "http://ollama.internal:11434/v1" })).status).toBe(200);
const text = await (await call("POST", "/api/ai/courses/sample/chat", "ann", { message: "hello" })).text();
expect(text).not.toContain("INTERNAL-SECRET-BODY");
const events = text.trim().split("\n").map((l) => JSON.parse(l));
expect(events.find((e) => e.type === "error").message).toBe("502: The AI provider returned an error. Check your key, model and endpoint under Account, AI.");
expect(seen).toEqual([{ url: "http://ollama.internal:11434/v1/chat/completions", redirect: "manual" }]);
expect(logs.some((l) => l.includes("INTERNAL-SECRET-BODY") && l.includes("ann"))).toBe(true);
// The admin's server provider keeps the full message for the owner to debug.
await call("DELETE", "/api/ai/account/key", "ann");
await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "openai", model: "m", baseUrl: "http://127.0.0.1:11434/v1" }, mcpServers: [] });
const t2 = await (await call("POST", "/api/ai/courses/sample/chat", "owner", { message: "hello" })).text();
expect(t2).toContain("502: INTERNAL-SECRET-BODY");
});
forEachDb("authoring uses core canView: the global author role reaches restricted courses (R14)", async (db) => {
const { call } = await setup(db);
await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "anthropic", apiKey: "k" }, mcpServers: [] });
const body = { courseId: "secret", kind: "questions", style: "pure-snake", topic: "Adding", sources: [{ kind: "lesson", lessonId: "m2-text" }] };
const courses = async (u: string) => ((await (await call("GET", "/api/ai/studio/courses", u)).json()) as { id: string }[]).map((c) => c.id).sort();
// The global author role views every course under core canView, so it may author the restricted one too.
expect(await courses("auth")).toEqual(["sample", "secret"]);
expect((await call("GET", "/api/ai/studio/sources?courseId=secret", "auth")).status).toBe(200);
expect((await call("POST", "/api/ai/studio/generate", "auth", body)).status).toBe(202);
// The owner sees everything.
expect(await courses("owner")).toEqual(["sample", "secret"]);
expect((await call("POST", "/api/ai/studio/generate", "owner", body)).status).toBe(202);
// A course_authors row makes a learner an author of that course only.
expect((await call("POST", "/api/ai/studio/generate", "bob", body)).status).toBe(403);
await db.sql`INSERT INTO course_authors (course_id, user_id) VALUES ('secret', 'bob')`;
expect(await courses("bob")).toEqual(["secret"]);
expect((await call("POST", "/api/ai/studio/generate", "bob", body)).status).toBe(202);
// A grant to view alone does not make a learner an author.
await db.sql`INSERT INTO course_acl (course_id, principal_type, principal_id) VALUES ('secret', 'user', 'ann')`;
expect((await call("POST", "/api/ai/studio/generate", "ann", body)).status).toBe(403);
});
packages/feature-ai/test/fixtures/fake-mcp.ts
Vendored
+33
View File
@@ -0,0 +1,33 @@
// A fake MCP server for tests. stdio: newline-delimited JSON-RPC on stdin/stdout.
// Exported `handle` is reused by the HTTP fake in mcp.test.ts.
export function handle(msg: any): any | null {
if (msg.id === undefined) return null; // notification
switch (msg.method) {
case "initialize":
return { jsonrpc: "2.0", id: msg.id, result: { protocolVersion: "2025-06-18", capabilities: { tools: {} }, serverInfo: { name: "fake", version: "1" } } };
case "tools/list":
return { jsonrpc: "2.0", id: msg.id, result: { tools: [{ name: "echo", description: "Echo text back", inputSchema: { type: "object", properties: { text: { type: "string" } }, required: ["text"] } }] } };
case "tools/call":
if (msg.params?.name === "env") return { jsonrpc: "2.0", id: msg.id, result: { content: [{ type: "text", text: JSON.stringify(process.env) }] } };
if (msg.params?.name !== "echo") return { jsonrpc: "2.0", id: msg.id, error: { code: -32602, message: `Unknown tool ${msg.params?.name}` } };
return { jsonrpc: "2.0", id: msg.id, result: { content: [{ type: "text", text: `echo: ${msg.params.arguments?.text}` }] } };
default:
return { jsonrpc: "2.0", id: msg.id, error: { code: -32601, message: "Method not found" } };
}
}
if (import.meta.main) {
if (process.argv.includes("--exit-immediately")) process.exit(3);
let buf = "";
for await (const chunk of Bun.stdin.stream()) {
buf += new TextDecoder().decode(chunk);
let nl: number;
while ((nl = buf.indexOf("\n")) !== -1) {
const line = buf.slice(0, nl).trim();
buf = buf.slice(nl + 1);
if (!line) continue;
const reply = handle(JSON.parse(line));
if (reply) process.stdout.write(JSON.stringify(reply) + "\n");
}
}
}
packages/feature-ai/test/generator.test.ts
+112
View File
@@ -0,0 +1,112 @@
import { afterAll, beforeAll, expect, test } from "bun:test";
import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createCourseRepository, type CourseRepository } from "@stc/course-format";
import { buildGeneratorPrompt, gatherMaterial, generateDraft, parseFiles, checkHouseStyle, type GenerateRequest } from "../src/generator/generate";
import { createFsDraftSink, type DraftSink } from "../src/generator/draft-sink";
import { fakeProvider } from "./helpers";
const FIXTURES = join(import.meta.dir, "../../../fixtures/courses");
let repo: CourseRepository;
let dataDir: string;
beforeAll(async () => {
repo = await createCourseRepository([FIXTURES]);
dataDir = mkdtempSync(join(tmpdir(), "stc-ai-gen-"));
});
afterAll(() => rmSync(dataDir, { recursive: true, force: true }));
const req = (over: Partial<GenerateRequest> = {}): GenerateRequest => ({
courseId: "sample", kind: "document", style: "pure-snake", topic: "Adding many numbers",
sources: [{ kind: "lesson", lessonId: "m2-text" }, { kind: "text", label: "Author notes", text: "Learners confuse sum and count." }], ...over,
});
test("Pure Snake style prompt carries the module shape and hint tiers", async () => {
const course = repo.get("sample")!;
const material = await gatherMaterial(repo, course, req().sources, dataDir);
const p = buildGeneratorPrompt(req(), material.text, course);
for (const s of ["## Why this module", "## Learning objectives", "## Concepts", "Think about it first, then open the explanation", "From everyday programming", "— nudge", "— stronger hint", "— near-solution", "## Check your understanding", "## In production", "## Further reading", "=== FILE:"]) {
expect(p.system).toContain(s);
}
expect(p.user).toContain("Functions take values and return values.");
expect(p.user).toContain("Learners confuse sum and count.");
expect(p.user).toContain("course/adding-many-numbers.md");
});
test("Systemic Design style prompt carries the beats", () => {
const course = repo.get("sample")!;
const p = buildGeneratorPrompt(req({ style: "systemic-design" }), "material", course);
const beats = ["## Observe", "## Predict", "## Spec", "## Build", "## Name it", "## Expansion Drop", "## Further reading"];
let last = -1;
for (const b of beats) { const i = p.system.indexOf(b); expect(i).toBeGreaterThan(last); last = i; }
for (const s of ["Hint 1 — nudge", "Hint 2 — concept", "Hint 3 — pseudo-code", "what this drop measures", "scenarios.md"]) expect(p.system).toContain(s);
});
test("exercise and question prompts", () => {
const course = repo.get("sample")!;
const ex = buildGeneratorPrompt(req({ kind: "exercise" }), "m", course);
for (const s of ["starter.js", "tests.js", "solution.js", "export const tests", "fixtures/", "nudge", "stronger hint", "near-solution"]) expect(ex.system + ex.user).toContain(s);
const q = buildGeneratorPrompt(req({ kind: "questions" }), "m", course);
expect(q.system).toContain("Check your understanding");
expect(q.system).toMatch(/predict/i);
expect(q.system).toMatch(/what would break if/i);
});
test("parseFiles reads file blocks and rejects unsafe paths", () => {
const out = parseFiles("intro\n=== FILE: course/a.md ===\n# A\n=== END FILE ===\n=== FILE: ../escape.md ===\nx\n=== END FILE ===\n=== FILE: /abs.md ===\ny\n=== END FILE ===\n=== FILE: exercises/b/starter.js ===\nexport const b = 1\n=== END FILE ===");
expect(out.files).toEqual([{ path: "course/a.md", content: "# A\n" }, { path: "exercises/b/starter.js", content: "export const b = 1\n" }]);
expect(out.warnings.join("\n")).toMatch(/escape\.md/);
expect(out.warnings.join("\n")).toMatch(/abs\.md/);
expect(parseFiles("no files here").warnings[0]).toMatch(/no files/i);
});
test("house style check flags missing tiers", () => {
const w = checkHouseStyle(req(), [{ path: "course/x.md", content: "# M\n## Why this module\n" }]);
expect(w.some((x) => x.includes("Hints"))).toBe(true);
expect(w.some((x) => x.includes("near-solution"))).toBe(true);
});
test("generation writes drafts through the sink and never touches the course", async () => {
const before = readdirSync(join(FIXTURES, "sample"), { recursive: true }).length;
const doc = [
"# Module 3 — Adding many numbers", "## Why this module", "x", "## Learning objectives", "## Concepts", "<details><summary>Think about it first, then open the explanation</summary>", "y", "</details>",
"## Exercises", "## Hints", "<details><summary>Exercise 3.1 (sum) — nudge</summary></details>", "<details><summary>Exercise 3.1 (sum) — stronger hint</summary></details>",
"<details><summary>Exercise 3.1 (sum) — near-solution (try the others first)</summary></details>", "## Check your understanding", "1. Predict…", "## In production", "## Further reading",
].join("\n");
const { provider, seen } = fakeProvider([() => ({ text: `Here you go.\n=== FILE: course/adding-many-numbers.md ===\n${doc}\n=== END FILE ===\n` })]);
const sink = createFsDraftSink(dataDir);
const r = await generateDraft({ provider, repo, req: req(), sink, authorId: "author-1", dataDir });
expect(seen[0]!.tools).toBeUndefined();
expect(r.paths).toEqual(["course/adding-many-numbers.md"]);
expect(r.location.startsWith(join(dataDir, "drafts", "sample"))).toBe(true);
expect(r.warnings).toEqual([]);
const written = readFileSync(join(r.location, "course/adding-many-numbers.md"), "utf8");
expect(written.startsWith("<!-- Draft generated by STCourses AI")).toBe(true);
expect(written).toContain("## Why this module");
const meta = JSON.parse(readFileSync(join(r.location, "DRAFT.json"), "utf8"));
expect(meta).toMatchObject({ courseId: "sample", authorId: "author-1", published: false });
expect(readdirSync(join(FIXTURES, "sample"), { recursive: true }).length).toBe(before);
expect(existsSync(join(FIXTURES, "sample", "course"))).toBe(false);
});
// The studio's DraftWriter (`createDraftWriter(ctx)` in @stc/feature-studio), reproduced structurally: it commits
// to `draft/<authorId>` and returns { branch, commit }. It must be usable as a DraftSink without an adapter.
interface StudioDraftWriterShape {
writeDraftFiles(courseId: string, authorId: string, files: { path: string; content: string }[], message: string): Promise<{ branch: string; commit: string }>;
}
test("a studio DraftWriter is a DraftSink as is, and the result names the draft branch", async () => {
const calls: { courseId: string; authorId: string; paths: string[]; message: string }[] = [];
const writer: StudioDraftWriterShape = {
async writeDraftFiles(courseId, authorId, files, message) {
calls.push({ courseId, authorId, paths: files.map((f) => f.path), message });
return { branch: `draft/${authorId}`, commit: "0123456789abcdef0123456789abcdef01234567" };
},
};
const sink: DraftSink = writer;
const { provider } = fakeProvider([() => ({ text: "=== FILE: questions/adding.md ===\n## Check your understanding\n=== END FILE ===\n" })]);
const r = await generateDraft({ provider, repo, req: req({ kind: "questions" }), sink, authorId: "author-1", dataDir });
expect(calls).toEqual([{ courseId: "sample", authorId: "author-1", paths: ["questions/adding.md"], message: expect.stringContaining("AI draft") }]);
expect(r).toMatchObject({ branch: "draft/author-1", commit: "0123456789abcdef0123456789abcdef01234567", paths: ["questions/adding.md"] });
expect(r.location).toBe("draft/author-1 @ 0123456");
});
packages/feature-ai/test/helpers.ts
+53
View File
@@ -0,0 +1,53 @@
import type { AiProvider, ChatMessage, ToolSpec } from "@stc/contracts";
/** SSE body from a list of [event, data] pairs, split into awkward chunks to exercise buffering. */
export function sseBody(events: [string | null, unknown][], chunk = 7): ReadableStream<Uint8Array> {
const text = events.map(([e, d]) => `${e ? `event: ${e}\n` : ""}data: ${typeof d === "string" ? d : JSON.stringify(d)}\n\n`).join("");
const bytes = new TextEncoder().encode(text);
let i = 0;
return new ReadableStream({
pull(c) {
if (i >= bytes.length) return c.close();
c.enqueue(bytes.slice(i, i + chunk));
i += chunk;
},
});
}
export type Captured = { url: string; init: RequestInit; body: any };
/** A fake fetch that records requests and answers from a queue of responses. */
export function fakeFetch(responses: (() => Response)[]) {
const calls: Captured[] = [];
const f = (async (url: string | URL | Request, init?: RequestInit) => {
calls.push({ url: String(url), init: init ?? {}, body: init?.body ? JSON.parse(String(init.body)) : null });
const next = responses.shift();
if (!next) throw new Error("fakeFetch: no response queued");
return next();
}) as unknown as typeof fetch;
return { fetch: f, calls };
}
export async function collect(it: AsyncIterable<{ text?: string; toolCall?: { id: string; name: string; input: unknown } }>) {
let text = "";
const toolCalls: { id: string; name: string; input: unknown }[] = [];
for await (const ev of it) {
if (ev.text) text += ev.text;
if (ev.toolCall) toolCalls.push(ev.toolCall);
}
return { text, toolCalls };
}
/** Scripted fake provider: each call to chat() plays the next turn and records the messages it got. */
export function fakeProvider(turns: ((messages: ChatMessage[], tools?: ToolSpec[]) => { text?: string; toolCalls?: { id: string; name: string; input: unknown }[] })[]) {
const seen: { messages: ChatMessage[]; tools?: ToolSpec[] }[] = [];
const provider: AiProvider = {
id: "fake",
async *chat(messages, opts) {
seen.push({ messages: structuredClone(messages), tools: opts.tools });
const turn = turns[Math.min(seen.length - 1, turns.length - 1)]!(messages, opts.tools);
if (turn.text) for (const part of turn.text.match(/.{1,8}/gs) ?? []) yield { text: part };
for (const tc of turn.toolCalls ?? []) yield { toolCall: tc };
},
};
return { provider, seen };
}
packages/feature-ai/test/mcp.test.ts
+82
View File
@@ -0,0 +1,82 @@
import { afterAll, expect, test } from "bun:test";
import { join } from "node:path";
import { connectMcp, mcpServersFor, mcpToolName } from "../src/mcp/client";
import { handle } from "./fixtures/fake-mcp";
const FAKE = join(import.meta.dir, "fixtures/fake-mcp.ts");
test("stdio: lists and calls tools", async () => {
const c = await connectMcp({ id: "fake", name: "Fake", transport: "stdio", command: process.execPath, args: [FAKE] });
try {
expect((await c.tools()).map((t) => t.name)).toEqual(["echo"]);
expect(await c.call("echo", { text: "hi" })).toBe("echo: hi");
await expect(c.call("nope", {})).rejects.toThrow(/Unknown tool/);
} finally { await c.close(); }
});
test("stdio: a server that exits rejects instead of hanging", async () => {
await expect(connectMcp({ id: "x", name: "X", transport: "stdio", command: process.execPath, args: [FAKE, "--exit-immediately"] }, { timeoutMs: 5000 })).rejects.toThrow();
});
const sessions: (string | null)[] = [];
const server = Bun.serve({
port: 0,
async fetch(req) {
if (req.method === "DELETE") return new Response(null, { status: 204 });
const msg = await req.json();
sessions.push(req.headers.get("mcp-session-id"));
const reply = handle(msg);
if (!reply) return new Response(null, { status: 202 });
const headers: Record<string, string> = msg.method === "initialize" ? { "mcp-session-id": "sess-42" } : {};
if (msg.method === "tools/call") {
return new Response(`event: message\ndata: ${JSON.stringify(reply)}\n\n`, { headers: { ...headers, "content-type": "text/event-stream" } });
}
return Response.json(reply, { headers });
},
});
afterAll(() => server.stop(true));
test("streamable HTTP: JSON and SSE replies, session id echoed", async () => {
const c = await connectMcp({ id: "web", name: "Web", transport: "http", url: `http://127.0.0.1:${server.port}/mcp` });
expect((await c.tools()).map((t) => t.name)).toEqual(["echo"]);
expect(await c.call("echo", { text: "over http" })).toBe("echo: over http");
await c.close();
expect(sessions[0]).toBeNull();
expect(sessions.slice(1).every((s) => s === "sess-42")).toBe(true);
});
test("server selection and tool names", () => {
const servers = [
{ id: "g", name: "G", transport: "http" as const, url: "http://x" },
{ id: "s", name: "S", transport: "http" as const, url: "http://y", courseIds: ["sample"] },
{ id: "o", name: "O", transport: "http" as const, url: "http://z", courseIds: ["other"] },
];
expect(mcpServersFor("sample", servers).map((s) => s.id)).toEqual(["g", "s"]);
expect(mcpToolName("my server", "read file!")).toBe("mcp_my_server__read_file_");
expect(mcpToolName("a".repeat(40), "b".repeat(40)).length).toBeLessThanOrEqual(64);
});
test("stdio: the child gets a minimal environment plus only its configured variables", async () => {
const saved = { ...process.env };
Object.assign(process.env, {
STC_SECRET: "server-secret", STC_DATABASE_URL: "postgres://u:p@db/x", DATABASE_URL: "postgres://u:p@db/y",
ANTHROPIC_API_KEY: "sk-ant-x", MY_SECRET: "s", AWS_SECRET_ACCESS_KEY: "aws", LANG: "C.UTF-8",
});
try {
const c = await connectMcp({ id: "fake", name: "Fake", transport: "stdio", command: process.execPath, args: [FAKE], env: { MCP_TOKEN: "t", STC_EXPLICIT: "configured" } });
try {
const env = JSON.parse(await c.call("env", {})) as Record<string, string>;
for (const k of ["STC_SECRET", "STC_DATABASE_URL", "DATABASE_URL", "ANTHROPIC_API_KEY", "MY_SECRET", "AWS_SECRET_ACCESS_KEY"]) expect(env[k]).toBeUndefined();
expect(env.MCP_TOKEN).toBe("t");
expect(env.STC_EXPLICIT).toBe("configured"); // explicitly configured by the admin
expect(env.LANG).toBe("C.UTF-8");
if (saved.PATH) expect(env.PATH).toBe(saved.PATH);
if (saved.HOME) expect(env.HOME).toBe(saved.HOME);
const unexpected = Object.keys(env).filter((k) => !["PATH", "HOME", "LANG", "MCP_TOKEN", "STC_EXPLICIT"].includes(k));
expect(unexpected.filter((k) => /STC_|DATABASE|SECRET|_KEY$/i.test(k))).toEqual([]);
} finally { await c.close(); }
} finally {
for (const k of Object.keys(process.env)) if (!(k in saved)) delete process.env[k];
Object.assign(process.env, saved);
}
});
packages/feature-ai/test/openai.test.ts
+61
View File
@@ -0,0 +1,61 @@
import { expect, test } from "bun:test";
import { createOpenAiProvider } from "../src/providers/openai";
import { createProvider } from "../src/providers";
import { collect, fakeFetch, sseBody } from "./helpers";
const ok = (events: [string | null, unknown][]) => () => new Response(sseBody(events));
const chunk = (delta: unknown, finish: string | null = null) => [null, { choices: [{ index: 0, delta, finish_reason: finish }] }] as [null, unknown];
test("streams text deltas from an Ollama-style endpoint without a key", async () => {
const ff = fakeFetch([ok([chunk({ role: "assistant", content: "Hel" }), chunk({ content: "lo" }), chunk({}, "stop"), [null, "[DONE]"]])]);
const p = createOpenAiProvider({ baseUrl: "http://ollama:11434/v1/", model: "llama3.1", fetch: ff.fetch });
const r = await collect(p.chat([{ role: "system", content: "sys" }, { role: "user", content: "hi" }], {}));
expect(r.text).toBe("Hello");
const c = ff.calls[0]!;
expect(c.url).toBe("http://ollama:11434/v1/chat/completions");
expect(new Headers(c.init.headers).get("authorization")).toBeNull();
expect(c.body).toMatchObject({ model: "llama3.1", stream: true, messages: [{ role: "system", content: "sys" }, { role: "user", content: "hi" }] });
});
test("assembles fragmented tool calls and maps a tool round back", async () => {
const ff = fakeFetch([
ok([
chunk({ tool_calls: [{ index: 0, id: "call_1", type: "function", function: { name: "read_lesson", arguments: "" } }] }),
chunk({ tool_calls: [{ index: 0, function: { arguments: "{\"lessonId\":" } }] }),
chunk({ tool_calls: [{ index: 0, function: { arguments: "\"m1-intro\"}" } }] }),
chunk({}, "tool_calls"),
[null, "[DONE]"],
]),
ok([chunk({ content: "Done" }), [null, "[DONE]"]]),
]);
const p = createOpenAiProvider({ baseUrl: "https://api.example.com/v1", apiKey: "sk-x", model: "gpt-x", fetch: ff.fetch });
const tools = [{ name: "read_lesson", description: "Read", inputSchema: { type: "object" } }];
const r = await collect(p.chat([{ role: "user", content: "q" }], { tools }));
expect(r.toolCalls).toEqual([{ id: "call_1", name: "read_lesson", input: { lessonId: "m1-intro" } }]);
expect(ff.calls[0]!.body.tools).toEqual([{ type: "function", function: { name: "read_lesson", description: "Read", parameters: { type: "object" } } }]);
expect(new Headers(ff.calls[0]!.init.headers).get("authorization")).toBe("Bearer sk-x");
await collect(p.chat([
{ role: "user", content: "q" },
{ role: "assistant", content: "Reading." },
{ role: "assistant", content: "{\"lessonId\":\"m1-intro\"}", toolCallId: "call_1", name: "read_lesson" },
{ role: "tool", content: "# Introduction", toolCallId: "call_1", name: "read_lesson" },
], { tools }));
expect(ff.calls[1]!.body.messages).toEqual([
{ role: "user", content: "q" },
{ role: "assistant", content: "Reading.", tool_calls: [{ id: "call_1", type: "function", function: { name: "read_lesson", arguments: "{\"lessonId\":\"m1-intro\"}" } }] },
{ role: "tool", tool_call_id: "call_1", content: "# Introduction" },
]);
});
test("errors carry the endpoint message", async () => {
const ff = fakeFetch([() => Response.json({ error: { message: "model not found" } }, { status: 404 })]);
await expect(collect(createOpenAiProvider({ baseUrl: "http://x/v1", model: "m", fetch: ff.fetch }).chat([{ role: "user", content: "x" }], {}))).rejects.toThrow(/model not found/);
});
test("createProvider picks the implementation and defaults", () => {
expect(createProvider({ kind: "anthropic" }, "k").id).toBe("anthropic");
expect(createProvider({ kind: "openai", baseUrl: "http://x/v1", model: "m" }, null).id).toBe("openai");
expect(() => createProvider({ kind: "anthropic" }, null)).toThrow(/key/);
expect(() => createProvider({ kind: "openai" }, null)).toThrow(/base URL|model/);
});
packages/feature-ai/test/retrieval.test.ts
+130
View File
@@ -0,0 +1,130 @@
import { afterAll, beforeAll, expect, test } from "bun:test";
import { cpSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createCourseRepository, type LoadedCourse } from "@stc/course-format";
import { Bm25, tokenize } from "../src/retrieval/bm25";
import { buildCourseIndex, getCourseIndex, isSolutionPath, noteChunks } from "../src/retrieval/course-index";
const FIXTURES = join(import.meta.dir, "../../../fixtures/courses");
let course: LoadedCourse;
let dataDir: string;
beforeAll(async () => {
const repo = await createCourseRepository([FIXTURES]);
course = repo.get("sample")!;
dataDir = mkdtempSync(join(tmpdir(), "stc-ai-"));
mkdirSync(join(dataDir, "cache/sample/decks/d1"), { recursive: true });
writeFileSync(join(dataDir, "cache/sample/decks/d1/page-3.txt"), "Container images are layered filesystems built from a Dockerfile.");
mkdirSync(join(dataDir, "cache/sample/transcripts"), { recursive: true });
writeFileSync(join(dataDir, "cache/sample/transcripts/m1-video.txt"), "In this clip the presenter explains the test pattern calibration.");
});
afterAll(() => rmSync(dataDir, { recursive: true, force: true }));
test("tokenize lowercases, drops stopwords and splits identifiers", () => {
expect(tokenize("The inBounds function, and dropLast!")).toEqual(["inbound", "in", "bound", "function", "droplast", "drop", "last"]);
expect(tokenize("Æble på dansk")).toEqual(["æble", "på", "dansk"]);
});
test("BM25 ranks the more relevant document first", () => {
const b = new Bm25<string>();
b.add("snake moves one cell per tick", "a");
b.add("apples spawn at random free cells", "b");
b.add("the snake grows when it eats an apple apple", "c");
expect(b.search("apple", 5).map((r) => r.item)).toEqual(["c", "b"]);
expect(b.search("zebra", 5)).toEqual([]);
});
test("finds lesson text and maps it to a lesson", async () => {
const idx = await buildCourseIndex(course, dataDir);
const hits = idx.search("sum of its arguments", 3);
expect(hits[0]!.path).toBe("lessons/04-exercise.md");
expect(["m2-text", "m2-ex-add"]).toContain(hits[0]!.lessonId!);
expect(hits[0]!.lessonTitle).toBeTruthy();
});
test("indexes code, transcripts and deck text from the cache", async () => {
const idx = await buildCourseIndex(course, dataDir);
const code = idx.search("hello.sh", 5).find((c) => c.kind === "code");
expect(code?.path).toBe("code/S01 - First steps/hello.sh");
const deck = idx.search("Dockerfile layered images", 3)[0]!;
expect(deck.kind).toBe("deck");
expect(deck.path).toBe("media/deck.pdf");
expect(deck.heading).toBe("Slide 3");
expect(deck.lessonId).toBe("m1-video");
const tr = idx.search("calibration presenter", 3)[0]!;
expect(tr.kind).toBe("transcript");
expect(tr.lessonId).toBe("m1-video");
});
test("solution files are never indexed", async () => {
const idx = await buildCourseIndex(course, dataDir);
for (const q of ["add", "return a + b", "export const add"]) {
expect(idx.search(q, 50).some((c) => c.path.includes("solution"))).toBe(false);
}
expect(isSolutionPath(course, "exercises/add/solution.js")).toBe(true);
expect(isSolutionPath(course, "js/solutions/rules.js")).toBe(true);
expect(isSolutionPath(course, "course/01-handles/reference/ts/src/engine.ts")).toBe(true);
expect(isSolutionPath(course, "course/01-handles/expansion/solution/x.rs")).toBe(true);
expect(isSolutionPath(course, "exercises/add/starter.js")).toBe(false);
});
test("the mentor file and the manifest are not retrievable", async () => {
const idx = await buildCourseIndex(course, dataDir);
expect(idx.search("mentor hints never full solutions", 50).some((c) => c.path === "MENTOR.md" || c.path === "stcourse.yaml")).toBe(false);
});
test("notes are searched alongside the course", async () => {
const idx = await getCourseIndex(course, dataDir);
expect(await getCourseIndex(course, dataDir)).toBe(idx);
const extra = noteChunks(course, [{ id: "n1", lessonId: "m1-intro", body: "Remember: zebras are striped horses", anchorKind: "none", anchorValue: null }]);
const hits = idx.search("zebras striped", 3, extra);
expect(hits[0]).toMatchObject({ kind: "note", lessonId: "m1-intro", lessonTitle: "Introduction" });
});
// The importer's layout (feature-import): <dataDir>/cache/decks/<courseId>/<deckId>/text.json ([{ page, text }])
// and <dataDir>/cache/transcripts/<courseId>/<lessonId>.json ({ lessonId, text, segments: [{ start, end, text }] }).
test("reads the importer's deck text.json and transcript cache", async () => {
const dir = mkdtempSync(join(tmpdir(), "stc-ai-imp-"));
try {
const courses = join(dir, "courses");
cpSync(join(FIXTURES, "sample"), join(courses, "sample"), { recursive: true });
const yaml = join(courses, "sample/stcourse.yaml");
writeFileSync(yaml, readFileSync(yaml, "utf8").replace(", transcript: lessons/02-video.md", ""));
const c = (await createCourseRepository([courses])).get("sample")!;
expect(c.status).toBe("ok");
const data = join(dir, "data");
mkdirSync(join(data, "cache/decks/sample/d1"), { recursive: true });
writeFileSync(join(data, "cache/decks/sample/d1/text.json"), JSON.stringify([
{ page: 1, text: "Title slide" },
{ page: 2, text: "Orchestration with Kubernetes pods and replica sets." },
{ page: 3, text: "" },
]));
mkdirSync(join(data, "cache/transcripts/sample"), { recursive: true });
writeFileSync(join(data, "cache/transcripts/sample/m1-video.json"), JSON.stringify({
lessonId: "m1-video",
text: "Welcome. Today the presenter shows a waveform monitor.",
segments: [{ start: 0, end: 4, text: "Welcome." }, { start: 65, end: 70, text: "Today the presenter shows a waveform monitor." }],
}));
const idx = await buildCourseIndex(c, data);
const deck = idx.search("Kubernetes replica sets", 3)[0]!;
expect(deck).toMatchObject({ kind: "deck", path: "media/deck.pdf", heading: "Slide 2", lessonId: "m1-video" });
const tr = idx.search("waveform monitor", 3)[0]!;
expect(tr).toMatchObject({ kind: "transcript", lessonId: "m1-video", lessonTitle: "Watch the clip" });
expect(tr.text).toContain("waveform monitor");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
test("a cached transcript is skipped when the lesson already has a transcript file", async () => {
const data = mkdtempSync(join(tmpdir(), "stc-ai-tr-"));
try {
mkdirSync(join(data, "cache/transcripts/sample"), { recursive: true });
writeFileSync(join(data, "cache/transcripts/sample/m1-video.json"), JSON.stringify({ lessonId: "m1-video", text: "zebracorn unique words", segments: [] }));
const idx = await buildCourseIndex(course, data);
expect(idx.search("zebracorn", 3)).toEqual([]);
} finally {
rmSync(data, { recursive: true, force: true });
}
});
packages/feature-ai/test/settings.test.ts
+63
View File
@@ -0,0 +1,63 @@
import { expect } from "bun:test";
import { forEachDb } from "../../db/test/helpers";
import { createSecretBox } from "../src/crypto";
import { aiSettingsRepo } from "../src/settings";
const box = createSecretBox("test-secret");
forEachDb("server config is empty by default and hides the key", async (db) => {
const r = aiSettingsRepo(db, box);
expect(await r.getServer()).toEqual({ provider: null, mcpServers: [], allowedBaseUrls: [] });
expect(await r.serverApiKey()).toBeNull();
await r.saveServer({ provider: { kind: "anthropic", model: "claude-opus-5-5", apiKey: "sk-ant-SECRET" }, mcpServers: [] });
const got = await r.getServer();
expect(got.provider).toEqual({ kind: "anthropic", model: "claude-opus-5-5", hasApiKey: true });
expect(JSON.stringify(got)).not.toContain("SECRET");
const raw = await db.sql`SELECT config_json FROM ai_settings WHERE scope = 'server'`;
expect(raw[0].config_json).not.toContain("SECRET");
expect(await r.serverApiKey()).toBe("sk-ant-SECRET");
// Saving without apiKey keeps the old key; clearApiKey removes it.
await r.saveServer({ provider: { kind: "anthropic", model: "other" }, mcpServers: [{ id: "fs", name: "Files", transport: "stdio", command: "x" }] });
expect(await r.serverApiKey()).toBe("sk-ant-SECRET");
expect((await r.getServer()).mcpServers).toHaveLength(1);
await r.saveServer({ provider: { kind: "openai", baseUrl: "http://ollama:11434/v1", model: "llama3", clearApiKey: true }, mcpServers: [] });
expect(await r.serverApiKey()).toBeNull();
expect((await r.getServer()).provider).toEqual({ kind: "openai", baseUrl: "http://ollama:11434/v1", model: "llama3", hasApiKey: false });
await r.saveServer({ provider: null, mcpServers: [] });
expect((await r.getServer()).provider).toBeNull();
});
forEachDb("BYOK keys are encrypted, per user and removable", async (db) => {
const r = aiSettingsRepo(db, box);
expect((await r.getUser("u1")).byok).toBeNull();
await r.saveUserKey("u1", { kind: "anthropic", apiKey: "sk-user-SECRET" });
expect((await r.getUser("u1")).byok).toEqual({ kind: "anthropic", hasApiKey: true });
expect((await r.getUser("u2")).byok).toBeNull();
const raw = await db.sql`SELECT ciphertext FROM user_api_keys WHERE user_id = 'u1'`;
expect(raw[0].ciphertext).not.toContain("SECRET");
expect(await r.userApiKey("u1")).toBe("sk-user-SECRET");
expect(await r.userApiKey("u2")).toBeNull();
// replacing with another provider drops the old key
await r.saveUserKey("u1", { kind: "openai", baseUrl: "https://api.example.com/v1", model: "m", apiKey: "k2" });
expect(await r.userApiKey("u1")).toBe("k2");
expect((await db.sql`SELECT COUNT(*) AS n FROM user_api_keys WHERE user_id = 'u1'`)[0].n.toString()).toBe("1");
await r.removeUserKey("u1");
expect((await r.getUser("u1")).byok).toBeNull();
expect(await r.userApiKey("u1")).toBeNull();
});
forEachDb("mentor mode is on by default and toggles per course", async (db) => {
const r = aiSettingsRepo(db, box);
expect(await r.isMentorOff("u1", "sample")).toBe(false);
await r.setMentorOff("u1", "sample", true);
expect(await r.isMentorOff("u1", "sample")).toBe(true);
expect(await r.isMentorOff("u1", "pure-snake")).toBe(false);
expect(await r.isMentorOff("u2", "sample")).toBe(false);
await r.saveUserKey("u1", { kind: "anthropic", apiKey: "k" });
expect(await r.isMentorOff("u1", "sample")).toBe(true);
await r.setMentorOff("u1", "sample", false);
expect(await r.isMentorOff("u1", "sample")).toBe(false);
expect((await r.getUser("u1")).byok?.kind).toBe("anthropic");
});
packages/feature-ai/test/web.test.tsx
+56
View File
@@ -0,0 +1,56 @@
import { expect, test } from "bun:test";
import { renderToString } from "react-dom/server";
import { readNdjson } from "../src/web/api";
import { MentorBanner, Markdown } from "../src/web/components";
import { aiWeb } from "../src/web";
test("readNdjson splits events across chunk boundaries", async () => {
const text = '{"type":"meta","n":1}\n{"type":"text","text":"héllo"}\n{"type":"done"}\n';
const bytes = new TextEncoder().encode(text);
const stream = new ReadableStream<Uint8Array>({
start(c) { for (let i = 0; i < bytes.length; i += 5) c.enqueue(bytes.slice(i, i + 5)); c.close(); },
});
const out: unknown[] = [];
for await (const ev of readNdjson(stream)) out.push(ev);
expect(out).toEqual([{ type: "meta", n: 1 }, { type: "text", text: "héllo" }, { type: "done" }]);
});
test("the mentor banner says so plainly when mentor mode is off", () => {
expect(renderToString(<MentorBanner on={false} available />)).toContain("Mentor mode off: I&#x27;ll give full answers");
expect(renderToString(<MentorBanner on available />)).toBe("");
expect(renderToString(<MentorBanner on={false} available={false} />)).toBe("");
});
test("markdown renders code and text without raw HTML", () => {
const html = renderToString(<Markdown text={"Use **this**:\n\n```js\nconst a = 1 < 2\n```\n\n- one `x`\n- <b>two</b>"} />);
expect(html).toContain("<strong>this</strong>");
expect(html).toContain('<pre class="code">');
expect(html).toContain("const a = 1 &lt; 2");
expect(html).toContain("<code>x</code>");
expect(html).toContain("&lt;b&gt;two&lt;/b&gt;");
});
test("web routes", () => {
expect(aiWeb.routes.map((r) => r.path)).toEqual(["/c/:courseId/assistant", "/admin/ai", "/account/ai", "/studio/generate"]);
expect(aiWeb.routes.find((r) => r.path === "/admin/ai")!.nav).toEqual({ label: "AI", area: "admin", roles: ["owner"] });
});
test("lesson links go through one builder", async () => {
const { lessonUrl, courseUrl } = await import("../src/web/urls");
expect(lessonUrl("sample", "m2-ex-add")).toBe("/c/sample/m2-ex-add");
expect(lessonUrl("S08 - Docker", "l 1/x")).toBe("/c/S08%20-%20Docker/l%201%2Fx");
expect(courseUrl("sample")).toBe("/c/sample");
const src = await Bun.file(new URL("../src/web/AssistantPage.tsx", import.meta.url)).text();
expect(src).toContain("lessonUrl(courseId, c.lessonId)");
expect(src).not.toMatch(/`\/c\//);
});
test("citation links use the learner's registered lesson route", async () => {
const { learnerWeb } = await import("@stc/feature-learner/web");
const { matchPath } = await import("react-router-dom");
const { lessonUrl } = await import("../src/web/urls");
const url = lessonUrl("pure snake", "m1-ü/x");
const hit = learnerWeb.routes.map((r) => matchPath(r.path, url)).find((m) => m && m.pattern.path === "/c/:courseId/:lessonId");
// matchPath leaves params encoded (the router decodes the pathname before matching).
expect(Object.fromEntries(Object.entries(hit?.params ?? {}).map(([k, v]) => [k, decodeURIComponent(v!)]))).toEqual({ courseId: "pure snake", lessonId: "m1-ü/x" });
});
packages/feature-import/dev/serve.ts
+23
View File
@@ -0,0 +1,23 @@
// Dev server for the import UI while real sign-in (learner package) is not merged: everyone is the owner.
// Usage: STC_PORT=3462 STC_DATA_DIR=/tmp/stc-import-data STC_PYTHON=… bun packages/feature-import/dev/serve.ts
import { resolve } from "node:path";
import type { AuthService, ServerContext } from "@stc/contracts";
import { createApp, createJobQueue, loadConfig } from "@stc/core";
import { createCourseRepository } from "@stc/course-format";
import { connect, migrate } from "@stc/db";
import indexHtml from "../../../apps/web/index.html";
import { importFeature } from "../src";
const config = loadConfig();
config.courseDirs.push(resolve(config.dataDir, "courses"));
const db = await connect(config.databaseUrl);
await migrate(db);
const auth: AuthService = { userFromRequest: async () => ({ id: "dev", email: "dev@localhost", name: "Dev owner", roles: ["owner"], groups: [] }) };
const ctx = { config, db, courses: await createCourseRepository(config.courseDirs), auth } as ServerContext;
const feature = importFeature(ctx);
const jobs = createJobQueue(db, feature.jobs!);
ctx.jobs = jobs;
const app = createApp(ctx, [feature]);
jobs.start();
const server = Bun.serve({ port: config.port, routes: { ...app.routes, "/api/*": (req) => app.fetch(req), "/*": indexHtml }, fetch: (req) => app.fetch(req) });
console.log(`import dev server on ${server.url}`);
packages/feature-import/package.json
+3 -1
View File
@@ -3,5 +3,7 @@
"version": "0.0.0",
"private": true,
"type": "module",
"license": "AGPL-3.0-only"
"license": "AGPL-3.0-only",
"main": "src/index.ts",
"exports": { ".": "./src/index.ts", "./web": "./src/web.ts" }
}
packages/feature-import/py/pdf_tool.py
+50
View File
@@ -0,0 +1,50 @@
"""PDF helper for the STCourses importer. Run as: python -I pdf_tool.py render <pdf> <outdir> [--width 1280] [--quality 80]
Writes outdir/p-001.jpg ... and outdir/text.json ([{"page": 1, "text": "..."}]).
Prints one "page N/M" line per page on stdout and a final JSON line {"pages": M, "title": "..."}.
"""
import json
import os
import sys
try:
import pymupdf as fitz
except ImportError: # older PyMuPDF
import fitz
def render(pdf, outdir, width, quality):
os.makedirs(outdir, exist_ok=True)
doc = fitz.open(pdf)
n = len(doc)
texts = []
for i, page in enumerate(doc, start=1):
zoom = width / max(page.rect.width, 1)
pix = page.get_pixmap(matrix=fitz.Matrix(zoom, zoom), alpha=False)
pix.save(os.path.join(outdir, "p-%03d.jpg" % i), jpg_quality=quality)
texts.append({"page": i, "text": page.get_text().strip()})
print("page %d/%d" % (i, n), flush=True)
with open(os.path.join(outdir, "text.json"), "w", encoding="utf-8") as f:
json.dump(texts, f, ensure_ascii=False)
print(json.dumps({"pages": n, "title": (doc.metadata or {}).get("title") or ""}), flush=True)
def main(argv):
if len(argv) < 4 or argv[1] != "render":
print("usage: pdf_tool.py render <pdf> <outdir> [--width N] [--quality N]", file=sys.stderr)
return 2
width, quality = 1280, 80
rest = argv[4:]
while rest:
flag, val = rest[0], rest[1] if len(rest) > 1 else ""
if flag == "--width":
width = int(val)
elif flag == "--quality":
quality = int(val)
rest = rest[2:]
render(argv[2], argv[3], width, quality)
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv))
packages/feature-import/src/context.ts
+112
View File
@@ -0,0 +1,112 @@
import { mkdir, readdir, rename, stat } from "node:fs/promises";
import { join, relative, resolve } from "node:path";
import type { ServerContext } from "@stc/contracts";
import { readRawManifest, writeManifest, type RawManifest } from "./util/manifest";
import { slugify, uniqueId } from "./util/text";
import { extractFrame, type Tools } from "./media";
export interface ImportEnv { [k: string]: string | undefined }
/** `env` supplies only tool paths and transcription credentials (STC_FFMPEG, STC_FFPROBE, STC_TRANSCRIBE_KEY, _MODEL). */
export interface ImportOptions { env?: ImportEnv }
export interface Deps {
ctx: ServerContext;
env: ImportEnv;
tools: Tools;
/** AppConfig.python (STC_PYTHON). */
python: string;
/** AppConfig.importRoots (STC_IMPORT_ROOTS). */
importRoots: string[];
/** AppConfig.transcribeUrl (STC_TRANSCRIBE_URL). */
transcribeUrl: string | null;
}
export const coursesDir = (d: Deps) => resolve(d.ctx.config.importedCoursesDir ?? resolve(d.ctx.config.dataDir, "courses"));
export const cacheDir = (d: Deps) => resolve(d.ctx.config.cacheDir ?? resolve(d.ctx.config.dataDir, "cache"));
const studioRepo = (d: Deps, id: string) => join(d.ctx.config.studioReposDir ?? resolve(d.ctx.config.dataDir, "studio", "repos"), `${id}.git`);
export const deckCacheDir = (d: Deps, courseId: string, deckId: string) => join(cacheDir(d), "decks", courseId, deckId);
export const coverCacheDir = (d: Deps, courseId: string) => join(cacheDir(d), "covers", courseId);
export const transcriptCacheDir = (d: Deps, courseId: string) => join(cacheDir(d), "transcripts", courseId);
export const isPlainId = (s: string) => /^[a-z0-9][a-z0-9-]*$/.test(s);
/** A fresh course id from a title, unique among managed folders and every course the server already knows. */
export async function freshCourseId(d: Deps, wanted: string | undefined, title: string): Promise<string> {
if (wanted !== undefined) {
if (!isPlainId(wanted)) throw new Error(`invalid courseId "${wanted}" (use lowercase letters, digits and dashes)`);
// Studio repos share the id space (ruling R9): refuse an id the studio already uses, even before it is published.
if (await exists(join(coursesDir(d), wanted)) || await exists(studioRepo(d, wanted)) || d.ctx.courses.get(wanted)) throw new Error(`a course with id "${wanted}" already exists`);
return wanted;
}
const taken = new Set<string>(d.ctx.courses.list().map((c) => c.id));
for (const e of await readdir(coursesDir(d)).catch(() => [] as string[])) taken.add(e);
const repos = d.ctx.config.studioReposDir ?? resolve(d.ctx.config.dataDir, "studio", "repos");
for (const e of await readdir(repos).catch(() => [] as string[])) if (e.endsWith(".git")) taken.add(e.slice(0, -4));
return uniqueId(slugify(title), (x) => taken.has(x));
}
export async function exists(p: string): Promise<boolean> {
return stat(p).then(() => true, () => false);
}
/** Scratch folder on the same filesystem as the courses, so finished work is moved in with one rename. */
export async function stageDir(d: Deps, label: string): Promise<string> {
const dir = resolve(d.ctx.config.dataDir, "tmp", `${label}-${Bun.randomUUIDv7()}`);
await mkdir(dir, { recursive: true });
return dir;
}
export async function publishCourse(d: Deps, stage: string, id: string): Promise<string> {
await mkdir(coursesDir(d), { recursive: true });
const dest = join(coursesDir(d), id);
await rename(stage, dest);
return dest;
}
export interface ManagedCourse { id: string; dir: string; manifest: RawManifest }
/** An existing course the importer may add to. Only folders directly under <dataDir>/courses qualify: overlays and authored repos are never edited. */
export async function managedCourse(d: Deps, courseId: string | undefined): Promise<ManagedCourse> {
if (!courseId) throw new Error("options.courseId is required: choose the course to add to");
const dir = join(coursesDir(d), courseId);
const rel = relative(coursesDir(d), dir);
if (!isPlainId(courseId) || rel.startsWith("..") || !(await exists(join(dir, "stcourse.yaml")))) {
throw new Error(`course "${courseId}" is not an imported course under ${coursesDir(d)}; the importer only edits courses it created`);
}
const manifest = await readRawManifest(dir);
if (manifest.id !== courseId) throw new Error(`course folder "${courseId}" has manifest id "${manifest.id}"`);
return { id: courseId, dir, manifest };
}
/** Reloads the repository and turns load problems into warnings. */
export async function finish(d: Deps, id: string, warnings: string[]): Promise<void> {
await d.ctx.courses.reload(id);
const c = d.ctx.courses.get(id);
if (!c) {
warnings.push(`course "${id}" is not visible to the server: add ${coursesDir(d)} to STC_COURSE_DIRS`);
return;
}
for (const diag of c.diagnostics) warnings.push(`${diag.level}: ${diag.message}${diag.path ? ` (${diag.path})` : ""}`);
}
export async function saveManifest(dir: string, manifest: RawManifest): Promise<void> {
await writeManifest(dir, manifest);
}
export function sourceBlock(kind: string, origin: string) {
return { kind, origin, importedAt: new Date().toISOString() };
}
/** Writes up to `count` frame candidates from evenly spaced clips; returns the candidate file names. Failures are skipped. */
export async function frameCandidates(d: Deps, courseId: string, clips: string[], count = 3): Promise<string[]> {
const out: string[] = [];
const n = Math.min(count, clips.length);
for (let i = 0; i < n; i++) {
const name = `frame-${i + 1}.jpg`;
try {
await extractFrame(d.tools, clips[Math.floor((i * clips.length) / n)]!, join(coverCacheDir(d, courseId), name), 0.2);
out.push(name);
} catch { /* a bad clip must not fail the import */ }
}
return out;
}
packages/feature-import/src/deck.ts
+49
View File
@@ -0,0 +1,49 @@
import { basename, extname, join } from "node:path";
import { copyFile, mkdir } from "node:fs/promises";
import { coverCacheDir, deckCacheDir, exists, type Deps, type ManagedCourse } from "./context";
import { putFile } from "./media";
import { renderPdf } from "./pdf";
import { slugify, uniqueId } from "./util/text";
export interface AttachedDeck { deckId: string; file: string; pages?: number; warning?: string }
/** All lesson and module ids of a raw manifest, for validating "attach to". */
export function attachTargets(manifest: Record<string, any>): Set<string> {
const ids = new Set<string>();
for (const m of manifest.modules ?? []) {
ids.add(m.id);
for (const l of m.lessons ?? []) ids.add(l.id);
}
return ids;
}
/**
* Copies a PDF into the course (media/decks/), registers it in the (in-memory) manifest, and builds the derived cache:
* page images + text.json under <dataDir>/cache/decks/<course>/<deck>/ and the first page as a cover candidate.
* A failure of the PDF tool only degrades to a warning: the deck stays attached and the cache can be rebuilt.
* The caller writes the manifest.
*/
export async function attachDeck(d: Deps, course: ManagedCourse, pdfPath: string, attachTo: string, opts: { title?: string; mode?: "copy" | "link"; onProgress?: (fraction: number) => void } = {}): Promise<AttachedDeck> {
if (!attachTarget(course, attachTo)) throw new Error(`attachTo "${attachTo}" is not a lesson or module of course "${course.id}"`);
const decks: any[] = (course.manifest.decks ??= []);
const baseName = basename(pdfPath, extname(pdfPath));
const deckId = uniqueId(`d-${slugify(baseName)}`, (x) => decks.some((k) => k.id === x));
let file = `media/decks/${basename(pdfPath)}`;
for (let n = 2; await exists(join(course.dir, file)); n++) file = `media/decks/${baseName}-${n}${extname(pdfPath)}`;
await putFile(pdfPath, join(course.dir, file), opts.mode ?? "copy");
decks.push({ id: deckId, file, title: opts.title ?? baseName.replace(/[_]+/g, " ").trim(), attachTo });
const out: AttachedDeck = { deckId, file };
try {
const r = await renderPdf(d.python, join(course.dir, file), deckCacheDir(d, course.id, deckId), { onPage: (p, t) => opts.onProgress?.(p / t) });
out.pages = r.pages;
await mkdir(coverCacheDir(d, course.id), { recursive: true });
await copyFile(join(deckCacheDir(d, course.id, deckId), "p-001.jpg"), join(coverCacheDir(d, course.id), `deck-${deckId}.jpg`));
} catch (e) {
out.warning = `deck "${basename(pdfPath)}" is attached, but page images and text could not be built (${e instanceof Error ? e.message.split("\n")[0] : e}); STC_PYTHON must point to Python with PyMuPDF`;
}
return out;
}
const attachTarget = (c: ManagedCourse, id: string) => attachTargets(c.manifest).has(id);
packages/feature-import/src/index.ts
+50
View File
@@ -0,0 +1,50 @@
import { resolve } from "node:path";
import type { Feature, JobHandler, ServerContext } from "@stc/contracts";
import type { Deps, ImportOptions } from "./context";
import { importCodeRepo } from "./jobs/code-repo";
import { importPacktVideo } from "./jobs/packt-video";
import { importPdfDeck } from "./jobs/pdf-deck";
import { transcribeCourse } from "./jobs/transcribe";
import { importVideoFolder } from "./jobs/video-folder";
import { ensureRoots } from "./roots";
import { importRoutes } from "./routes";
export type { ImportOptions } from "./context";
export { deckCacheDir, coverCacheDir, transcriptCacheDir } from "./context";
/**
* `ctx` is only read when a job runs or a request arrives, never while this function runs, so the server can pass
* its context object before the job queue (which needs these handlers) exists.
* Config: AppConfig.python, importRoots, transcribeUrl. Environment: STC_FFMPEG / STC_FFPROBE, STC_TRANSCRIBE_KEY / _MODEL.
*/
export function importFeature(ctx: ServerContext, opts: ImportOptions = {}): Feature {
const deps = (): Deps => {
const env = opts.env ?? process.env;
const c = ctx.config;
return {
ctx, env,
tools: { ffmpeg: env.STC_FFMPEG || "ffmpeg", ffprobe: env.STC_FFPROBE || "ffprobe" },
python: c.python || "python3",
importRoots: c.importRoots?.length ? c.importRoots : [resolve(c.dataDir, "import-inbox")],
transcribeUrl: c.transcribeUrl ?? null,
};
};
const job = (fn: (d: Deps, p: unknown, jc: Parameters<JobHandler>[1]) => Promise<unknown>): JobHandler => (p, jc) => fn(deps(), p, jc);
ensureRoots(deps());
return {
name: "import",
routes: importRoutes(deps),
jobs: {
"import.packt-video": job(importPacktVideo),
"import.code-repo": job(importCodeRepo),
"import.pdf-deck": job(importPdfDeck),
"import.video-folder": job(importVideoFolder),
"import.transcribe": job(transcribeCourse),
},
};
}
export { importWeb } from "./web";
/** The importer adds no tables. */
export const migrationsDirs: string[] = [];
packages/feature-import/src/jobs/code-repo.ts
+135
View File
@@ -0,0 +1,135 @@
import { cp, lstat, mkdir, readdir } from "node:fs/promises";
import { basename, extname, join } from "node:path";
import type { ImportResult, JobContext } from "@stc/contracts";
import { finish, managedCourse, saveManifest, stageDir, type Deps } from "../context";
import { attachDeck } from "../deck";
import { putFile } from "../media";
import { run } from "../util/exec";
import { assertAllowedPath } from "../roots";
import { openSource } from "../util/source";
import { containment } from "../util/text";
import { optStr, parsePayload, withStage } from "./common";
const GIT_URL = /^(https?:\/\/|ssh:\/\/|git@)[^\s]+$/;
const SKIP_DIRS = new Set([".git", "node_modules", "__pycache__", ".venv"]);
const NUMBER = /^\s*(?:s|sec(?:tion)?|mod(?:ule)?)?\s*0*(\d+)\b\s*[-–—:.)_]*\s*/i;
export function folderNumber(name: string): number | null {
const m = NUMBER.exec(name);
return m ? Number(m[1]) : null;
}
export function folderName(name: string): string {
return name.replace(NUMBER, "").trim();
}
function moduleNumber(m: { id: string; title: string }, index: number): number {
const byId = /^m0*(\d+)$/i.exec(m.id);
if (byId) return Number(byId[1]);
return folderNumber(m.title) ?? index + 1;
}
/** Single-directory wrappers ("Repo-main/…") of a zip are skipped. */
async function effectiveRoot(dir: string): Promise<string> {
for (let i = 0; i < 3; i++) {
const entries = (await readdir(dir, { withFileTypes: true })).filter((e) => !e.name.startsWith(".") && e.name !== "__MACOSX");
if (entries.length === 1 && entries[0]!.isDirectory()) dir = join(dir, entries[0]!.name);
else break;
}
return dir;
}
async function findPdfs(dir: string, depth = 0, out: string[] = []): Promise<string[]> {
for (const e of await readdir(dir, { withFileTypes: true })) {
if (SKIP_DIRS.has(e.name)) continue;
if (e.isDirectory() && depth < 2) await findPdfs(join(dir, e.name), depth + 1, out);
else if (e.isFile() && extname(e.name).toLowerCase() === ".pdf") out.push(join(dir, e.name));
}
return out;
}
export async function importCodeRepo(d: Deps, payload: unknown, jc: JobContext): Promise<ImportResult> {
const { path, options } = parsePayload(payload, "code-repo");
const warnings: string[] = [];
const course = await managedCourse(d, optStr(options, "courseId"));
if (!GIT_URL.test(path)) await assertAllowedPath(d, path);
const stage = await stageDir(d, "repo");
return withStage(stage, async () => {
let root: string;
if (GIT_URL.test(path)) {
await jc.reportProgress(0.05, "Cloning the repository");
await run(["git", "clone", "--depth", "1", "--", path, join(stage, "repo")], { signal: jc.signal, env: { GIT_TERMINAL_PROMPT: "0" } });
root = join(stage, "repo");
} else if (/\.zip$/i.test(path)) {
await jc.reportProgress(0.05, "Unpacking the repository zip");
const src = await openSource(path);
await src.extract(src.entries.filter((e) => !e.startsWith("__MACOSX/")), join(stage, "repo"), jc.signal);
root = await effectiveRoot(join(stage, "repo")); // GitHub-style zips wrap everything in one folder
} else {
root = path; // an extracted folder is only read
}
const modules: any[] = course.manifest.modules ?? [];
const folders = (await readdir(root, { withFileTypes: true })).filter((e) => e.isDirectory() && !SKIP_DIRS.has(e.name) && !e.name.startsWith(".")).map((e) => e.name).sort();
const attachments: any[] = (course.manifest.attachments ??= []);
const claimed = new Set<string>();
let matched = 0;
for (const [i, folder] of folders.entries()) {
await jc.reportProgress(0.1 + 0.8 * (i / Math.max(1, folders.length)), `Matching ${folder}`);
const num = folderNumber(folder);
const name = folderName(folder);
let mod = num !== null ? modules.find((m, idx) => moduleNumber(m, idx) === num) : undefined;
if (mod && name && containment(name, mod.title) < 0.3 && containment(mod.title, name) < 0.3) {
warnings.push(`"${folder}" was matched to module ${num} ("${mod.title}") by number only; the names differ`);
}
if (!mod && name) {
let best = 0;
for (const m of modules) {
const score = containment(name, m.title);
if (score > best) { best = score; mod = m; }
}
if (best < 0.6) mod = undefined;
}
if (!mod) { warnings.push(`folder "${folder}" matches no module and was skipped`); continue; }
if (claimed.has(mod.id)) { warnings.push(`folder "${folder}" also matches module "${mod.id}", which already got code; skipped`); continue; }
if (mod.code) { warnings.push(`module "${mod.id}" already has a code folder; "${folder}" skipped`); continue; }
claimed.add(mod.id);
matched++;
const srcDir = join(root, folder);
const dest = `code/${folder}`;
await mkdir(join(course.dir, "code"), { recursive: true });
await cp(srcDir, join(course.dir, dest), {
recursive: true,
filter: async (s) => {
const base = basename(s);
if (SKIP_DIRS.has(base)) return false;
const st = await lstat(s);
if (st.isSymbolicLink()) return false; // a link could point outside the course
return !(st.isFile() && extname(base).toLowerCase() === ".pdf");
},
});
mod.code = { dir: dest };
for (const pdf of await findPdfs(srcDir)) {
const title = basename(pdf, ".pdf").replace(/_/g, " ").trim();
if (/cheat|quick.?ref|reference|handout/i.test(basename(pdf))) {
let rel = `media/files/${basename(pdf)}`;
await putFile(pdf, join(course.dir, rel));
attachments.push({ file: rel, title, module: mod.id });
} else {
const r = await attachDeck(d, course, pdf, mod.id, { title });
if (r.warning) warnings.push(r.warning);
}
}
}
if (attachments.length === 0) delete course.manifest.attachments;
if (matched === 0) warnings.push("no folder of the repository matched a module; nothing was attached");
await saveManifest(course.dir, course.manifest);
await finish(d, course.id, warnings);
await jc.reportProgress(1, `Attached code to ${matched} module(s)`);
return { courseId: course.id, manifestPath: join(course.dir, "stcourse.yaml"), warnings };
});
}
packages/feature-import/src/jobs/common.ts
+48
View File
@@ -0,0 +1,48 @@
import type { ImportResult, ImportSource, JobContext } from "@stc/contracts";
import { copyFile, mkdir, rm } from "node:fs/promises";
import { dirname, join } from "node:path";
import { coverCacheDir, exists, type Deps } from "../context";
export type Opts = Record<string, unknown>;
export function parsePayload(payload: unknown, kind: ImportSource["kind"]): { path: string; options: Opts } {
const p = payload as Partial<ImportSource> | null;
if (!p || typeof p.path !== "string" || p.path === "") throw new Error("payload.path is required");
if (p.kind !== undefined && p.kind !== kind) throw new Error(`payload.kind "${p.kind}" does not match job kind "${kind}"`);
return { path: p.path, options: (p.options ?? {}) as Opts };
}
export const optStr = (o: Opts, k: string): string | undefined => (typeof o[k] === "string" && (o[k] as string).trim() !== "" ? (o[k] as string).trim() : undefined);
export const optBool = (o: Opts, k: string): boolean => o[k] === true || o[k] === "true";
/** Runs a job body with a scratch folder that is always removed, and a result shape shared by all import jobs. */
export async function withStage<T>(stage: string, fn: () => Promise<T>): Promise<T> {
try {
return await fn();
} finally {
await rm(stage, { recursive: true, force: true });
}
}
/** Copies the first cover candidate into the course (relative path returned) so the library has a cover straight away. */
export async function adoptCover(d: Deps, courseId: string, courseDir: string, candidate: string | undefined): Promise<string | undefined> {
if (!candidate) return undefined;
const src = join(coverCacheDir(d, courseId), candidate);
if (!(await exists(src))) return undefined;
const rel = "media/cover.jpg";
await mkdir(dirname(join(courseDir, rel)), { recursive: true });
await copyFile(src, join(courseDir, rel));
return rel;
}
export async function maybeQueueTranscription(d: Deps, courseId: string, wanted: boolean, warnings: string[]): Promise<void> {
if (!wanted) return;
if (!d.transcribeUrl) {
warnings.push("transcription requested but STC_TRANSCRIBE_URL is not set; skipped");
return;
}
await d.ctx.jobs.enqueue("import.transcribe", { courseId });
}
export type Reporter = JobContext["reportProgress"];
export type { ImportResult };
packages/feature-import/src/jobs/packt-video.ts
+89
View File
@@ -0,0 +1,89 @@
import { mkdir, rename, rm } from "node:fs/promises";
import { basename, dirname, join, posix } from "node:path";
import type { ImportResult, JobContext } from "@stc/contracts";
import { finish, frameCandidates, freshCourseId, publishCourse, saveManifest, sourceBlock, stageDir, type Deps } from "../context";
import { probeDuration } from "../media";
import { parsePacktIndex } from "../packt";
import { assertAllowedPath } from "../roots";
import { openSource } from "../util/source";
import { adoptCover, maybeQueueTranscription, optBool, optStr, parsePayload, withStage } from "./common";
export async function importPacktVideo(d: Deps, payload: unknown, jc: JobContext): Promise<ImportResult> {
const { path, options } = parsePayload(payload, "packt-video");
const warnings: string[] = [];
await assertAllowedPath(d, path);
await jc.reportProgress(0.02, "Reading the package");
const src = await openSource(path);
// The player page is the shallowest index.html that lists sections.
const candidates = src.entries.filter((e) => /(^|\/)index\.html$/i.test(e)).sort((a, b) => a.split("/").length - b.split("/").length);
let indexEntry: string | undefined;
let index: ReturnType<typeof parsePacktIndex> | undefined;
for (const c of candidates) {
try { index = parsePacktIndex(await src.readText(c)); indexEntry = c; break; } catch { /* try the next */ }
}
if (!index || indexEntry === undefined) throw new Error("no Packt player index.html (with \"Section N: …\" headings) found in the package");
warnings.push(...index.warnings);
const base = dirname(indexEntry) === "." ? "" : dirname(indexEntry);
const at = (rel: string) => posix.normalize(base ? `${base}/${rel}` : rel);
const title = optStr(options, "title") ?? index.title;
const id = await freshCourseId(d, optStr(options, "courseId"), title);
const stage = await stageDir(d, "packt");
return withStage(stage, async () => {
const known = new Set(src.entries);
const modules: any[] = [];
const used = new Set<string>();
const clipFiles: string[] = [];
const total = index.sections.reduce((n, s) => n + s.clips.length, 0) || 1;
let done = 0;
for (const sec of index.sections) {
const mid = `m${sec.number}`;
const entries: { clip: (typeof sec.clips)[number]; entry: string }[] = [];
for (const clip of sec.clips) {
const entry = at(clip.video);
if (!known.has(entry)) { warnings.push(`video for ${sec.number}.${clip.number} is missing from the package: ${clip.video}`); continue; }
entries.push({ clip, entry });
used.add(entry);
}
await jc.reportProgress(0.05 + 0.65 * (done / total), `Copying section ${sec.number}: ${sec.title}`);
const tmp = join(stage, ".extract");
await src.extract(entries.map((e) => e.entry), tmp, jc.signal);
const lessons: any[] = [];
for (const { clip, entry } of entries) {
const rel = `media/${basename(entry)}`;
await mkdir(join(stage, "media"), { recursive: true });
await rename(join(tmp, entry), join(stage, rel));
clipFiles.push(join(stage, rel));
const duration = await probeDuration(d.tools, join(stage, rel));
lessons.push({ id: `${mid}-l${clip.number}`, title: clip.title, type: "video", file: rel, ...(duration ? { duration } : {}) });
done++;
}
await rm(tmp, { recursive: true, force: true });
modules.push({ id: mid, title: sec.title.replace(/^Section\s+\d+\s*[:.-]\s*/i, ""), lessons });
}
const unused = src.entries.filter((e) => /\.(mp4|webm|m4v|mov)$/i.test(e) && !used.has(e));
if (unused.length) warnings.push(`${unused.length} video file(s) in the package are not listed in index.html and were left out`);
await jc.reportProgress(0.75, "Making cover candidates");
const covers = await frameCandidates(d, id, clipFiles);
if (covers.length === 0 && clipFiles.length) warnings.push("could not extract cover frames (is ffmpeg available?)");
const cover = await adoptCover(d, id, stage, covers[0]);
const manifest = {
schema: 1, id, title,
visibility: "restricted",
...(cover ? { cover } : {}),
source: sourceBlock("packt-video", basename(path)),
modules,
};
await saveManifest(stage, manifest);
await jc.reportProgress(0.9, "Publishing the course");
const dir = await publishCourse(d, stage, id);
await finish(d, id, warnings);
await maybeQueueTranscription(d, id, optBool(options, "transcribe"), warnings);
await jc.reportProgress(1, `Imported ${total} clips`);
return { courseId: id, manifestPath: join(dir, "stcourse.yaml"), warnings };
});
}
packages/feature-import/src/jobs/pdf-deck.ts
+28
View File
@@ -0,0 +1,28 @@
import { stat } from "node:fs/promises";
import { join } from "node:path";
import type { ImportResult, JobContext } from "@stc/contracts";
import { finish, managedCourse, saveManifest, type Deps } from "../context";
import { assertAllowedPath } from "../roots";
import { attachDeck } from "../deck";
import { optStr, parsePayload } from "./common";
export async function importPdfDeck(d: Deps, payload: unknown, jc: JobContext): Promise<ImportResult> {
const { path, options } = parsePayload(payload, "pdf-deck");
const warnings: string[] = [];
await assertAllowedPath(d, path);
if (!(await stat(path).catch(() => null))?.isFile() || !/\.pdf$/i.test(path)) throw new Error(`not a PDF file: ${path}`);
const course = await managedCourse(d, optStr(options, "courseId"));
const attachTo = optStr(options, "attachTo");
if (!attachTo) throw new Error("options.attachTo is required: the lesson or module id the deck belongs to");
await jc.reportProgress(0.05, "Copying the deck");
const r = await attachDeck(d, course, path, attachTo, {
title: optStr(options, "title"),
mode: options.media === "link" ? "link" : "copy",
onProgress: (f) => void jc.reportProgress(0.1 + 0.8 * f, "Rendering slides"),
});
if (r.warning) warnings.push(r.warning);
await saveManifest(course.dir, course.manifest);
await finish(d, course.id, warnings);
await jc.reportProgress(1, r.pages ? `Deck attached (${r.pages} slides)` : "Deck attached");
return { courseId: course.id, manifestPath: join(course.dir, "stcourse.yaml"), warnings };
}
packages/feature-import/src/jobs/transcribe.ts
+82
View File
@@ -0,0 +1,82 @@
import { mkdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
import type { JobContext } from "@stc/contracts";
import { finish, managedCourse, saveManifest, stageDir, transcriptCacheDir, type Deps } from "../context";
import { run } from "../util/exec";
import { optStr, withStage } from "./common";
export interface TranscribeResult { courseId: string; skipped?: boolean; reason?: string; transcribed: number; warnings: string[] }
interface Segment { start: number; end: number; text: string }
const stamp = (s: number) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
/** Groups timed segments into paragraphs of roughly 30 seconds, each starting with a [m:ss] marker. */
export function segmentsToMarkdown(title: string, segments: Segment[], fallbackText: string): string {
if (segments.length === 0) return `# ${title}\n\n${fallbackText.trim()}\n`;
const paras: string[] = [];
let cur: string[] = [];
let startedAt = segments[0]!.start;
for (const s of segments) {
if (cur.length && s.start - startedAt >= 30) {
paras.push(`[${stamp(startedAt)}] ${cur.join(" ")}`);
cur = []; startedAt = s.start;
}
cur.push(s.text.trim());
}
if (cur.length) paras.push(`[${stamp(startedAt)}] ${cur.join(" ")}`);
return `# ${title}\n\n${paras.join("\n\n")}\n`;
}
/**
* Optional job: sends each video lesson's audio to a Whisper-compatible endpoint (STC_TRANSCRIBE_URL, OpenAI
* /v1/audio/transcriptions shape) and attaches the result as the lesson's transcript. The model never runs in this process.
*/
export async function transcribeCourse(d: Deps, payload: unknown, jc: JobContext): Promise<TranscribeResult> {
const p = (payload ?? {}) as Record<string, unknown>;
const url = d.transcribeUrl;
const courseId = optStr(p, "courseId") ?? "";
if (!url) return { courseId, skipped: true, reason: "STC_TRANSCRIBE_URL is not set", transcribed: 0, warnings: [] };
const course = await managedCourse(d, courseId);
const only = Array.isArray(p.lessonIds) ? new Set(p.lessonIds.map(String)) : null;
const lessons = (course.manifest.modules as any[]).flatMap((m) => m.lessons as any[]).filter((l) => l.type === "video" && !l.transcript && (!only || only.has(l.id)));
const warnings: string[] = [];
const stage = await stageDir(d, "transcribe");
let done = 0;
return withStage(stage, async () => {
try {
for (const [i, l] of lessons.entries()) {
if (jc.signal.aborted) break;
await jc.reportProgress(i / Math.max(1, lessons.length), `Transcribing ${l.title}`);
try {
const audio = join(stage, `${l.id}.ogg`);
await run([d.tools.ffmpeg, "-v", "error", "-y", "-i", join(course.dir, l.file), "-vn", "-ac", "1", "-ar", "16000", "-c:a", "libopus", "-b:a", "24k", audio], { signal: jc.signal });
const form = new FormData();
form.set("file", Bun.file(audio), `${l.id}.ogg`);
form.set("model", d.env.STC_TRANSCRIBE_MODEL || "whisper-1");
form.set("response_format", "verbose_json");
const res = await fetch(url, {
method: "POST", body: form, signal: jc.signal,
headers: d.env.STC_TRANSCRIBE_KEY ? { Authorization: `Bearer ${d.env.STC_TRANSCRIBE_KEY}` } : {},
});
if (!res.ok) throw new Error(`HTTP ${res.status} ${(await res.text()).slice(0, 200)}`);
const body = (await res.json()) as { text?: string; segments?: Segment[] };
const segments = (body.segments ?? []).map((s) => ({ start: Number(s.start) || 0, end: Number(s.end) || 0, text: String(s.text ?? "") }));
const rel = `transcripts/${l.id}.md`;
await mkdir(join(course.dir, "transcripts"), { recursive: true });
await writeFile(join(course.dir, rel), segmentsToMarkdown(l.title, segments, body.text ?? ""));
await mkdir(transcriptCacheDir(d, course.id), { recursive: true });
await writeFile(join(transcriptCacheDir(d, course.id), `${l.id}.json`), JSON.stringify({ lessonId: l.id, text: body.text ?? "", segments }));
l.transcript = rel;
done++;
} catch (e) {
warnings.push(`${l.title}: ${e instanceof Error ? e.message : String(e)}`);
}
}
} finally {
if (done > 0) await saveManifest(course.dir, course.manifest); // keep what finished even if the job was cancelled
}
if (done > 0) await finish(d, course.id, warnings);
await jc.reportProgress(1, `Transcribed ${done} of ${lessons.length} lessons`);
return { courseId: course.id, transcribed: done, warnings };
});
}
packages/feature-import/src/jobs/video-folder.ts
+64
View File
@@ -0,0 +1,64 @@
import { stat } from "node:fs/promises";
import { basename, extname, join } from "node:path";
import type { ImportResult, JobContext } from "@stc/contracts";
import { finish, frameCandidates, freshCourseId, publishCourse, saveManifest, sourceBlock, stageDir, type Deps } from "../context";
import { probeDuration, putFile } from "../media";
import { assertAllowedPath } from "../roots";
import { naturalSort } from "../util/text";
import { openSource } from "../util/source";
import { adoptCover, maybeQueueTranscription, optBool, optStr, parsePayload, withStage } from "./common";
const VIDEO = /\.(mp4|m4v|webm|mov|mkv|avi)$/i;
const BROWSER_OK = /\.(mp4|m4v|webm|mov)$/i;
export function lessonTitle(file: string): string {
const base = basename(file, extname(file));
const t = base.replace(/^\s*\d+\s*[-_.)\s]+(?=\S)/, "").replace(/_/g, " ").replace(/\s+/g, " ").trim();
return t || base;
}
export async function importVideoFolder(d: Deps, payload: unknown, jc: JobContext): Promise<ImportResult> {
const { path, options } = parsePayload(payload, "video-folder");
const warnings: string[] = [];
await assertAllowedPath(d, path);
if (!(await stat(path).catch(() => null))?.isDirectory()) throw new Error(`not a folder: ${path}`);
const src = await openSource(path);
const files = naturalSort(src.entries.filter((e) => VIDEO.test(e) && !e.split("/").some((p) => p.startsWith("."))));
if (files.length === 0) throw new Error(`no video files found in ${path}`);
const title = optStr(options, "title") ?? basename(path).replace(/[_]+/g, " ");
const id = await freshCourseId(d, optStr(options, "courseId"), title);
const mode = options.media === "link" ? "link" : "copy";
const stage = await stageDir(d, "videos");
return withStage(stage, async () => {
const lessons: any[] = [];
const placed: string[] = [];
const used = new Set<string>();
for (const [i, f] of files.entries()) {
await jc.reportProgress(0.05 + 0.7 * (i / files.length), `Copying ${basename(f)}`);
let rel = `media/${f}`;
if (used.has(rel.toLowerCase())) rel = `media/${i + 1}-${basename(f)}`;
used.add(rel.toLowerCase());
await putFile(join(path, f), join(stage, rel), mode);
placed.push(join(stage, rel));
if (!BROWSER_OK.test(f)) warnings.push(`${f} may not play in browsers (${extname(f)}); consider converting it to MP4`);
const duration = await probeDuration(d.tools, join(stage, rel));
lessons.push({ id: `l${i + 1}`, title: lessonTitle(f), type: "video", file: rel, ...(duration ? { duration } : {}) });
}
await jc.reportProgress(0.8, "Making cover candidates");
const covers = await frameCandidates(d, id, placed);
if (covers.length === 0) warnings.push("could not extract cover frames (is ffmpeg available?)");
const cover = await adoptCover(d, id, stage, covers[0]);
await saveManifest(stage, {
schema: 1, id, title, visibility: "restricted", ...(cover ? { cover } : {}),
source: sourceBlock("video-folder", path),
modules: [{ id: "m1", title: "Videos", lessons }],
});
await jc.reportProgress(0.92, "Publishing the course");
const dir = await publishCourse(d, stage, id);
await finish(d, id, warnings);
await maybeQueueTranscription(d, id, optBool(options, "transcribe"), warnings);
await jc.reportProgress(1, `Imported ${files.length} videos`);
return { courseId: id, manifestPath: join(dir, "stcourse.yaml"), warnings };
});
}
packages/feature-import/src/media.ts
+40
View File
@@ -0,0 +1,40 @@
import { link, mkdir, copyFile } from "node:fs/promises";
import { dirname } from "node:path";
import { run } from "./util/exec";
export interface Tools { ffmpeg: string; ffprobe: string }
export async function probeDuration(t: Tools, file: string): Promise<number | undefined> {
try {
const out = await run([t.ffprobe, "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", file]);
const s = Number(out.trim());
return Number.isFinite(s) && s > 0 ? Math.round(s) : undefined;
} catch {
return undefined;
}
}
/** One JPEG frame at `fraction` of the clip, 640px wide. */
export async function extractFrame(t: Tools, file: string, out: string, fraction = 0.2): Promise<void> {
await mkdir(dirname(out), { recursive: true });
const dur = (await probeDuration(t, file)) ?? 0;
const at = Math.max(0, dur * fraction);
await run([t.ffmpeg, "-v", "error", "-y", "-ss", String(at), "-i", file, "-frames:v", "1", "-vf", "scale=640:-2", "-q:v", "4", out]);
}
/**
* Puts a large file into the course. "copy" uses `cp --reflink=auto` (a CoW clone where the filesystem supports it,
* a plain copy elsewhere); "link" makes a hard link and falls back to copying across filesystems.
* Either way the original is only read, and the course folder stays self-contained (the media route refuses symlinks that leave it).
*/
export async function putFile(src: string, dest: string, mode: "copy" | "link" = "copy"): Promise<void> {
await mkdir(dirname(dest), { recursive: true });
if (mode === "link") {
try { await link(src, dest); return; } catch { /* cross-device or unsupported: copy */ }
}
try {
await run(["cp", "--reflink=auto", "--", src, dest]);
} catch {
await copyFile(src, dest);
}
}
packages/feature-import/src/packt.ts
+74
View File
@@ -0,0 +1,74 @@
import { decodeEntities, stripTags } from "./util/text";
export interface PacktClip { number: number; title: string; video: string; }
export interface PacktSection { number: number; title: string; clips: PacktClip[] }
export interface PacktIndex { title: string; sections: PacktSection[]; warnings: string[] }
/**
* Parses the Packt player's index.html structurally:
* - the section overview: `<h1>Section N: …</h1>` followed by a `<ul>` of clip titles, in order, usually WITHOUT "Video N.M:" prefixes;
* - the per-section chapter blocks (`class="chapter" id="content-N"`) with one `<li>` per clip: `<h1>N.M title</h1>` and `data-video="…"`.
* Chapter blocks are authoritative for the video file; the overview list fills in any title or whole section that is missing.
*/
export function parsePacktIndex(html: string): PacktIndex {
const warnings: string[] = [];
const intro = /<div[^>]*class="intro"[\s\S]*?<h1[^>]*>([\s\S]*?)<\/h1>/i.exec(html);
const titleTag = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(html);
const title = stripTags(intro?.[1] ?? titleTag?.[1] ?? "") || "Imported course";
const chapterStart = html.search(/<div[^>]*class="chapter"/i);
const overview = chapterStart >= 0 ? html.slice(0, chapterStart) : html;
const chaptersHtml = chapterStart >= 0 ? html.slice(chapterStart) : "";
// overview: sections in document order
const secRe = /<h1[^>]*>\s*Section\s+(\d+)\s*[:.-]\s*([\s\S]*?)<\/h1>/gi;
const heads = [...overview.matchAll(secRe)];
if (heads.length === 0) throw new Error("not a Packt video package: no \"Section N: …\" headings found in index.html");
const overviewTitles = new Map<number, string[]>();
const sectionTitles = new Map<number, string>();
heads.forEach((h, i) => {
const n = Number(h[1]);
sectionTitles.set(n, stripTags(h[2]!));
const block = overview.slice(h.index! + h[0].length, heads[i + 1]?.index ?? overview.length);
const ul = /<ul[^>]*>([\s\S]*?)<\/ul>/i.exec(block);
const items = ul ? [...ul[1]!.matchAll(/<li[^>]*>([\s\S]*?)<\/li>/gi)].map((m) => stripTags(m[1]!)).filter(Boolean) : [];
const firstP = /<p[^>]*>\s*Video\s+\d+\.\d+\s*:\s*([\s\S]*?)<\/p>/i.exec(block);
if (items.length === 0 && firstP) items.push(stripTags(firstP[1]!));
overviewTitles.set(n, items);
});
// chapters: clips with their files
const chapterClips = new Map<number, PacktClip[]>();
const chapRe = /<div[^>]*class="chapter"[^>]*id="content-(\d+)"[^>]*>/gi;
const chaps = [...chaptersHtml.matchAll(chapRe)];
chaps.forEach((c, i) => {
const sec = Number(c[1]);
const block = chaptersHtml.slice(c.index! + c[0].length, chaps[i + 1]?.index ?? chaptersHtml.length);
const clipRe = /<h1[^>]*>\s*(?:Video\s+)?(\d+)\.(\d+)\s*[:.]?\s*([\s\S]*?)<\/h1>/gi;
const hs = [...block.matchAll(clipRe)];
const clips: PacktClip[] = [];
hs.forEach((h, j) => {
const part = block.slice(h.index!, hs[j + 1]?.index ?? block.length);
const vid = /data-video="([^"]+)"/i.exec(part)?.[1] ?? /href="([^"]+\.(?:mp4|webm|m4v|mov))"/i.exec(part)?.[1];
const num = Number(h[2]);
if (!vid) { warnings.push(`clip ${sec}.${num} has no video file in index.html`); return; }
clips.push({ number: num, title: stripTags(h[3]!), video: decodeEntities(vid) });
});
chapterClips.set(sec, clips);
});
const sections: PacktSection[] = [];
for (const n of [...sectionTitles.keys()].sort((a, b) => a - b)) {
let clips = chapterClips.get(n);
const titles = overviewTitles.get(n) ?? [];
if (!clips || clips.length === 0) {
warnings.push(`section ${n} has no chapter block; clips were taken from the overview list and mapped to videos/video${n}_M.mp4 in order`);
clips = titles.map((t, i) => ({ number: i + 1, title: t, video: `videos/video${n}_${i + 1}.mp4` }));
} else {
clips = clips.map((c, i) => ({ ...c, title: c.title || titles[i] || `Video ${n}.${c.number}` }));
if (titles.length && titles.length !== clips.length) warnings.push(`section ${n}: overview lists ${titles.length} clips but the chapter has ${clips.length}`);
}
sections.push({ number: n, title: sectionTitles.get(n)!, clips });
}
return { title, sections, warnings };
}
packages/feature-import/src/pdf.ts
+23
View File
@@ -0,0 +1,23 @@
import { join } from "node:path";
import { run } from "./util/exec";
export interface DeckRender { pages: number; title: string }
const TOOL = join(import.meta.dir, "..", "py", "pdf_tool.py");
/**
* Renders page images (p-001.jpg ...) and text.json into `outDir` with PyMuPDF in a subprocess,
* so the server process never holds a PDF engine. `python` is STC_PYTHON (default python3).
*/
export async function renderPdf(python: string, pdf: string, outDir: string, opts: { onPage?: (page: number, total: number) => void; signal?: AbortSignal } = {}): Promise<DeckRender> {
let last = "";
await run([python, "-I", TOOL, "render", pdf, outDir], {
signal: opts.signal,
onLine(l) {
const m = /^page (\d+)\/(\d+)$/.exec(l);
if (m) opts.onPage?.(Number(m[1]), Number(m[2]));
else if (l.trim()) last = l;
},
});
return JSON.parse(last) as DeckRender;
}

Some files were not shown because too many files have changed in this diff Show More