From a57222ecc33e318e75d417d6ba49c376c694be42 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 03:25:10 +0100 Subject: [PATCH] feat(ai): HTTP routes, chat streaming and generation job Co-Authored-By: Claude Opus 5.5 --- packages/feature-ai/src/feature.ts | 425 ++++++++++++++++++ packages/feature-ai/src/index.ts | 1 + .../feature-ai/src/retrieval/course-index.ts | 7 +- packages/feature-ai/test/feature.test.ts | 156 +++++++ packages/feature-ai/test/retrieval.test.ts | 5 + 5 files changed, 592 insertions(+), 2 deletions(-) create mode 100644 packages/feature-ai/src/feature.ts create mode 100644 packages/feature-ai/test/feature.test.ts diff --git a/packages/feature-ai/src/feature.ts b/packages/feature-ai/src/feature.ts new file mode 100644 index 0000000..02f59e2 --- /dev/null +++ b/packages/feature-ai/src/feature.ts @@ -0,0 +1,425 @@ +import { readdir } from "node:fs/promises"; +import { extname, join, relative, sep } from "node:path"; +import type { AiProvider, AuthUser, ChatMessage, Feature, JobContext, RequestContext, ServerContext } from "@stc/contracts"; +import { canView } from "@stc/core"; +import type { LoadedCourse } from "@stc/course-format"; +import { aclRepo, notesRepo } from "@stc/db"; +import { buildSystemPrompt, citationsFor, readMentorText } from "./assistant/prompt"; +import { combineTools, courseTools, mcpTools, type ToolSet } from "./assistant/tools"; +import { runAssistant } from "./assistant/loop"; +import { createSecretBox } from "./crypto"; +import { createFsDraftSink, type DraftSink } from "./generator/draft-sink"; +import { generateDraft, type GenerateRequest, type GenerateSource } from "./generator/generate"; +import { connectMcp, mcpServersFor } from "./mcp/client"; +import { createProvider } from "./providers"; +import { getCourseIndex, noteChunks } from "./retrieval/course-index"; +import { aiSettingsRepo, type McpServerConfig, type ProviderConfig } from "./settings"; + +export interface AiFeatureOptions { + /** Used by providers and HTTP MCP servers. Tests pass a fake; production uses global fetch. */ + fetch?: typeof fetch; + /** Where generated drafts go. Phase 2 passes the studio's createDraftWriter(ctx); default writes under /drafts. */ + draftSink?: DraftSink; + /** Overrides provider construction (tests use a fake AiProvider). */ + providerFactory?: (cfg: ProviderConfig, apiKey: string | null) => AiProvider; + mcpTimeoutMs?: number; +} + +const json = (body: unknown, status = 200) => Response.json(body, { status }); +const err = (status: number, message: string) => json({ error: message }, status); +const KINDS = new Set(["anthropic", "openai"]); +const EFFORTS = new Set(["low", "medium", "high", "xhigh", "max"]); +const HISTORY_TURNS = 20; +const MAX_MESSAGE = 8_000; + +async function body(req: Request): Promise | null> { + try { + const b = await req.json(); + return b && typeof b === "object" && !Array.isArray(b) ? (b as Record) : null; + } catch { return null; } +} +const str = (v: unknown): string | undefined => (typeof v === "string" && v.trim() ? v.trim() : undefined); + +function parseProvider(v: unknown): (ProviderConfig & { apiKey?: string; clearApiKey?: boolean }) | null | "invalid" { + if (v === null || v === undefined) return null; + if (typeof v !== "object") return "invalid"; + const p = v as Record; + if (!KINDS.has(p.kind as string)) return "invalid"; + if (p.effort !== undefined && p.effort !== "" && !EFFORTS.has(p.effort as string)) return "invalid"; + const out: ProviderConfig & { apiKey?: string; clearApiKey?: boolean } = { kind: p.kind as ProviderConfig["kind"] }; + if (str(p.model)) out.model = str(p.model); + if (str(p.baseUrl)) { + try { new URL(str(p.baseUrl)!); } catch { return "invalid"; } + out.baseUrl = str(p.baseUrl); + } + if (str(p.effort)) out.effort = str(p.effort) as ProviderConfig["effort"]; + if (str(p.apiKey)) out.apiKey = str(p.apiKey); + if (p.clearApiKey === true) out.clearApiKey = true; + if (out.kind === "openai" && (!out.baseUrl || !out.model)) return "invalid"; + return out; +} + +function parseMcp(v: unknown): McpServerConfig[] | "invalid" { + if (v === undefined || v === null) return []; + if (!Array.isArray(v)) return "invalid"; + const out: McpServerConfig[] = []; + const ids = new Set(); + for (const raw of v) { + if (!raw || typeof raw !== "object") return "invalid"; + const s = raw as Record; + const id = str(s.id); + if (!id || !/^[a-zA-Z0-9_-]{1,40}$/.test(id) || ids.has(id)) return "invalid"; + ids.add(id); + const transport = s.transport === "stdio" || s.transport === "http" ? s.transport : null; + if (!transport) return "invalid"; + const cfg: McpServerConfig = { id, name: str(s.name) ?? id, transport }; + if (transport === "stdio") { + if (!str(s.command)) return "invalid"; + cfg.command = str(s.command); + if (Array.isArray(s.args)) cfg.args = s.args.map(String); + if (s.env && typeof s.env === "object") cfg.env = Object.fromEntries(Object.entries(s.env).map(([k, val]) => [k, String(val)])); + } else { + const url = str(s.url); + if (!url) return "invalid"; + try { new URL(url); } catch { return "invalid"; } + cfg.url = url; + if (s.headers && typeof s.headers === "object") cfg.headers = Object.fromEntries(Object.entries(s.headers).map(([k, val]) => [k, String(val)])); + } + if (Array.isArray(s.courseIds)) cfg.courseIds = s.courseIds.map(String).filter(Boolean); + out.push(cfg); + } + return out; +} + +function parseGenerate(b: Record): GenerateRequest | null { + const courseId = str(b.courseId); + const topic = str(b.topic); + if (!courseId || !topic) return null; + if (!["document", "exercise", "questions"].includes(b.kind as string)) return null; + if (!["pure-snake", "systemic-design"].includes(b.style as string)) return null; + if (!Array.isArray(b.sources)) return null; + const sources: GenerateSource[] = []; + for (const s of b.sources) { + if (!s || typeof s !== "object") return null; + const x = s as Record; + if (x.kind === "lesson" && str(x.lessonId)) sources.push({ kind: "lesson", lessonId: str(x.lessonId)! }); + else if (x.kind === "file" && str(x.path)) sources.push({ kind: "file", path: str(x.path)! }); + else if (x.kind === "cache" && str(x.path)) sources.push({ kind: "cache", path: str(x.path)! }); + else if (x.kind === "text" && typeof x.text === "string" && x.text.trim()) sources.push({ kind: "text", label: str(x.label) ?? "Pasted text", text: x.text }); + else return null; + } + return { courseId, topic: topic.slice(0, 200), kind: b.kind as GenerateRequest["kind"], style: b.style as GenerateRequest["style"], sources, instructions: typeof b.instructions === "string" ? b.instructions.slice(0, 8_000) : undefined }; +} + +const LISTABLE = new Set([".md", ".markdown", ".txt", ".js", ".mjs", ".ts", ".tsx", ".jsx", ".py", ".rs", ".scala", ".fs", ".go", ".java", ".sh", ".json", ".yaml", ".yml", ".sql", ".html", ".css", ".toml", ".vtt", ".srt"]); +async function listFiles(root: string, limit = 800, keep: (rel: string) => boolean = () => true): Promise { + const out: string[] = []; + const walk = async (dir: string, depth: number) => { + if (depth > 8 || out.length >= limit) return; + let entries; + try { entries = await readdir(dir, { withFileTypes: true }); } catch { return; } + entries.sort((a, b) => a.name.localeCompare(b.name)); + for (const e of entries) { + if (out.length >= limit) return; + if (e.name.startsWith(".") || e.name === "node_modules" || e.name === "target" || e.name === "dist") continue; + const abs = join(dir, e.name); + if (e.isDirectory()) await walk(abs, depth + 1); + else if (e.isFile() && LISTABLE.has(extname(e.name).toLowerCase())) { + const rel = relative(root, abs).split(sep).join("/"); + if (keep(rel)) out.push(rel); + } + } + }; + await walk(root, 0); + return out; +} + +export function aiFeature(ctx: ServerContext, opts: AiFeatureOptions = {}): Feature { + const settings = aiSettingsRepo(ctx.db, createSecretBox(ctx.config.secret)); + const acl = aclRepo(ctx.db); + const notes = notesRepo(ctx.db); + const sink = opts.draftSink ?? createFsDraftSink(ctx.config.dataDir); + const makeProvider = opts.providerFactory ?? ((cfg: ProviderConfig, key: string | null) => createProvider(cfg, key, opts.fetch)); + + /** BYOK first ("used only for their own requests"), then the server provider. */ + const resolveProvider = async (userId: string | null): Promise<{ source: "byok" | "server"; cfg: ProviderConfig; key: string | null } | null> => { + if (userId) { + const u = await settings.getUser(userId); + if (u.byok) { + const key = await settings.userApiKey(userId).catch(() => null); + if (key) return { source: "byok", cfg: u.byok, key }; + } + } + const s = await settings.getServer(); + if (!s.provider) return null; + if (s.provider.kind === "anthropic" && !s.provider.hasApiKey) return null; + const key = s.provider.hasApiKey ? await settings.serverApiKey().catch(() => null) : null; + if (s.provider.kind === "anthropic" && !key) return null; + return { source: "server", cfg: s.provider, key }; + }; + + const viewable = async (user: AuthUser | null, courseId: string): Promise => { + const c = ctx.courses.get(courseId); + if (!c || c.status !== "ok") return null; + const ok = canView(user, c, await acl.listForCourse(c.id), user ? await acl.isAuthor(c.id, user.id) : false); + return ok ? c : null; + }; + const canAuthor = async (user: AuthUser, courseId: string) => + user.roles.includes("owner") || user.roles.includes("author") || (await acl.isAuthor(courseId, user.id)); + + const conversationOf = async (id: string, userId: string) => { + const rows = await ctx.db.sql`SELECT id, user_id, course_id, title, created_at FROM ai_conversations WHERE id = ${id} AND user_id = ${userId}`; + return rows[0] as { id: string; user_id: string; course_id: string; title: string | null; created_at: string } | undefined; + }; + const addMessage = async (conversationId: string, role: "user" | "assistant", content: string) => { + await ctx.db.sql`INSERT INTO ai_messages (id, conversation_id, role, content, created_at) VALUES (${Bun.randomUUIDv7()}, ${conversationId}, ${role}, ${content}, ${new Date().toISOString()})`; + }; + const messagesOf = async (conversationId: string) => + (await ctx.db.sql`SELECT id, role, content, created_at FROM ai_messages WHERE conversation_id = ${conversationId}`) + .map((r: { id: string; role: string; content: string; created_at: string }) => ({ id: r.id, role: r.role, content: r.content, createdAt: r.created_at })) + .sort((a: { id: string }, b: { id: string }) => (a.id < b.id ? -1 : 1)) as { id: string; role: "user" | "assistant"; content: string; createdAt: string }[]; + + const signedIn = (rc: RequestContext) => (rc.user ? null : err(401, "Sign in to use the assistant.")); + const ownerOnly = (rc: RequestContext) => (!rc.user ? err(401, "Sign in first.") : rc.user.roles.includes("owner") ? null : err(403, "Only the owner can change AI settings.")); + + const chat = async (req: Request, rc: RequestContext): Promise => { + const deny = signedIn(rc); + if (deny) return deny; + const user = rc.user!; + const course = await viewable(user, rc.params.courseId ?? ""); + if (!course) return err(404, "Course not found."); + const b = await body(req); + const message = typeof b?.message === "string" ? b.message.trim() : ""; + if (!message) return err(400, "Write a message."); + if (message.length > MAX_MESSAGE) return err(400, "That message is too long."); + const resolved = await resolveProvider(user.id); + if (!resolved) return err(409, "AI is not set up. An admin can configure a server provider, or you can add your own key under Account, AI."); + let provider: AiProvider; + try { provider = makeProvider(resolved.cfg, resolved.key); } catch (e) { return err(409, (e as Error).message); } + + let conversationId = str(b?.conversationId); + if (conversationId) { + const conv = await conversationOf(conversationId, user.id); + if (!conv || conv.course_id !== course.id) return err(404, "Conversation not found."); + } else { + conversationId = Bun.randomUUIDv7(); + await ctx.db.sql`INSERT INTO ai_conversations (id, user_id, course_id, title, created_at) VALUES (${conversationId}, ${user.id}, ${course.id}, ${message.slice(0, 80)}, ${new Date().toISOString()})`; + } + const history = (await messagesOf(conversationId)).slice(-HISTORY_TURNS); + await addMessage(conversationId, "user", message); + + const lessonId = str(b?.lessonId); + const lesson = lessonId ? course.lessonIndex.get(lessonId)?.lesson : undefined; + const index = await getCourseIndex(course, ctx.config.dataDir); + const userNotes = await notes.list(user.id, course.id); + const sources = index.search(lesson ? `${message} ${lesson.title}` : message, 6, noteChunks(course, userNotes)); + const mentorText = await readMentorText(ctx.courses, course); + const mentorOn = !(await settings.isMentorOff(user.id, course.id)); + const convId = conversationId; + + const stream = new ReadableStream({ + async start(controller) { + const enc = new TextEncoder(); + const send = (ev: unknown) => { try { controller.enqueue(enc.encode(JSON.stringify(ev) + "\n")); } catch {} }; + send({ type: "meta", conversationId: convId, mentorOn, mentorAvailable: !!mentorText, source: resolved.source, citations: citationsFor(sources) }); + let tools: ToolSet | null = null; + try { + const course_ = courseTools({ repo: ctx.courses, course, index, listNotes: (lid) => notes.list(user.id, course.id, lid) }); + const servers = mcpServersFor(course.id, (await settings.getServer()).mcpServers); + const mcp = servers.length ? await mcpTools(servers, { fetch: opts.fetch, timeoutMs: opts.mcpTimeoutMs }) : null; + for (const w of mcp?.warnings ?? []) send({ type: "notice", message: w }); + tools = combineTools(mcp ? [course_, mcp] : [course_]); + const messages: ChatMessage[] = [ + { role: "system", content: buildSystemPrompt({ course, mentorText, mentorOn, sources, lesson: lesson ? { id: lesson.id, title: lesson.title } : null, extraTools: mcp?.names }) }, + ...history.map((m) => ({ role: m.role, content: m.content }) as ChatMessage), + { role: "user", content: message }, + ]; + let answer = ""; + for await (const ev of runAssistant({ provider, messages, tools, signal: req.signal })) { + if (ev.type === "done") { + answer = ev.text; + send({ type: "done", text: ev.text, citations: citationsFor([...sources, ...tools.sources]) }); + } else send(ev); + } + if (answer) await addMessage(convId, "assistant", answer); + } catch (e) { + send({ type: "error", message: (e as Error).message }); + } finally { + await tools?.close?.().catch(() => {}); + try { controller.close(); } catch {} + } + }, + }); + return new Response(stream, { headers: { "content-type": "application/x-ndjson; charset=utf-8", "cache-control": "no-store" } }); + }; + + const generateJob = async (payload: unknown, jc: JobContext) => { + const p = payload as { req: GenerateRequest; userId: string }; + const resolved = await resolveProvider(p.userId); + if (!resolved) throw new Error("AI is not set up for this user."); + const provider = makeProvider(resolved.cfg, resolved.key); + return generateDraft({ provider, repo: ctx.courses, req: p.req, sink, authorId: p.userId, dataDir: ctx.config.dataDir, signal: jc.signal, onProgress: (f, m) => jc.reportProgress(f, m) }); + }; + + return { + name: "ai", + jobs: { "ai.generate": generateJob }, + routes: { + "/api/ai/status": { + GET: async (_req, rc) => { + const resolved = rc.user ? await resolveProvider(rc.user.id) : null; + const out: Record = { enabled: !!resolved, source: resolved?.source ?? null, signedIn: !!rc.user }; + const courseId = rc.url.searchParams.get("courseId"); + if (courseId) { + const course = await viewable(rc.user, courseId); + if (course) { + out.course = { id: course.id, title: course.manifest?.title ?? course.id }; + out.mentor = { available: !!course.manifest?.mentor, on: rc.user ? !(await settings.isMentorOff(rc.user.id, course.id)) : true }; + } + } + return json(out); + }, + }, + "/api/ai/admin/config": { + GET: async (_req, rc) => ownerOnly(rc) ?? json(await settings.getServer()), + PUT: async (req, rc) => { + const deny = ownerOnly(rc); + if (deny) return deny; + const b = await body(req); + if (!b) return err(400, "Invalid JSON."); + const provider = parseProvider(b.provider); + const mcpServers = parseMcp(b.mcpServers); + if (provider === "invalid") return err(400, "Invalid provider. OpenAI-compatible endpoints need a base URL and a model."); + if (mcpServers === "invalid") return err(400, "Invalid MCP server list."); + await settings.saveServer({ provider, mcpServers }); + return json(await settings.getServer()); + }, + }, + "/api/ai/admin/mcp/test": { + POST: async (req, rc) => { + const deny = ownerOnly(rc); + if (deny) return deny; + const b = await body(req); + const parsed = parseMcp(b ? [b] : null); + if (parsed === "invalid" || !parsed.length) return err(400, "Invalid MCP server."); + try { + const c = await connectMcp(parsed[0]!, { fetch: opts.fetch, timeoutMs: opts.mcpTimeoutMs ?? 15_000 }); + try { return json({ ok: true, tools: (await c.tools()).map((t) => ({ name: t.name, description: t.description })) }); } finally { await c.close(); } + } catch (e) { + return json({ ok: false, error: (e as Error).message }); + } + }, + }, + "/api/ai/account": { + GET: async (_req, rc) => { + if (!rc.user) return err(401, "Sign in first."); + const u = await settings.getUser(rc.user.id); + const s = await settings.getServer(); + return json({ byok: u.byok, serverProvider: s.provider ? { kind: s.provider.kind, model: s.provider.model ?? null } : null }); + }, + }, + "/api/ai/account/key": { + PUT: async (req, rc) => { + if (!rc.user) return err(401, "Sign in first."); + const b = await body(req); + const p = b ? parseProvider(b) : "invalid"; + if (!p || p === "invalid" || !p.apiKey) return err(400, "Choose a provider and paste your key. OpenAI-compatible endpoints also need a base URL and a model."); + await settings.saveUserKey(rc.user.id, { ...p, apiKey: p.apiKey }); + return json((await settings.getUser(rc.user.id)).byok); + }, + DELETE: async (_req, rc) => { + if (!rc.user) return err(401, "Sign in first."); + await settings.removeUserKey(rc.user.id); + return json({ ok: true }); + }, + }, + "/api/ai/courses/:courseId/mentor": { + PUT: async (req, rc) => { + const deny = signedIn(rc); + if (deny) return deny; + const course = await viewable(rc.user, rc.params.courseId ?? ""); + if (!course) return err(404, "Course not found."); + const b = await body(req); + if (typeof b?.off !== "boolean") return err(400, "Send { off: true | false }."); + await settings.setMentorOff(rc.user!.id, course.id, b.off); + return json({ mentor: { available: !!course.manifest?.mentor, on: !b.off } }); + }, + }, + "/api/ai/courses/:courseId/chat": { POST: chat }, + "/api/ai/courses/:courseId/conversations": { + GET: async (_req, rc) => { + const deny = signedIn(rc); + if (deny) return deny; + const rows = await ctx.db.sql`SELECT id, title, created_at FROM ai_conversations WHERE user_id = ${rc.user!.id} AND course_id = ${rc.params.courseId ?? ""}`; + return json(rows.map((r: { id: string; title: string | null; created_at: string }) => ({ id: r.id, title: r.title, createdAt: r.created_at })).sort((a: { createdAt: string }, b: { createdAt: string }) => (a.createdAt < b.createdAt ? 1 : -1))); + }, + }, + "/api/ai/conversations/:id": { + GET: async (_req, rc) => { + const deny = signedIn(rc); + if (deny) return deny; + const conv = await conversationOf(rc.params.id ?? "", rc.user!.id); + if (!conv) return err(404, "Conversation not found."); + return json({ id: conv.id, courseId: conv.course_id, title: conv.title, messages: await messagesOf(conv.id) }); + }, + DELETE: async (_req, rc) => { + const deny = signedIn(rc); + if (deny) return deny; + const conv = await conversationOf(rc.params.id ?? "", rc.user!.id); + if (!conv) return err(404, "Conversation not found."); + await ctx.db.sql.begin(async (tx) => { + await tx`DELETE FROM ai_messages WHERE conversation_id = ${conv.id}`; + await tx`DELETE FROM ai_conversations WHERE id = ${conv.id}`; + }); + return json({ ok: true }); + }, + }, + "/api/ai/studio/courses": { + GET: async (_req, rc) => { + if (!rc.user) return err(401, "Sign in first."); + const out = []; + for (const c of ctx.courses.list()) { + if (c.status === "ok" && (await canAuthor(rc.user, c.id))) out.push({ id: c.id, title: c.manifest?.title ?? c.id, tracks: c.manifest?.tracks ?? [] }); + } + return json(out); + }, + }, + "/api/ai/studio/sources": { + GET: async (_req, rc) => { + if (!rc.user) return err(401, "Sign in first."); + const courseId = rc.url.searchParams.get("courseId") ?? ""; + const course = ctx.courses.get(courseId); + if (!course || course.status !== "ok") return err(404, "Course not found."); + if (!(await canAuthor(rc.user, course.id))) return err(403, "Only authors can generate drafts."); + const lessons = (course.manifest?.modules ?? []).flatMap((m) => m.lessons.map((l) => ({ id: l.id, title: l.title, type: l.type, module: m.title, hasText: l.type !== "video" || !!l.transcript }))); + const files = await listFiles(course.root, 800); // authors may use any course file, solutions included + const cache = await listFiles(join(ctx.config.dataDir, "cache", course.id), 400); + return json({ lessons, files, cache }); + }, + }, + "/api/ai/studio/generate": { + POST: async (req, rc) => { + if (!rc.user) return err(401, "Sign in first."); + const b = await body(req); + const g = b ? parseGenerate(b) : null; + if (!g) return err(400, "Choose a course, a kind (document, exercise or questions), a style and a topic."); + const course = ctx.courses.get(g.courseId); + if (!course || course.status !== "ok") return err(404, "Course not found."); + if (!(await canAuthor(rc.user, course.id))) return err(403, "Only authors can generate drafts."); + if (!(await resolveProvider(rc.user.id))) return err(409, "AI is not set up."); + const jobId = await ctx.jobs.enqueue("ai.generate", { req: g, userId: rc.user.id }); + return json({ jobId }, 202); + }, + }, + "/api/ai/studio/jobs/:id": { + GET: async (_req, rc) => { + if (!rc.user) return err(401, "Sign in first."); + const job = await ctx.jobs.get(rc.params.id ?? ""); + const owner = (job?.payload as { userId?: string } | null)?.userId; + if (!job || job.kind !== "ai.generate" || (owner !== rc.user.id && !rc.user.roles.includes("owner"))) return err(404, "Job not found."); + return json({ id: job.id, status: job.status, progress: job.progress, message: job.message, result: job.result, error: job.error }); + }, + }, + }, + }; +} diff --git a/packages/feature-ai/src/index.ts b/packages/feature-ai/src/index.ts index 75a46aa..62f5df7 100644 --- a/packages/feature-ai/src/index.ts +++ b/packages/feature-ai/src/index.ts @@ -13,3 +13,4 @@ export * from "./assistant/loop"; export * from "./generator/draft-sink"; export * from "./generator/generate"; export * from "./generator/templates"; +export { aiFeature, type AiFeatureOptions } from "./feature"; diff --git a/packages/feature-ai/src/retrieval/course-index.ts b/packages/feature-ai/src/retrieval/course-index.ts index 616c498..57af7fb 100644 --- a/packages/feature-ai/src/retrieval/course-index.ts +++ b/packages/feature-ai/src/retrieval/course-index.ts @@ -19,7 +19,9 @@ const MD = new Set([".md", ".markdown"]); const CODE = new Set([".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx", ".py", ".rs", ".scala", ".sc", ".fs", ".fsx", ".go", ".java", ".kt", ".rb", ".php", ".c", ".h", ".cpp", ".cs", ".sh", ".bash", ".ps1", ".sql", ".yaml", ".yml", ".toml", ".json", ".tf", ".html", ".css"]); const CODE_NAMES = new Set(["Dockerfile", "Makefile", "docker-compose.yml", "Containerfile"]); const CACHE_TEXT = new Set([".txt", ".md", ".vtt", ".srt"]); -const SKIP_FILES = new Set(["package-lock.json", "bun.lock", "yarn.lock", "pnpm-lock.yaml", "Cargo.lock"]); +// Lock files, the manifest, and instructions written for AI agents (the mentor file goes into the system +// prompt only while mentor mode is on, so it must not leak back in through retrieval). +const SKIP_FILES = new Set(["package-lock.json", "bun.lock", "yarn.lock", "pnpm-lock.yaml", "Cargo.lock", "stcourse.yaml", "AGENTS.md", "CLAUDE.md", "GEMINI.md"]); const MAX_FILE = 200_000; const MAX_FILES = 2_000; const MAX_CHUNK = 1_500; @@ -130,6 +132,7 @@ export async function buildCourseIndex(course: LoadedCourse, dataDir: string): P const bm = new Bm25(); const seen = new Set(); const refs = lessonRefs(course); + const mentor = course.manifest?.mentor ? posix(course.manifest.mentor).replace(/^\.\//, "") : null; const codeDirs = (course.manifest?.modules ?? []).flatMap((m) => (m.code ? [posix(m.code.dir).replace(/\/$/, "") + "/"] : [])); const add = (c: Chunk) => { const key = `${c.kind}\u0000${c.text}`; @@ -146,7 +149,7 @@ export async function buildCourseIndex(course: LoadedCourse, dataDir: string): P const name = basename(rel); const isMd = MD.has(ext); const isCode = !isMd && (CODE.has(ext) || CODE_NAMES.has(name) || codeDirs.some((d) => rel.startsWith(d))); - if ((!isMd && !isCode) || SKIP_FILES.has(name) || isSolutionPath(course, rel)) return true; + if ((!isMd && !isCode) || SKIP_FILES.has(name) || rel === mentor || isSolutionPath(course, rel)) return true; let size = 0; try { size = (await stat(abs)).size; } catch { return true; } if (size > MAX_FILE || size === 0) return true; diff --git a/packages/feature-ai/test/feature.test.ts b/packages/feature-ai/test/feature.test.ts new file mode 100644 index 0000000..3912dfc --- /dev/null +++ b/packages/feature-ai/test/feature.test.ts @@ -0,0 +1,156 @@ +import { expect } from "bun:test"; +import { cpSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import type { AuthUser, JobRecord, ServerContext } from "@stc/contracts"; +import { createApp, createJobQueue } from "@stc/core"; +import { createCourseRepository } from "@stc/course-format"; +import type { Db } from "@stc/db"; +import { forEachDb } from "../../db/test/helpers"; +import { aiFeature } from "../src/feature"; +import { fakeProvider } from "./helpers"; + +const SAMPLE = resolve(import.meta.dir, "../../../fixtures/courses/sample"); +const USERS: Record = { + owner: { id: "owner", email: "o@x", name: "Owner", roles: ["owner"], groups: [] }, + ann: { id: "ann", email: "a@x", name: "Ann", roles: ["learner"], groups: [] }, + bob: { id: "bob", email: "b@x", name: "Bob", roles: ["learner"], groups: [] }, + auth: { id: "auth", email: "w@x", name: "Writer", roles: ["author"], groups: [] }, +}; + +async function setup(db: Db, turns: Parameters[0] = [() => ({ text: "ok" })]) { + const dir = mkdtempSync(join(tmpdir(), "stc-ai-feat-")); + cpSync(SAMPLE, join(dir, "courses/sample"), { recursive: true }); + cpSync(SAMPLE, join(dir, "courses/secret"), { recursive: true }); + const m = readFileSync(join(SAMPLE, "stcourse.yaml"), "utf8"); + writeFileSync(join(dir, "courses/secret/stcourse.yaml"), m.replace("id: sample", "id: secret").replace("visibility: public", "visibility: restricted")); + const courses = await createCourseRepository([join(dir, "courses")]); + const fake = fakeProvider(turns); + const keys: (string | null)[] = []; + const ctx = { + db, courses, + config: { port: 0, databaseUrl: "", dataDir: join(dir, "data"), courseDirs: [], secret: "test-secret", publicUrl: "" }, + auth: { userFromRequest: async (req: Request) => USERS[req.headers.get("x-user") ?? ""] ?? null }, + } as unknown as ServerContext; + const feature = aiFeature(ctx, { providerFactory: (_cfg, key) => { keys.push(key); return fake.provider; } }); + const jobs = createJobQueue(db, feature.jobs!); + ctx.jobs = jobs; + const app = createApp(ctx, [feature]); + const call = async (method: string, path: string, user?: string, body?: unknown) => { + const headers: Record = {}; + if (user) headers["x-user"] = user; + if (body !== undefined) headers["content-type"] = "application/json"; + return app.fetch(new Request(`http://x${path}`, { method, headers, body: body === undefined ? undefined : JSON.stringify(body) })); + }; + return { ctx, dir, call, fake, keys, feature }; +} + +const ndjson = async (r: Response) => (await r.text()).trim().split("\n").map((l) => JSON.parse(l)); + +forEachDb("AI is off by default and admin config is owner-only and hides keys", async (db) => { + const { call } = await setup(db); + expect(await (await call("GET", "/api/ai/status?courseId=sample", "ann")).json()).toMatchObject({ enabled: false, source: null }); + expect((await call("POST", "/api/ai/courses/sample/chat", "ann", { message: "hi" })).status).toBe(409); + expect((await call("GET", "/api/ai/admin/config", "ann")).status).toBe(403); + expect((await call("GET", "/api/ai/admin/config")).status).toBe(401); + const put = await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "anthropic", apiKey: "sk-ant-SERVER" }, mcpServers: [] }); + expect(put.status).toBe(200); + const cfg = await (await call("GET", "/api/ai/admin/config", "owner")).text(); + expect(cfg).not.toContain("SERVER"); + expect(JSON.parse(cfg).provider).toEqual({ kind: "anthropic", hasApiKey: true }); + expect((await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "nope" }, mcpServers: [] })).status).toBe(400); + expect(await (await call("GET", "/api/ai/status?courseId=sample", "ann")).json()).toMatchObject({ enabled: true, source: "server", mentor: { available: true, on: true } }); +}); + +forEachDb("BYOK enables AI for that user only and is used before the server key", async (db) => { + const { call, keys } = await setup(db); + expect((await call("PUT", "/api/ai/account/key", "ann", { kind: "anthropic", apiKey: "sk-ann" })).status).toBe(200); + const acct = await (await call("GET", "/api/ai/account", "ann")).text(); + expect(acct).not.toContain("sk-ann"); + expect(JSON.parse(acct).byok).toEqual({ kind: "anthropic", hasApiKey: true }); + expect(await (await call("GET", "/api/ai/status?courseId=sample", "ann")).json()).toMatchObject({ enabled: true, source: "byok" }); + expect(await (await call("GET", "/api/ai/status?courseId=sample", "bob")).json()).toMatchObject({ enabled: false }); + await call("POST", "/api/ai/courses/sample/chat", "ann", { message: "hello" }).then((r) => r.text()); + expect(keys).toEqual(["sk-ann"]); + expect((await call("DELETE", "/api/ai/account/key", "ann")).status).toBe(200); + expect(await (await call("GET", "/api/ai/status?courseId=sample", "ann")).json()).toMatchObject({ enabled: false }); + expect((await call("PUT", "/api/ai/account/key", "ann", { kind: "anthropic" })).status).toBe(400); +}); + +forEachDb("chat cites course files, applies mentor rules, and stores the conversation", async (db) => { + const { call, fake } = await setup(db, [ + () => ({ toolCalls: [{ id: "t1", name: "read_lesson", input: { lessonId: "m2-ex-add" } }] }), + () => ({ text: "Which operator combines two numbers? See [lessons/04-exercise.md ยท Adding numbers]." }), + ]); + await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "anthropic", apiKey: "k" }, mcpServers: [] }); + const r = await call("POST", "/api/ai/courses/sample/chat", "ann", { message: "How do I return the sum of its arguments?" }); + expect(r.status).toBe(200); + expect(r.headers.get("content-type")).toContain("application/x-ndjson"); + const events = await ndjson(r); + expect(events[0]).toMatchObject({ type: "meta", mentorOn: true }); + expect(events[0].citations.map((c: { path: string }) => c.path)).toContain("lessons/04-exercise.md"); + expect(events.some((e) => e.type === "tool" && e.name === "read_lesson")).toBe(true); + const done = events.find((e) => e.type === "done"); + expect(done.text).toContain("Which operator"); + expect(done.citations.some((c: { path: string; lessonId?: string }) => c.path === "lessons/04-exercise.md")).toBe(true); + const system = fake.seen[0]!.messages[0]!; + expect(system.role).toBe("system"); + expect(system.content).toContain("Give hints, never full solutions."); + + const convs = await (await call("GET", "/api/ai/courses/sample/conversations", "ann")).json(); + expect(convs).toHaveLength(1); + const conv = await (await call("GET", `/api/ai/conversations/${convs[0].id}`, "ann")).json(); + expect(conv.messages.map((m: { role: string }) => m.role)).toEqual(["user", "assistant"]); + expect((await call("GET", `/api/ai/conversations/${convs[0].id}`, "bob")).status).toBe(404); + + // Learner switches mentor mode off for this course: rules leave the prompt. + expect((await call("PUT", "/api/ai/courses/sample/mentor", "ann", { off: true })).status).toBe(200); + expect(await (await call("GET", "/api/ai/status?courseId=sample", "ann")).json()).toMatchObject({ mentor: { available: true, on: false } }); + const r2 = await ndjson(await call("POST", "/api/ai/courses/sample/chat", "ann", { message: "Show me the full solution", conversationId: convs[0].id })); + expect(r2[0]).toMatchObject({ type: "meta", mentorOn: false, conversationId: convs[0].id }); + const sys2 = fake.seen.at(-1)!.messages[0]!.content; + expect(sys2).not.toContain("never full solutions"); + expect(sys2).toContain("Mentor mode is off"); + // history is replayed + expect(fake.seen.at(-1)!.messages.some((m) => m.role === "assistant" && m.content.includes("Which operator"))).toBe(true); + // Bob still has mentor mode on + expect(await (await call("GET", "/api/ai/status?courseId=sample", "bob")).json()).toMatchObject({ mentor: { on: true } }); + + expect((await call("DELETE", `/api/ai/conversations/${convs[0].id}`, "ann")).status).toBe(200); + expect(await (await call("GET", "/api/ai/courses/sample/conversations", "ann")).json()).toEqual([]); +}); + +forEachDb("course visibility and sign-in are enforced", async (db) => { + const { call } = await setup(db); + await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "anthropic", apiKey: "k" }, mcpServers: [] }); + expect((await call("POST", "/api/ai/courses/secret/chat", "ann", { message: "hi" })).status).toBe(404); + expect((await call("POST", "/api/ai/courses/sample/chat", undefined, { message: "hi" })).status).toBe(401); + expect((await call("POST", "/api/ai/courses/nope/chat", "ann", { message: "hi" })).status).toBe(404); + expect((await call("POST", "/api/ai/courses/secret/chat", "owner", { message: "hi" })).status).toBe(200); +}); + +forEachDb("generation is for authors and writes drafts, not course files", async (db) => { + const doc = "=== FILE: questions/adding.md ===\n## Check your understanding\n1. Predict: add(2, 3)?\n=== END FILE ==="; + const { call, ctx, dir } = await setup(db, [() => ({ text: doc })]); + await call("PUT", "/api/ai/admin/config", "owner", { provider: { kind: "anthropic", apiKey: "k" }, mcpServers: [] }); + const body = { courseId: "sample", kind: "questions", style: "pure-snake", topic: "Adding", sources: [{ kind: "lesson", lessonId: "m2-text" }] }; + expect((await call("POST", "/api/ai/studio/generate", "ann", body)).status).toBe(403); + expect((await call("GET", "/api/ai/studio/sources?courseId=sample", "ann")).status).toBe(403); + const src = await (await call("GET", "/api/ai/studio/sources?courseId=sample", "auth")).json(); + expect(src.lessons.map((l: { id: string }) => l.id)).toContain("m2-text"); + expect(src.files).toContain("code/S01 - First steps/hello.sh"); + expect((await call("POST", "/api/ai/studio/generate", "auth", { ...body, kind: "poem" })).status).toBe(400); + const g = await call("POST", "/api/ai/studio/generate", "auth", body); + expect(g.status).toBe(202); + const { jobId } = await g.json(); + // run the queued job by hand (the worker is not started in tests) + const job = (await ctx.jobs.get(jobId)) as JobRecord; + expect(job.kind).toBe("ai.generate"); + const feature = aiFeature(ctx, { providerFactory: () => fakeProvider([() => ({ text: doc })]).provider }); + const result = (await feature.jobs!["ai.generate"]!(job.payload, { reportProgress: async () => {}, signal: new AbortController().signal })) as { location: string; paths: string[] }; + expect(result.paths).toEqual(["questions/adding.md"]); + expect(result.location.startsWith(join(dir, "data", "drafts", "sample"))).toBe(true); + expect(await Bun.file(join(dir, "courses/sample/questions/adding.md")).exists()).toBe(false); + expect((await call("GET", `/api/ai/studio/jobs/${jobId}`, "auth")).status).toBe(200); + expect((await call("GET", `/api/ai/studio/jobs/${jobId}`, "bob")).status).toBe(404); +}); diff --git a/packages/feature-ai/test/retrieval.test.ts b/packages/feature-ai/test/retrieval.test.ts index de15aef..96f51b9 100644 --- a/packages/feature-ai/test/retrieval.test.ts +++ b/packages/feature-ai/test/retrieval.test.ts @@ -69,6 +69,11 @@ test("solution files are never indexed", async () => { expect(isSolutionPath(course, "exercises/add/starter.js")).toBe(false); }); +test("the mentor file and the manifest are not retrievable", async () => { + const idx = await buildCourseIndex(course, dataDir); + expect(idx.search("mentor hints never full solutions", 50).some((c) => c.path === "MENTOR.md" || c.path === "stcourse.yaml")).toBe(false); +}); + test("notes are searched alongside the course", async () => { const idx = await getCourseIndex(course, dataDir); expect(await getCourseIndex(course, dataDir)).toBe(idx);