diff --git a/packages/feature-ai/src/assistant/loop.ts b/packages/feature-ai/src/assistant/loop.ts new file mode 100644 index 0000000..0a3a0e0 --- /dev/null +++ b/packages/feature-ai/src/assistant/loop.ts @@ -0,0 +1,52 @@ +import type { AiProvider, ChatMessage } from "@stc/contracts"; +import type { ToolSet } from "./tools"; + +export type AssistantEvent = + | { type: "text"; text: string } + | { type: "tool"; name: string } + | { type: "error"; message: string } + | { type: "done"; text: string }; + +/** + * Runs the model with tools until it answers without calling a tool, or until maxSteps model calls. + * Mutates `messages` by appending the tool round trips, so callers can inspect them. + */ +export async function* runAssistant(o: { + provider: AiProvider; + messages: ChatMessage[]; + tools: ToolSet; + signal?: AbortSignal; + maxSteps?: number; +}): AsyncIterable { + const maxSteps = o.maxSteps ?? 6; + let answer = ""; + for (let step = 0; step < maxSteps; step++) { + let text = ""; + const calls: { id: string; name: string; input: unknown }[] = []; + try { + for await (const ev of o.provider.chat(o.messages, { tools: o.tools.specs, signal: o.signal })) { + if (ev.text) { text += ev.text; yield { type: "text", text: ev.text }; } + if (ev.toolCall) calls.push(ev.toolCall); + } + } catch (e) { + yield { type: "error", message: (e as Error).message }; + return; + } + answer += text; + if (!calls.length) { + yield { type: "done", text: answer }; + return; + } + if (text) o.messages.push({ role: "assistant", content: text }); + for (const c of calls) o.messages.push({ role: "assistant", content: JSON.stringify(c.input ?? {}), toolCallId: c.id, name: c.name }); + for (const c of calls) { + yield { type: "tool", name: c.name }; + let result: string; + try { result = await o.tools.run(c.name, c.input); } catch (e) { result = `Tool error: ${(e as Error).message}`; } + o.messages.push({ role: "tool", content: result, toolCallId: c.id, name: c.name }); + } + } + const note = "\n\n(I stopped after several lookups. Ask again to continue.)"; + yield { type: "text", text: note }; + yield { type: "done", text: answer + note }; +} diff --git a/packages/feature-ai/src/assistant/prompt.ts b/packages/feature-ai/src/assistant/prompt.ts new file mode 100644 index 0000000..c70e0e7 --- /dev/null +++ b/packages/feature-ai/src/assistant/prompt.ts @@ -0,0 +1,72 @@ +import type { CourseRepository, LoadedCourse } from "@stc/course-format"; +import type { Chunk } from "../retrieval/course-index"; + +export interface Citation { path: string; kind: Chunk["kind"]; lessonId?: string; lessonTitle?: string } + +const MAX_MENTOR = 40_000; + +/** The manifest's `mentor` file (for example Pure Snake's MENTOR.md), or null when the course has none. */ +export async function readMentorText(repo: CourseRepository, course: LoadedCourse): Promise { + const rel = course.manifest?.mentor; + if (!rel) return null; + const abs = repo.resolvePath(course.id, rel); + if (!abs) return null; + try { return (await Bun.file(abs).text()).slice(0, MAX_MENTOR); } catch { return null; } +} + +const label = (c: Chunk) => [c.path, c.lessonTitle && `${c.lessonTitle} (${c.lessonId})`, c.heading].filter(Boolean).join(" · "); + +export function buildSystemPrompt(o: { + course: LoadedCourse; + mentorText: string | null; + mentorOn: boolean; + sources: Chunk[]; + lesson?: { id: string; title: string } | null; + extraTools?: string[]; +}): string { + const title = o.course.manifest?.title ?? o.course.id; + const parts: string[] = []; + parts.push( + `You are the course assistant for "${title}" on STCourses, a self-hosted course platform. You help one learner with this course: you answer questions, explain ideas, and quiz them when they ask.`, + [ + "Ground your answers in the course material. Use the retrieved sources below and your tools (search_course, read_lesson, read_file, list_notes) to look things up rather than guessing.", + "When you rely on a source, cite its file path and lesson in square brackets, for example [lessons/04-exercise.md · Adding numbers]. If the course does not cover something, say so plainly.", + "Reference solutions are for authors only and are not available to you. Do not try to reconstruct them.", + "Keep answers short and plain. Use Markdown for code.", + ].join("\n"), + ); + if (o.extraTools?.length) parts.push(`The administrator also connected these external tools: ${o.extraTools.join(", ")}. Use them only when the course material is not enough.`); + if (o.lesson) parts.push(`The learner is currently on the lesson "${o.lesson.title}" (id ${o.lesson.id}).`); + if (o.mentorText && o.mentorOn) { + parts.push( + `## Course mentor rules\n\nThis course ships the following rules for an AI tutor. Follow them for the whole conversation; they take priority over being maximally helpful. Where they mention files, tools or a workspace you cannot access, apply their spirit through this chat.\n\n\n${o.mentorText.trim()}\n`, + ); + } else if (o.mentorText && !o.mentorOn) { + parts.push("## Mentor mode is off\n\nThe learner switched off this course's mentor rules for themselves. Give full, direct answers, including complete worked solutions when they ask for them."); + } + if (o.sources.length) { + parts.push( + `## Retrieved sources\n\nThese excerpts matched the learner's question. They may be incomplete; use the tools to read more.\n\n${o.sources + .map((c, i) => `[${i + 1}] ${label(c)}\n${c.text}`) + .join("\n\n---\n\n")}`, + ); + } + return parts.join("\n\n"); +} + +/** Course files and lessons behind an answer, deduplicated. Notes are the learner's own and are not cited as files. */ +export function citationsFor(chunks: Chunk[]): Citation[] { + const seen = new Set(); + const out: Citation[] = []; + for (const c of chunks) { + if (c.kind === "note") continue; + const key = `${c.path}\u0000${c.lessonId ?? ""}`; + if (seen.has(key)) continue; + seen.add(key); + const cit: Citation = { path: c.path, kind: c.kind }; + if (c.lessonId) cit.lessonId = c.lessonId; + if (c.lessonTitle) cit.lessonTitle = c.lessonTitle; + out.push(cit); + } + return out; +} diff --git a/packages/feature-ai/src/assistant/tools.ts b/packages/feature-ai/src/assistant/tools.ts new file mode 100644 index 0000000..142cc61 --- /dev/null +++ b/packages/feature-ai/src/assistant/tools.ts @@ -0,0 +1,157 @@ +import { stat } from "node:fs/promises"; +import type { ToolSpec } from "@stc/contracts"; +import type { CourseRepository, LoadedCourse } from "@stc/course-format"; +import type { McpServerConfig } from "../settings"; +import { connectMcp, mcpToolName, type McpClient } from "../mcp/client"; +import { isSolutionPath, noteChunks, type Chunk, type CourseIndex } from "../retrieval/course-index"; + +export interface ToolSet { + specs: ToolSpec[]; + /** Runs a tool and returns text for the model. Never throws for expected failures. */ + run(name: string, input: unknown): Promise; + /** Course chunks the tools surfaced (for citations). */ + sources: Chunk[]; + close?(): Promise; +} + +type NoteLike = { id: string; lessonId: string; body: string; anchorKind: string; anchorValue: string | null }; +const MAX_READ = 60_000; +const str = (v: unknown) => (typeof v === "string" ? v : ""); +const obj = (v: unknown): Record => (v && typeof v === "object" ? (v as Record) : {}); + +export function courseTools(o: { + repo: CourseRepository; + course: LoadedCourse; + index: CourseIndex; + listNotes: (lessonId?: string) => Promise; +}): ToolSet { + const sources: Chunk[] = []; + const { course } = o; + const lessonChunk = (lessonId: string): Chunk | null => { + const e = course.lessonIndex.get(lessonId); + if (!e) return null; + const path = e.lesson.type === "video" ? (e.lesson.transcript ?? e.lesson.file ?? "") : (e.lesson.file ?? ""); + return { path, kind: e.lesson.type === "video" ? "transcript" : "lesson", text: "", lessonId, lessonTitle: e.lesson.title }; + }; + const specs: ToolSpec[] = [ + { + name: "search_course", + description: "Search this course's lessons, transcripts, slide text and code files. Returns the best matching excerpts with their file paths and lessons.", + inputSchema: { type: "object", properties: { query: { type: "string", description: "Words to search for" } }, required: ["query"], additionalProperties: false }, + }, + { + name: "read_lesson", + description: "Read the full Markdown of one lesson (or a video lesson's transcript) by lesson id.", + inputSchema: { type: "object", properties: { lessonId: { type: "string" } }, required: ["lessonId"], additionalProperties: false }, + }, + { + name: "read_file", + description: "Read a text or code file from the course by its course-relative path, for example a companion code file.", + inputSchema: { type: "object", properties: { path: { type: "string" } }, required: ["path"], additionalProperties: false }, + }, + { + name: "list_notes", + description: "List the learner's own notes in this course, optionally for one lesson.", + inputSchema: { type: "object", properties: { lessonId: { type: "string" } }, additionalProperties: false }, + }, + ]; + const lessonList = () => + (course.manifest?.modules ?? []).map((m) => `${m.title}: ${m.lessons.map((l) => `${l.id} (${l.title})`).join(", ")}`).join("\n"); + + return { + specs, + sources, + async run(name, input) { + const a = obj(input); + switch (name) { + case "search_course": { + const q = str(a.query); + if (!q.trim()) return "Give a non-empty query."; + const hits = o.index.search(q, 6); + sources.push(...hits); + if (!hits.length) return "No matches in this course."; + return hits.map((c, i) => `[${i + 1}] ${[c.path, c.lessonTitle && `${c.lessonTitle} (${c.lessonId})`, c.heading].filter(Boolean).join(" · ")}\n${c.text}`).join("\n\n---\n\n"); + } + case "read_lesson": { + const id = str(a.lessonId); + const ref = lessonChunk(id); + if (!ref) return `No lesson with id "${id}". Lessons:\n${lessonList()}`; + if (isSolutionPath(course, ref.path)) return "That file is for authors only."; + const md = await o.repo.readLessonMarkdown(course.id, id); + if (md === null) return `Lesson "${id}" has no readable text (it may be a video without a transcript).`; + sources.push(ref); + return md.slice(0, MAX_READ); + } + case "read_file": { + const rel = str(a.path).replace(/^\/+/, ""); + if (isSolutionPath(course, rel)) return "That file is for authors only and is not available."; + const abs = o.repo.resolvePath(course.id, rel); + const info = abs ? await stat(abs).catch(() => null) : null; + if (!abs || !info?.isFile()) return `No such file in this course: ${rel}`; + if (info.size > 2_000_000) return "That file is too large to read here."; + const text = await Bun.file(abs).text(); + if (text.includes("\u0000")) return "That is a binary file."; + sources.push({ path: rel, kind: /\.(md|markdown)$/i.test(rel) ? "markdown" : "code", text: "" }); + return text.slice(0, MAX_READ); + } + case "list_notes": { + const lessonId = str(a.lessonId) || undefined; + const notes = await o.listNotes(lessonId); + if (!notes.length) return "The learner has no notes here yet."; + return noteChunks(course, notes).map((c) => `- ${[c.lessonTitle ?? c.lessonId, c.heading].filter(Boolean).join(", ")}: ${c.text}`).join("\n"); + } + default: + return `Unknown tool "${name}".`; + } + }, + }; +} + +/** Tools from admin-configured MCP servers. Servers that fail to connect are skipped and reported. */ +export async function mcpTools(servers: McpServerConfig[], opts: { fetch?: typeof fetch; timeoutMs?: number } = {}): Promise { + const clients: McpClient[] = []; + const routes = new Map(); + const specs: ToolSpec[] = []; + const warnings: string[] = []; + await Promise.all( + servers.map(async (s) => { + try { + const client = await connectMcp(s, opts); + clients.push(client); + for (const t of await client.tools()) { + const name = mcpToolName(s.id, t.name); + routes.set(name, { client, tool: t.name }); + specs.push({ name, description: `[${s.name}] ${t.description}`.slice(0, 1024), inputSchema: t.inputSchema }); + } + } catch (e) { + warnings.push(`MCP server "${s.name}" is unavailable: ${(e as Error).message}`); + } + }), + ); + return { + specs, + sources: [], + warnings, + names: specs.map((s) => s.name), + async run(name, input) { + const r = routes.get(name); + if (!r) return `Unknown tool "${name}".`; + try { return (await r.client.call(r.tool, input)).slice(0, MAX_READ); } catch (e) { return `Tool error: ${(e as Error).message}`; } + }, + async close() { await Promise.all(clients.map((c) => c.close().catch(() => {}))); }, + }; +} + +export function combineTools(sets: ToolSet[]): ToolSet { + const owner = new Map(); + for (const s of sets) for (const spec of s.specs) if (!owner.has(spec.name)) owner.set(spec.name, s); + return { + specs: [...owner.keys()].map((n) => owner.get(n)!.specs.find((s) => s.name === n)!), + get sources() { return sets.flatMap((s) => s.sources); }, + async run(name, input) { + const s = owner.get(name); + return s ? s.run(name, input) : `Unknown tool "${name}".`; + }, + async close() { await Promise.all(sets.map((s) => s.close?.())); }, + }; +} diff --git a/packages/feature-ai/src/index.ts b/packages/feature-ai/src/index.ts index e7c28f0..1fa0c30 100644 --- a/packages/feature-ai/src/index.ts +++ b/packages/feature-ai/src/index.ts @@ -7,3 +7,6 @@ export * from "./providers"; export * from "./retrieval/course-index"; export { Bm25, tokenize } from "./retrieval/bm25"; export * from "./mcp/client"; +export * from "./assistant/prompt"; +export * from "./assistant/tools"; +export * from "./assistant/loop"; diff --git a/packages/feature-ai/test/assistant.test.ts b/packages/feature-ai/test/assistant.test.ts new file mode 100644 index 0000000..7b6b8f0 --- /dev/null +++ b/packages/feature-ai/test/assistant.test.ts @@ -0,0 +1,91 @@ +import { beforeAll, expect, test } from "bun:test"; +import { join } from "node:path"; +import type { ChatMessage } from "@stc/contracts"; +import { createCourseRepository, type CourseRepository, type LoadedCourse } from "@stc/course-format"; +import { buildSystemPrompt, citationsFor, readMentorText } from "../src/assistant/prompt"; +import { courseTools, combineTools } from "../src/assistant/tools"; +import { runAssistant, type AssistantEvent } from "../src/assistant/loop"; +import { buildCourseIndex, type CourseIndex } from "../src/retrieval/course-index"; +import { fakeProvider } from "./helpers"; + +const FIXTURES = join(import.meta.dir, "../../../fixtures/courses"); +let repo: CourseRepository; +let course: LoadedCourse; +let index: CourseIndex; +beforeAll(async () => { + repo = await createCourseRepository([FIXTURES]); + course = repo.get("sample")!; + index = await buildCourseIndex(course, "/nonexistent"); +}); + +test("mentor rules are in the system prompt when mentor mode is on", async () => { + const mentorText = await readMentorText(repo, course); + expect(mentorText).toContain("never full solutions"); + const on = buildSystemPrompt({ course, mentorText, mentorOn: true, sources: index.search("add", 2) }); + expect(on).toContain("## Course mentor rules"); + expect(on).toContain("Give hints, never full solutions."); + expect(on).toContain("[1] lessons/04-exercise.md"); + const off = buildSystemPrompt({ course, mentorText, mentorOn: false, sources: [] }); + expect(off).not.toContain("never full solutions"); + expect(off).toContain("Mentor mode is off"); +}); + +test("citations are deduplicated by path and lesson", () => { + const c = citationsFor([ + { path: "a.md", kind: "lesson", text: "x", lessonId: "l1", lessonTitle: "L1" }, + { path: "a.md", kind: "lesson", text: "y", lessonId: "l1", lessonTitle: "L1" }, + { path: "notes", kind: "note", text: "z", lessonId: "l1" }, + { path: "b.js", kind: "code", text: "z" }, + ]); + expect(c).toEqual([ + { path: "a.md", kind: "lesson", lessonId: "l1", lessonTitle: "L1" }, + { path: "b.js", kind: "code" }, + ]); +}); + +test("course tools search, read lessons and files, list notes, and refuse solutions", async () => { + const t = courseTools({ repo, course, index, listNotes: async () => [{ id: "n", lessonId: "m1-intro", body: "my note", anchorKind: "none", anchorValue: null }] }); + expect(t.specs.map((s) => s.name).sort()).toEqual(["list_notes", "read_file", "read_lesson", "search_course"]); + expect(await t.run("search_course", { query: "sum of its arguments" })).toContain("lessons/04-exercise.md"); + expect(t.sources.some((s) => s.path === "lessons/04-exercise.md")).toBe(true); + expect(await t.run("read_lesson", { lessonId: "m2-ex-add" })).toContain("Implement `add(a, b)`"); + expect(await t.run("read_lesson", { lessonId: "nope" })).toMatch(/No lesson/); + expect(await t.run("read_file", { path: "exercises/add/starter.js" })).toContain("add"); + expect(await t.run("read_file", { path: "exercises/add/solution.js" })).toMatch(/authors only/); + expect(await t.run("read_file", { path: "../../../etc/passwd" })).toMatch(/No such file/); + expect(await t.run("list_notes", {})).toContain("my note"); +}); + +test("the loop runs tools and feeds results back until the model answers", async () => { + const t = courseTools({ repo, course, index, listNotes: async () => [] }); + const { provider, seen } = fakeProvider([ + () => ({ text: "Searching.", toolCalls: [{ id: "c1", name: "search_course", input: { query: "add arguments" } }] }), + (msgs) => ({ text: `Found it in lessons/04-exercise.md. (${msgs.filter((m) => m.role === "tool").length} tool result)` }), + ]); + const messages: ChatMessage[] = [{ role: "system", content: "sys" }, { role: "user", content: "How do I add?" }]; + const events: AssistantEvent[] = []; + for await (const e of runAssistant({ provider, messages, tools: combineTools([t]) })) events.push(e); + expect(events.filter((e) => e.type === "tool")).toEqual([{ type: "tool", name: "search_course" }]); + const done = events.find((e) => e.type === "done") as { type: "done"; text: string }; + expect(done.text).toBe("Searching.Found it in lessons/04-exercise.md. (1 tool result)"); + const second = seen[1]!.messages; + expect(second.at(-2)).toMatchObject({ role: "assistant", toolCallId: "c1", name: "search_course" }); + expect(second.at(-1)!.role).toBe("tool"); + expect(second.at(-1)!.content).toContain("lessons/04-exercise.md"); + expect(seen[0]!.tools!.map((x) => x.name)).toContain("search_course"); +}); + +test("unknown tools and provider errors are reported, and the step limit holds", async () => { + const t = courseTools({ repo, course, index, listNotes: async () => [] }); + const looping = fakeProvider([() => ({ toolCalls: [{ id: crypto.randomUUID(), name: "nope", input: {} }] })]); + const events: AssistantEvent[] = []; + for await (const e of runAssistant({ provider: looping.provider, messages: [{ role: "user", content: "x" }], tools: combineTools([t]), maxSteps: 3 })) events.push(e); + expect(looping.seen).toHaveLength(3); + expect(looping.seen[1]!.messages.at(-1)!.content).toMatch(/Unknown tool/); + expect(events.at(-1)!.type).toBe("done"); + + const broken = { id: "b", async *chat() { throw new Error("boom"); } }; + const ev2: AssistantEvent[] = []; + for await (const e of runAssistant({ provider: broken, messages: [{ role: "user", content: "x" }], tools: combineTools([t]) })) ev2.push(e); + expect(ev2).toEqual([{ type: "error", message: "boom" }]); +});