feat(ai): assistant prompt, course tools and tool loop

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

packages/feature-ai/src/assistant/loop.ts

@@ -0,0 +1,52 @@
+import type { AiProvider, ChatMessage } from "@stc/contracts";
+import type { ToolSet } from "./tools";
+
+export type AssistantEvent =
+  | { type: "text"; text: string }
+  | { type: "tool"; name: string }
+  | { type: "error"; message: string }
+  | { type: "done"; text: string };
+
+/**
+ * Runs the model with tools until it answers without calling a tool, or until maxSteps model calls.
+ * Mutates `messages` by appending the tool round trips, so callers can inspect them.
+ */
+export async function* runAssistant(o: {
+  provider: AiProvider;
+  messages: ChatMessage[];
+  tools: ToolSet;
+  signal?: AbortSignal;
+  maxSteps?: number;
+}): AsyncIterable<AssistantEvent> {
+  const maxSteps = o.maxSteps ?? 6;
+  let answer = "";
+  for (let step = 0; step < maxSteps; step++) {
+    let text = "";
+    const calls: { id: string; name: string; input: unknown }[] = [];
+    try {
+      for await (const ev of o.provider.chat(o.messages, { tools: o.tools.specs, signal: o.signal })) {
+        if (ev.text) { text += ev.text; yield { type: "text", text: ev.text }; }
+        if (ev.toolCall) calls.push(ev.toolCall);
+      }
+    } catch (e) {
+      yield { type: "error", message: (e as Error).message };
+      return;
+    }
+    answer += text;
+    if (!calls.length) {
+      yield { type: "done", text: answer };
+      return;
+    }
+    if (text) o.messages.push({ role: "assistant", content: text });
+    for (const c of calls) o.messages.push({ role: "assistant", content: JSON.stringify(c.input ?? {}), toolCallId: c.id, name: c.name });
+    for (const c of calls) {
+      yield { type: "tool", name: c.name };
+      let result: string;
+      try { result = await o.tools.run(c.name, c.input); } catch (e) { result = `Tool error: ${(e as Error).message}`; }
+      o.messages.push({ role: "tool", content: result, toolCallId: c.id, name: c.name });
+    }
+  }
+  const note = "\n\n(I stopped after several lookups. Ask again to continue.)";
+  yield { type: "text", text: note };
+  yield { type: "done", text: answer + note };
+}

packages/feature-ai/src/assistant/prompt.ts

@@ -0,0 +1,72 @@
+import type { CourseRepository, LoadedCourse } from "@stc/course-format";
+import type { Chunk } from "../retrieval/course-index";
+
+export interface Citation { path: string; kind: Chunk["kind"]; lessonId?: string; lessonTitle?: string }
+
+const MAX_MENTOR = 40_000;
+
+/** The manifest's `mentor` file (for example Pure Snake's MENTOR.md), or null when the course has none. */
+export async function readMentorText(repo: CourseRepository, course: LoadedCourse): Promise<string | null> {
+  const rel = course.manifest?.mentor;
+  if (!rel) return null;
+  const abs = repo.resolvePath(course.id, rel);
+  if (!abs) return null;
+  try { return (await Bun.file(abs).text()).slice(0, MAX_MENTOR); } catch { return null; }
+}
+
+const label = (c: Chunk) => [c.path, c.lessonTitle && `${c.lessonTitle} (${c.lessonId})`, c.heading].filter(Boolean).join(" · ");
+
+export function buildSystemPrompt(o: {
+  course: LoadedCourse;
+  mentorText: string | null;
+  mentorOn: boolean;
+  sources: Chunk[];
+  lesson?: { id: string; title: string } | null;
+  extraTools?: string[];
+}): string {
+  const title = o.course.manifest?.title ?? o.course.id;
+  const parts: string[] = [];
+  parts.push(
+    `You are the course assistant for "${title}" on STCourses, a self-hosted course platform. You help one learner with this course: you answer questions, explain ideas, and quiz them when they ask.`,
+    [
+      "Ground your answers in the course material. Use the retrieved sources below and your tools (search_course, read_lesson, read_file, list_notes) to look things up rather than guessing.",
+      "When you rely on a source, cite its file path and lesson in square brackets, for example [lessons/04-exercise.md · Adding numbers]. If the course does not cover something, say so plainly.",
+      "Reference solutions are for authors only and are not available to you. Do not try to reconstruct them.",
+      "Keep answers short and plain. Use Markdown for code.",
+    ].join("\n"),
+  );
+  if (o.extraTools?.length) parts.push(`The administrator also connected these external tools: ${o.extraTools.join(", ")}. Use them only when the course material is not enough.`);
+  if (o.lesson) parts.push(`The learner is currently on the lesson "${o.lesson.title}" (id ${o.lesson.id}).`);
+  if (o.mentorText && o.mentorOn) {
+    parts.push(
+      `## Course mentor rules\n\nThis course ships the following rules for an AI tutor. Follow them for the whole conversation; they take priority over being maximally helpful. Where they mention files, tools or a workspace you cannot access, apply their spirit through this chat.\n\n<mentor_rules>\n${o.mentorText.trim()}\n</mentor_rules>`,
+    );
+  } else if (o.mentorText && !o.mentorOn) {
+    parts.push("## Mentor mode is off\n\nThe learner switched off this course's mentor rules for themselves. Give full, direct answers, including complete worked solutions when they ask for them.");
+  }
+  if (o.sources.length) {
+    parts.push(
+      `## Retrieved sources\n\nThese excerpts matched the learner's question. They may be incomplete; use the tools to read more.\n\n${o.sources
+        .map((c, i) => `[${i + 1}] ${label(c)}\n${c.text}`)
+        .join("\n\n---\n\n")}`,
+    );
+  }
+  return parts.join("\n\n");
+}
+
+/** Course files and lessons behind an answer, deduplicated. Notes are the learner's own and are not cited as files. */
+export function citationsFor(chunks: Chunk[]): Citation[] {
+  const seen = new Set<string>();
+  const out: Citation[] = [];
+  for (const c of chunks) {
+    if (c.kind === "note") continue;
+    const key = `${c.path}\u0000${c.lessonId ?? ""}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    const cit: Citation = { path: c.path, kind: c.kind };
+    if (c.lessonId) cit.lessonId = c.lessonId;
+    if (c.lessonTitle) cit.lessonTitle = c.lessonTitle;
+    out.push(cit);
+  }
+  return out;
+}

packages/feature-ai/src/assistant/tools.ts

@@ -0,0 +1,157 @@
+import { stat } from "node:fs/promises";
+import type { ToolSpec } from "@stc/contracts";
+import type { CourseRepository, LoadedCourse } from "@stc/course-format";
+import type { McpServerConfig } from "../settings";
+import { connectMcp, mcpToolName, type McpClient } from "../mcp/client";
+import { isSolutionPath, noteChunks, type Chunk, type CourseIndex } from "../retrieval/course-index";
+
+export interface ToolSet {
+  specs: ToolSpec[];
+  /** Runs a tool and returns text for the model. Never throws for expected failures. */
+  run(name: string, input: unknown): Promise<string>;
+  /** Course chunks the tools surfaced (for citations). */
+  sources: Chunk[];
+  close?(): Promise<void>;
+}
+
+type NoteLike = { id: string; lessonId: string; body: string; anchorKind: string; anchorValue: string | null };
+const MAX_READ = 60_000;
+const str = (v: unknown) => (typeof v === "string" ? v : "");
+const obj = (v: unknown): Record<string, unknown> => (v && typeof v === "object" ? (v as Record<string, unknown>) : {});
+
+export function courseTools(o: {
+  repo: CourseRepository;
+  course: LoadedCourse;
+  index: CourseIndex;
+  listNotes: (lessonId?: string) => Promise<NoteLike[]>;
+}): ToolSet {
+  const sources: Chunk[] = [];
+  const { course } = o;
+  const lessonChunk = (lessonId: string): Chunk | null => {
+    const e = course.lessonIndex.get(lessonId);
+    if (!e) return null;
+    const path = e.lesson.type === "video" ? (e.lesson.transcript ?? e.lesson.file ?? "") : (e.lesson.file ?? "");
+    return { path, kind: e.lesson.type === "video" ? "transcript" : "lesson", text: "", lessonId, lessonTitle: e.lesson.title };
+  };
+  const specs: ToolSpec[] = [
+    {
+      name: "search_course",
+      description: "Search this course's lessons, transcripts, slide text and code files. Returns the best matching excerpts with their file paths and lessons.",
+      inputSchema: { type: "object", properties: { query: { type: "string", description: "Words to search for" } }, required: ["query"], additionalProperties: false },
+    },
+    {
+      name: "read_lesson",
+      description: "Read the full Markdown of one lesson (or a video lesson's transcript) by lesson id.",
+      inputSchema: { type: "object", properties: { lessonId: { type: "string" } }, required: ["lessonId"], additionalProperties: false },
+    },
+    {
+      name: "read_file",
+      description: "Read a text or code file from the course by its course-relative path, for example a companion code file.",
+      inputSchema: { type: "object", properties: { path: { type: "string" } }, required: ["path"], additionalProperties: false },
+    },
+    {
+      name: "list_notes",
+      description: "List the learner's own notes in this course, optionally for one lesson.",
+      inputSchema: { type: "object", properties: { lessonId: { type: "string" } }, additionalProperties: false },
+    },
+  ];
+  const lessonList = () =>
+    (course.manifest?.modules ?? []).map((m) => `${m.title}: ${m.lessons.map((l) => `${l.id} (${l.title})`).join(", ")}`).join("\n");
+
+  return {
+    specs,
+    sources,
+    async run(name, input) {
+      const a = obj(input);
+      switch (name) {
+        case "search_course": {
+          const q = str(a.query);
+          if (!q.trim()) return "Give a non-empty query.";
+          const hits = o.index.search(q, 6);
+          sources.push(...hits);
+          if (!hits.length) return "No matches in this course.";
+          return hits.map((c, i) => `[${i + 1}] ${[c.path, c.lessonTitle && `${c.lessonTitle} (${c.lessonId})`, c.heading].filter(Boolean).join(" · ")}\n${c.text}`).join("\n\n---\n\n");
+        }
+        case "read_lesson": {
+          const id = str(a.lessonId);
+          const ref = lessonChunk(id);
+          if (!ref) return `No lesson with id "${id}". Lessons:\n${lessonList()}`;
+          if (isSolutionPath(course, ref.path)) return "That file is for authors only.";
+          const md = await o.repo.readLessonMarkdown(course.id, id);
+          if (md === null) return `Lesson "${id}" has no readable text (it may be a video without a transcript).`;
+          sources.push(ref);
+          return md.slice(0, MAX_READ);
+        }
+        case "read_file": {
+          const rel = str(a.path).replace(/^\/+/, "");
+          if (isSolutionPath(course, rel)) return "That file is for authors only and is not available.";
+          const abs = o.repo.resolvePath(course.id, rel);
+          const info = abs ? await stat(abs).catch(() => null) : null;
+          if (!abs || !info?.isFile()) return `No such file in this course: ${rel}`;
+          if (info.size > 2_000_000) return "That file is too large to read here.";
+          const text = await Bun.file(abs).text();
+          if (text.includes("\u0000")) return "That is a binary file.";
+          sources.push({ path: rel, kind: /\.(md|markdown)$/i.test(rel) ? "markdown" : "code", text: "" });
+          return text.slice(0, MAX_READ);
+        }
+        case "list_notes": {
+          const lessonId = str(a.lessonId) || undefined;
+          const notes = await o.listNotes(lessonId);
+          if (!notes.length) return "The learner has no notes here yet.";
+          return noteChunks(course, notes).map((c) => `- ${[c.lessonTitle ?? c.lessonId, c.heading].filter(Boolean).join(", ")}: ${c.text}`).join("\n");
+        }
+        default:
+          return `Unknown tool "${name}".`;
+      }
+    },
+  };
+}
+
+/** Tools from admin-configured MCP servers. Servers that fail to connect are skipped and reported. */
+export async function mcpTools(servers: McpServerConfig[], opts: { fetch?: typeof fetch; timeoutMs?: number } = {}): Promise<ToolSet & { warnings: string[]; names: string[] }> {
+  const clients: McpClient[] = [];
+  const routes = new Map<string, { client: McpClient; tool: string }>();
+  const specs: ToolSpec[] = [];
+  const warnings: string[] = [];
+  await Promise.all(
+    servers.map(async (s) => {
+      try {
+        const client = await connectMcp(s, opts);
+        clients.push(client);
+        for (const t of await client.tools()) {
+          const name = mcpToolName(s.id, t.name);
+          routes.set(name, { client, tool: t.name });
+          specs.push({ name, description: `[${s.name}] ${t.description}`.slice(0, 1024), inputSchema: t.inputSchema });
+        }
+      } catch (e) {
+        warnings.push(`MCP server "${s.name}" is unavailable: ${(e as Error).message}`);
+      }
+    }),
+  );
+  return {
+    specs,
+    sources: [],
+    warnings,
+    names: specs.map((s) => s.name),
+    async run(name, input) {
+      const r = routes.get(name);
+      if (!r) return `Unknown tool "${name}".`;
+      try { return (await r.client.call(r.tool, input)).slice(0, MAX_READ); } catch (e) { return `Tool error: ${(e as Error).message}`; }
+    },
+    async close() { await Promise.all(clients.map((c) => c.close().catch(() => {}))); },
+  };
+}
+
+export function combineTools(sets: ToolSet[]): ToolSet {
+  const owner = new Map<string, ToolSet>();
+  for (const s of sets) for (const spec of s.specs) if (!owner.has(spec.name)) owner.set(spec.name, s);
+  return {
+    specs: [...owner.keys()].map((n) => owner.get(n)!.specs.find((s) => s.name === n)!),
+    get sources() { return sets.flatMap((s) => s.sources); },
+    async run(name, input) {
+      const s = owner.get(name);
+      return s ? s.run(name, input) : `Unknown tool "${name}".`;
+    },
+    async close() { await Promise.all(sets.map((s) => s.close?.())); },
+  };
+}

packages/feature-ai/src/index.ts

@@ -7,3 +7,6 @@ export * from "./providers";
 export * from "./retrieval/course-index";
 export { Bm25, tokenize } from "./retrieval/bm25";
 export * from "./mcp/client";
+export * from "./assistant/prompt";
+export * from "./assistant/tools";
+export * from "./assistant/loop";

packages/feature-ai/test/assistant.test.ts

@@ -0,0 +1,91 @@
+import { beforeAll, expect, test } from "bun:test";
+import { join } from "node:path";
+import type { ChatMessage } from "@stc/contracts";
+import { createCourseRepository, type CourseRepository, type LoadedCourse } from "@stc/course-format";
+import { buildSystemPrompt, citationsFor, readMentorText } from "../src/assistant/prompt";
+import { courseTools, combineTools } from "../src/assistant/tools";
+import { runAssistant, type AssistantEvent } from "../src/assistant/loop";
+import { buildCourseIndex, type CourseIndex } from "../src/retrieval/course-index";
+import { fakeProvider } from "./helpers";
+
+const FIXTURES = join(import.meta.dir, "../../../fixtures/courses");
+let repo: CourseRepository;
+let course: LoadedCourse;
+let index: CourseIndex;
+beforeAll(async () => {
+  repo = await createCourseRepository([FIXTURES]);
+  course = repo.get("sample")!;
+  index = await buildCourseIndex(course, "/nonexistent");
+});
+
+test("mentor rules are in the system prompt when mentor mode is on", async () => {
+  const mentorText = await readMentorText(repo, course);
+  expect(mentorText).toContain("never full solutions");
+  const on = buildSystemPrompt({ course, mentorText, mentorOn: true, sources: index.search("add", 2) });
+  expect(on).toContain("## Course mentor rules");
+  expect(on).toContain("Give hints, never full solutions.");
+  expect(on).toContain("[1] lessons/04-exercise.md");
+  const off = buildSystemPrompt({ course, mentorText, mentorOn: false, sources: [] });
+  expect(off).not.toContain("never full solutions");
+  expect(off).toContain("Mentor mode is off");
+});
+
+test("citations are deduplicated by path and lesson", () => {
+  const c = citationsFor([
+    { path: "a.md", kind: "lesson", text: "x", lessonId: "l1", lessonTitle: "L1" },
+    { path: "a.md", kind: "lesson", text: "y", lessonId: "l1", lessonTitle: "L1" },
+    { path: "notes", kind: "note", text: "z", lessonId: "l1" },
+    { path: "b.js", kind: "code", text: "z" },
+  ]);
+  expect(c).toEqual([
+    { path: "a.md", kind: "lesson", lessonId: "l1", lessonTitle: "L1" },
+    { path: "b.js", kind: "code" },
+  ]);
+});
+
+test("course tools search, read lessons and files, list notes, and refuse solutions", async () => {
+  const t = courseTools({ repo, course, index, listNotes: async () => [{ id: "n", lessonId: "m1-intro", body: "my note", anchorKind: "none", anchorValue: null }] });
+  expect(t.specs.map((s) => s.name).sort()).toEqual(["list_notes", "read_file", "read_lesson", "search_course"]);
+  expect(await t.run("search_course", { query: "sum of its arguments" })).toContain("lessons/04-exercise.md");
+  expect(t.sources.some((s) => s.path === "lessons/04-exercise.md")).toBe(true);
+  expect(await t.run("read_lesson", { lessonId: "m2-ex-add" })).toContain("Implement `add(a, b)`");
+  expect(await t.run("read_lesson", { lessonId: "nope" })).toMatch(/No lesson/);
+  expect(await t.run("read_file", { path: "exercises/add/starter.js" })).toContain("add");
+  expect(await t.run("read_file", { path: "exercises/add/solution.js" })).toMatch(/authors only/);
+  expect(await t.run("read_file", { path: "../../../etc/passwd" })).toMatch(/No such file/);
+  expect(await t.run("list_notes", {})).toContain("my note");
+});
+
+test("the loop runs tools and feeds results back until the model answers", async () => {
+  const t = courseTools({ repo, course, index, listNotes: async () => [] });
+  const { provider, seen } = fakeProvider([
+    () => ({ text: "Searching.", toolCalls: [{ id: "c1", name: "search_course", input: { query: "add arguments" } }] }),
+    (msgs) => ({ text: `Found it in lessons/04-exercise.md. (${msgs.filter((m) => m.role === "tool").length} tool result)` }),
+  ]);
+  const messages: ChatMessage[] = [{ role: "system", content: "sys" }, { role: "user", content: "How do I add?" }];
+  const events: AssistantEvent[] = [];
+  for await (const e of runAssistant({ provider, messages, tools: combineTools([t]) })) events.push(e);
+  expect(events.filter((e) => e.type === "tool")).toEqual([{ type: "tool", name: "search_course" }]);
+  const done = events.find((e) => e.type === "done") as { type: "done"; text: string };
+  expect(done.text).toBe("Searching.Found it in lessons/04-exercise.md. (1 tool result)");
+  const second = seen[1]!.messages;
+  expect(second.at(-2)).toMatchObject({ role: "assistant", toolCallId: "c1", name: "search_course" });
+  expect(second.at(-1)!.role).toBe("tool");
+  expect(second.at(-1)!.content).toContain("lessons/04-exercise.md");
+  expect(seen[0]!.tools!.map((x) => x.name)).toContain("search_course");
+});
+
+test("unknown tools and provider errors are reported, and the step limit holds", async () => {
+  const t = courseTools({ repo, course, index, listNotes: async () => [] });
+  const looping = fakeProvider([() => ({ toolCalls: [{ id: crypto.randomUUID(), name: "nope", input: {} }] })]);
+  const events: AssistantEvent[] = [];
+  for await (const e of runAssistant({ provider: looping.provider, messages: [{ role: "user", content: "x" }], tools: combineTools([t]), maxSteps: 3 })) events.push(e);
+  expect(looping.seen).toHaveLength(3);
+  expect(looping.seen[1]!.messages.at(-1)!.content).toMatch(/Unknown tool/);
+  expect(events.at(-1)!.type).toBe("done");
+
+  const broken = { id: "b", async *chat() { throw new Error("boom"); } };
+  const ev2: AssistantEvent[] = [];
+  for await (const e of runAssistant({ provider: broken, messages: [{ role: "user", content: "x" }], tools: combineTools([t]) })) ev2.push(e);
+  expect(ev2).toEqual([{ type: "error", message: "boom" }]);
+});