92 lines
5.4 KiB
TypeScript
92 lines
5.4 KiB
TypeScript
import { beforeAll, expect, test } from "bun:test";
|
|
import { join } from "node:path";
|
|
import type { ChatMessage } from "@stc/contracts";
|
|
import { createCourseRepository, type CourseRepository, type LoadedCourse } from "@stc/course-format";
|
|
import { buildSystemPrompt, citationsFor, readMentorText } from "../src/assistant/prompt";
|
|
import { courseTools, combineTools } from "../src/assistant/tools";
|
|
import { runAssistant, type AssistantEvent } from "../src/assistant/loop";
|
|
import { buildCourseIndex, type CourseIndex } from "../src/retrieval/course-index";
|
|
import { fakeProvider } from "./helpers";
|
|
|
|
const FIXTURES = join(import.meta.dir, "../../../fixtures/courses");
|
|
let repo: CourseRepository;
|
|
let course: LoadedCourse;
|
|
let index: CourseIndex;
|
|
beforeAll(async () => {
|
|
repo = await createCourseRepository([FIXTURES]);
|
|
course = repo.get("sample")!;
|
|
index = await buildCourseIndex(course, "/nonexistent");
|
|
});
|
|
|
|
test("mentor rules are in the system prompt when mentor mode is on", async () => {
|
|
const mentorText = await readMentorText(repo, course);
|
|
expect(mentorText).toContain("never full solutions");
|
|
const on = buildSystemPrompt({ course, mentorText, mentorOn: true, sources: index.search("add", 2) });
|
|
expect(on).toContain("## Course mentor rules");
|
|
expect(on).toContain("Give hints, never full solutions.");
|
|
expect(on).toContain("[1] lessons/04-exercise.md");
|
|
const off = buildSystemPrompt({ course, mentorText, mentorOn: false, sources: [] });
|
|
expect(off).not.toContain("never full solutions");
|
|
expect(off).toContain("Mentor mode is off");
|
|
});
|
|
|
|
test("citations are deduplicated by path and lesson", () => {
|
|
const c = citationsFor([
|
|
{ path: "a.md", kind: "lesson", text: "x", lessonId: "l1", lessonTitle: "L1" },
|
|
{ path: "a.md", kind: "lesson", text: "y", lessonId: "l1", lessonTitle: "L1" },
|
|
{ path: "notes", kind: "note", text: "z", lessonId: "l1" },
|
|
{ path: "b.js", kind: "code", text: "z" },
|
|
]);
|
|
expect(c).toEqual([
|
|
{ path: "a.md", kind: "lesson", lessonId: "l1", lessonTitle: "L1" },
|
|
{ path: "b.js", kind: "code" },
|
|
]);
|
|
});
|
|
|
|
test("course tools search, read lessons and files, list notes, and refuse solutions", async () => {
|
|
const t = courseTools({ repo, course, index, listNotes: async () => [{ id: "n", lessonId: "m1-intro", body: "my note", anchorKind: "none", anchorValue: null }] });
|
|
expect(t.specs.map((s) => s.name).sort()).toEqual(["list_notes", "read_file", "read_lesson", "search_course"]);
|
|
expect(await t.run("search_course", { query: "sum of its arguments" })).toContain("lessons/04-exercise.md");
|
|
expect(t.sources.some((s) => s.path === "lessons/04-exercise.md")).toBe(true);
|
|
expect(await t.run("read_lesson", { lessonId: "m2-ex-add" })).toContain("Implement `add(a, b)`");
|
|
expect(await t.run("read_lesson", { lessonId: "nope" })).toMatch(/No lesson/);
|
|
expect(await t.run("read_file", { path: "exercises/add/starter.js" })).toContain("add");
|
|
expect(await t.run("read_file", { path: "exercises/add/solution.js" })).toMatch(/authors only/);
|
|
expect(await t.run("read_file", { path: "../../../etc/passwd" })).toMatch(/No such file/);
|
|
expect(await t.run("list_notes", {})).toContain("my note");
|
|
});
|
|
|
|
test("the loop runs tools and feeds results back until the model answers", async () => {
|
|
const t = courseTools({ repo, course, index, listNotes: async () => [] });
|
|
const { provider, seen } = fakeProvider([
|
|
() => ({ text: "Searching.", toolCalls: [{ id: "c1", name: "search_course", input: { query: "add arguments" } }] }),
|
|
(msgs) => ({ text: `Found it in lessons/04-exercise.md. (${msgs.filter((m) => m.role === "tool").length} tool result)` }),
|
|
]);
|
|
const messages: ChatMessage[] = [{ role: "system", content: "sys" }, { role: "user", content: "How do I add?" }];
|
|
const events: AssistantEvent[] = [];
|
|
for await (const e of runAssistant({ provider, messages, tools: combineTools([t]) })) events.push(e);
|
|
expect(events.filter((e) => e.type === "tool")).toEqual([{ type: "tool", name: "search_course" }]);
|
|
const done = events.find((e) => e.type === "done") as { type: "done"; text: string };
|
|
expect(done.text).toBe("Searching.Found it in lessons/04-exercise.md. (1 tool result)");
|
|
const second = seen[1]!.messages;
|
|
expect(second.at(-2)).toMatchObject({ role: "assistant", toolCallId: "c1", name: "search_course" });
|
|
expect(second.at(-1)!.role).toBe("tool");
|
|
expect(second.at(-1)!.content).toContain("lessons/04-exercise.md");
|
|
expect(seen[0]!.tools!.map((x) => x.name)).toContain("search_course");
|
|
});
|
|
|
|
test("unknown tools and provider errors are reported, and the step limit holds", async () => {
|
|
const t = courseTools({ repo, course, index, listNotes: async () => [] });
|
|
const looping = fakeProvider([() => ({ toolCalls: [{ id: crypto.randomUUID(), name: "nope", input: {} }] })]);
|
|
const events: AssistantEvent[] = [];
|
|
for await (const e of runAssistant({ provider: looping.provider, messages: [{ role: "user", content: "x" }], tools: combineTools([t]), maxSteps: 3 })) events.push(e);
|
|
expect(looping.seen).toHaveLength(3);
|
|
expect(looping.seen[1]!.messages.at(-1)!.content).toMatch(/Unknown tool/);
|
|
expect(events.at(-1)!.type).toBe("done");
|
|
|
|
const broken = { id: "b", async *chat() { throw new Error("boom"); } };
|
|
const ev2: AssistantEvent[] = [];
|
|
for await (const e of runAssistant({ provider: broken, messages: [{ role: "user", content: "x" }], tools: combineTools([t]) })) ev2.push(e);
|
|
expect(ev2).toEqual([{ type: "error", message: "boom" }]);
|
|
});
|