From 6fb0019cd92a22ae46bb9b6c77429339deb55b11 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 04:36:48 +0100 Subject: [PATCH] feat(ai): read the importer's deck and transcript cache; studio DraftWriter fits DraftSink as is Retrieval now reads /cache/decks///text.json and /cache/transcripts//.json (skipped when the lesson has a transcript file). DraftSink returns { location?, branch?, commit? }, so the studio's createDraftWriter(ctx) needs no adapter; generateDraft computes paths itself and reports the draft branch. Co-Authored-By: Claude Opus 5.5 --- .../feature-ai/src/generator/draft-sink.ts | 22 ++++++-- packages/feature-ai/src/generator/generate.ts | 13 ++++- .../feature-ai/src/retrieval/course-index.ts | 56 ++++++++++++++++++- packages/feature-ai/src/web/GeneratePage.tsx | 4 +- packages/feature-ai/test/generator.test.ts | 24 +++++++- packages/feature-ai/test/retrieval.test.ts | 49 +++++++++++++++- 6 files changed, 156 insertions(+), 12 deletions(-) diff --git a/packages/feature-ai/src/generator/draft-sink.ts b/packages/feature-ai/src/generator/draft-sink.ts index 720c811..794c75d 100644 --- a/packages/feature-ai/src/generator/draft-sink.ts +++ b/packages/feature-ai/src/generator/draft-sink.ts @@ -4,11 +4,25 @@ import { dirname, join, resolve, sep } from "node:path"; export interface DraftFile { path: string; content: string } /** - * Where generated files go. Same signature as the studio's `createDraftWriter(ctx).writeDraftFiles`, which - * commits onto the author's draft branch; Phase 2 passes that in. Drafts are never published by the AI package. + * What a sink reports back. The studio's writer returns `{ branch, commit }` (a commit on `draft/`); + * the filesystem fallback returns `{ location }` (a folder under `/drafts`). + */ +export interface DraftWriteResult { location?: string; branch?: string; commit?: string } + +/** + * Where generated files go. Same method as the studio's `DraftWriter` (`createDraftWriter(ctx, { identity? })` in + * @stc/feature-studio), so that writer is a `DraftSink` as is, with no adapter: Phase 2 passes it in as + * `aiFeature(ctx, { draftSink: createDraftWriter(ctx) })`. Drafts are never published by the AI package. */ export interface DraftSink { - writeDraftFiles(courseId: string, authorId: string, files: DraftFile[], message: string): Promise<{ location: string; paths: string[] }>; + writeDraftFiles(courseId: string, authorId: string, files: DraftFile[], message: string): Promise; +} + +/** Human-readable place of a draft: the folder, or `draft/ @ `. */ +export function draftLocation(r: DraftWriteResult): string { + if (r.location) return r.location; + if (r.branch) return r.commit ? `${r.branch} @ ${r.commit.slice(0, 7)}` : r.branch; + return "draft"; } export function safeRelPath(p: string): string | null { @@ -40,7 +54,7 @@ export function createFsDraftSink(dataDir: string): DraftSink { join(location, "DRAFT.json"), JSON.stringify({ courseId, authorId, message, createdAt: new Date().toISOString(), published: false, files: paths }, null, 2), ); - return { location, paths }; + return { location }; }, }; } diff --git a/packages/feature-ai/src/generator/generate.ts b/packages/feature-ai/src/generator/generate.ts index c0da188..6d8b23d 100644 --- a/packages/feature-ai/src/generator/generate.ts +++ b/packages/feature-ai/src/generator/generate.ts @@ -2,7 +2,7 @@ import { stat } from "node:fs/promises"; import { join } from "node:path"; import type { AiProvider } from "@stc/contracts"; import { resolveInside, type CourseRepository, type LoadedCourse } from "@stc/course-format"; -import { safeRelPath, type DraftFile, type DraftSink } from "./draft-sink"; +import { draftLocation, safeRelPath, type DraftFile, type DraftSink } from "./draft-sink"; import { OUTPUT_FORMAT, REQUIRED, VOICE, styleTemplate, type Kind, type Style } from "./templates"; export type GenerateSource = @@ -20,7 +20,8 @@ export interface GenerateRequest { instructions?: string; } -export interface GenerateResult { location: string; paths: string[]; warnings: string[] } +/** `location` is a folder (filesystem sink) or `draft/ @ ` (studio writer). */ +export interface GenerateResult { location: string; branch?: string; commit?: string; paths: string[]; warnings: string[] } const MAX_MATERIAL = 400_000; @@ -165,5 +166,11 @@ export async function generateDraft(o: { const files = parsed.files.map((f) => (/\.(md|markdown)$/i.test(f.path) ? { ...f, content: marker(o.req) + f.content } : f)); await o.onProgress?.(0.9, "Writing drafts"); const written = await o.sink.writeDraftFiles(course.id, o.authorId, files, `AI draft: ${o.req.kind} "${o.req.topic}" (${o.req.style} style)`); - return { ...written, warnings: [...material.warnings, ...parsed.warnings, ...checkHouseStyle(o.req, parsed.files)] }; + return { + location: draftLocation(written), + ...(written.branch ? { branch: written.branch } : {}), + ...(written.commit ? { commit: written.commit } : {}), + paths: files.map((f) => f.path), + warnings: [...material.warnings, ...parsed.warnings, ...checkHouseStyle(o.req, parsed.files)], + }; } diff --git a/packages/feature-ai/src/retrieval/course-index.ts b/packages/feature-ai/src/retrieval/course-index.ts index 57af7fb..0dadd53 100644 --- a/packages/feature-ai/src/retrieval/course-index.ts +++ b/packages/feature-ai/src/retrieval/course-index.ts @@ -128,6 +128,18 @@ function lessonTitleOf(course: LoadedCourse, lessonId: string | undefined): stri return lessonId ? course.lessonIndex.get(lessonId)?.lesson.title : undefined; } +async function readJson(path: string): Promise { + try { + const f = Bun.file(path); + if (!(await f.exists()) || f.size > MAX_FILE * 25) return null; + return (await f.json()) as T; + } catch { + return null; + } +} + +const clock = (s: number) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`; + export async function buildCourseIndex(course: LoadedCourse, dataDir: string): Promise { const bm = new Bm25(); const seen = new Set(); @@ -179,8 +191,50 @@ export async function buildCourseIndex(course: LoadedCourse, dataDir: string): P }); } - // Derived text from the importer and transcription jobs: /cache//**. const decks = new Map((course.manifest?.decks ?? []).map((d) => [d.id, d])); + + // The importer's deck cache: /cache/decks///text.json = [{ page, text }]. + for (const deck of decks.values()) { + const pages = await readJson<{ page?: number; text?: string }[]>(join(dataDir, "cache", "decks", course.id, deck.id, "text.json")); + if (!Array.isArray(pages)) continue; + const lessonId = course.lessonIndex.has(deck.attachTo) ? deck.attachTo : undefined; + for (const p of pages) { + const text = typeof p?.text === "string" ? p.text.trim() : ""; + if (!text) continue; + for (const part of splitLong(text)) { + add({ path: posix(deck.file), kind: "deck", text: part, heading: p.page ? `Slide ${p.page}` : deck.title, lessonId, lessonTitle: lessonTitleOf(course, lessonId) }); + } + } + } + + // The importer's transcription cache: /cache/transcripts//.json = + // { lessonId, text, segments: [{ start, end, text }] }. Skipped when the lesson has a transcript file in the + // course, which the transcription job writes alongside and which is indexed above. + const trDir = join(dataDir, "cache", "transcripts", course.id); + for (const name of await readdir(trDir).catch(() => [] as string[])) { + if (extname(name) !== ".json") continue; + const lessonId = name.slice(0, -5); + const entry = course.lessonIndex.get(lessonId); + if (!entry || entry.lesson.transcript) continue; + const tr = await readJson<{ text?: string; segments?: { start?: number; text?: string }[] }>(join(trDir, name)); + if (!tr) continue; + const segs = Array.isArray(tr.segments) ? tr.segments.filter((s) => typeof s?.text === "string" && s.text.trim()) : []; + const blocks: { at: number; text: string }[] = []; + if (segs.length) { + for (const s of segs) { + const last = blocks[blocks.length - 1]; + if (last && last.text.length + s.text!.length < MAX_CHUNK) last.text += ` ${s.text!.trim()}`; + else blocks.push({ at: Number(s.start) || 0, text: s.text!.trim() }); + } + } else if (typeof tr.text === "string" && tr.text.trim()) { + blocks.push(...splitLong(tr.text.trim()).map((t) => ({ at: -1, text: t }))); + } + for (const b of blocks) { + add({ path: entry.lesson.file ? posix(entry.lesson.file) : `transcripts/${lessonId}`, kind: "transcript", text: b.text, heading: b.at >= 0 ? `at ${clock(b.at)}` : undefined, lessonId, lessonTitle: entry.lesson.title }); + } + } + + // Older layout: plain text under /cache//**. const cacheRoot = join(dataDir, "cache", course.id); await walk(cacheRoot, async (abs, rel) => { if (!CACHE_TEXT.has(extname(rel).toLowerCase())) return true; diff --git a/packages/feature-ai/src/web/GeneratePage.tsx b/packages/feature-ai/src/web/GeneratePage.tsx index 2162102..c96c0fe 100644 --- a/packages/feature-ai/src/web/GeneratePage.tsx +++ b/packages/feature-ai/src/web/GeneratePage.tsx @@ -5,7 +5,7 @@ import { Field, PageHead } from "./components"; interface CourseOpt { id: string; title: string } interface Sources { lessons: { id: string; title: string; type: string; module: string; hasText: boolean }[]; files: string[]; cache: string[] } -interface Job { id: string; status: "queued" | "running" | "done" | "failed"; progress: number; message: string | null; error: string | null; result: { location: string; paths: string[]; warnings: string[] } | null } +interface Job { id: string; status: "queued" | "running" | "done" | "failed"; progress: number; message: string | null; error: string | null; result: { location: string; branch?: string; commit?: string; paths: string[]; warnings: string[] } | null } const KINDS = [ { value: "document", label: "Document" }, @@ -178,7 +178,7 @@ export default function GeneratePage() { {job.status === "failed" ?

{job.error}

: null} {job.result ? ( <> - Saved as a draft at {job.result.location}. Review it in the studio before publishing. + {job.result.branch ? "Committed to your draft branch " : "Saved as a draft at "}{job.result.location}. Review it in the studio before publishing.
    {job.result.paths.map((p) =>
  • {p}draft
    • )}
{job.result.warnings.length ? (
diff --git a/packages/feature-ai/test/generator.test.ts b/packages/feature-ai/test/generator.test.ts index 394a39e..8f91074 100644 --- a/packages/feature-ai/test/generator.test.ts +++ b/packages/feature-ai/test/generator.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { createCourseRepository, type CourseRepository } from "@stc/course-format"; import { buildGeneratorPrompt, gatherMaterial, generateDraft, parseFiles, checkHouseStyle, type GenerateRequest } from "../src/generator/generate"; -import { createFsDraftSink } from "../src/generator/draft-sink"; +import { createFsDraftSink, type DraftSink } from "../src/generator/draft-sink"; import { fakeProvider } from "./helpers"; const FIXTURES = join(import.meta.dir, "../../../fixtures/courses"); @@ -88,3 +88,25 @@ test("generation writes drafts through the sink and never touches the course", a expect(readdirSync(join(FIXTURES, "sample"), { recursive: true }).length).toBe(before); expect(existsSync(join(FIXTURES, "sample", "course"))).toBe(false); }); + +// The studio's DraftWriter (`createDraftWriter(ctx)` in @stc/feature-studio), reproduced structurally: it commits +// to `draft/` and returns { branch, commit }. It must be usable as a DraftSink without an adapter. +interface StudioDraftWriterShape { + writeDraftFiles(courseId: string, authorId: string, files: { path: string; content: string }[], message: string): Promise<{ branch: string; commit: string }>; +} + +test("a studio DraftWriter is a DraftSink as is, and the result names the draft branch", async () => { + const calls: { courseId: string; authorId: string; paths: string[]; message: string }[] = []; + const writer: StudioDraftWriterShape = { + async writeDraftFiles(courseId, authorId, files, message) { + calls.push({ courseId, authorId, paths: files.map((f) => f.path), message }); + return { branch: `draft/${authorId}`, commit: "0123456789abcdef0123456789abcdef01234567" }; + }, + }; + const sink: DraftSink = writer; + const { provider } = fakeProvider([() => ({ text: "=== FILE: questions/adding.md ===\n## Check your understanding\n=== END FILE ===\n" })]); + const r = await generateDraft({ provider, repo, req: req({ kind: "questions" }), sink, authorId: "author-1", dataDir }); + expect(calls).toEqual([{ courseId: "sample", authorId: "author-1", paths: ["questions/adding.md"], message: expect.stringContaining("AI draft") }]); + expect(r).toMatchObject({ branch: "draft/author-1", commit: "0123456789abcdef0123456789abcdef01234567", paths: ["questions/adding.md"] }); + expect(r.location).toBe("draft/author-1 @ 0123456"); +}); diff --git a/packages/feature-ai/test/retrieval.test.ts b/packages/feature-ai/test/retrieval.test.ts index 96f51b9..f9d94c3 100644 --- a/packages/feature-ai/test/retrieval.test.ts +++ b/packages/feature-ai/test/retrieval.test.ts @@ -1,5 +1,5 @@ import { afterAll, beforeAll, expect, test } from "bun:test"; -import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { cpSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { createCourseRepository, type LoadedCourse } from "@stc/course-format"; @@ -81,3 +81,50 @@ test("notes are searched alongside the course", async () => { const hits = idx.search("zebras striped", 3, extra); expect(hits[0]).toMatchObject({ kind: "note", lessonId: "m1-intro", lessonTitle: "Introduction" }); }); + +// The importer's layout (feature-import): /cache/decks///text.json ([{ page, text }]) +// and /cache/transcripts//.json ({ lessonId, text, segments: [{ start, end, text }] }). +test("reads the importer's deck text.json and transcript cache", async () => { + const dir = mkdtempSync(join(tmpdir(), "stc-ai-imp-")); + try { + const courses = join(dir, "courses"); + cpSync(join(FIXTURES, "sample"), join(courses, "sample"), { recursive: true }); + const yaml = join(courses, "sample/stcourse.yaml"); + writeFileSync(yaml, readFileSync(yaml, "utf8").replace(", transcript: lessons/02-video.md", "")); + const c = (await createCourseRepository([courses])).get("sample")!; + expect(c.status).toBe("ok"); + const data = join(dir, "data"); + mkdirSync(join(data, "cache/decks/sample/d1"), { recursive: true }); + writeFileSync(join(data, "cache/decks/sample/d1/text.json"), JSON.stringify([ + { page: 1, text: "Title slide" }, + { page: 2, text: "Orchestration with Kubernetes pods and replica sets." }, + { page: 3, text: "" }, + ])); + mkdirSync(join(data, "cache/transcripts/sample"), { recursive: true }); + writeFileSync(join(data, "cache/transcripts/sample/m1-video.json"), JSON.stringify({ + lessonId: "m1-video", + text: "Welcome. Today the presenter shows a waveform monitor.", + segments: [{ start: 0, end: 4, text: "Welcome." }, { start: 65, end: 70, text: "Today the presenter shows a waveform monitor." }], + })); + const idx = await buildCourseIndex(c, data); + const deck = idx.search("Kubernetes replica sets", 3)[0]!; + expect(deck).toMatchObject({ kind: "deck", path: "media/deck.pdf", heading: "Slide 2", lessonId: "m1-video" }); + const tr = idx.search("waveform monitor", 3)[0]!; + expect(tr).toMatchObject({ kind: "transcript", lessonId: "m1-video", lessonTitle: "Watch the clip" }); + expect(tr.text).toContain("waveform monitor"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("a cached transcript is skipped when the lesson already has a transcript file", async () => { + const data = mkdtempSync(join(tmpdir(), "stc-ai-tr-")); + try { + mkdirSync(join(data, "cache/transcripts/sample"), { recursive: true }); + writeFileSync(join(data, "cache/transcripts/sample/m1-video.json"), JSON.stringify({ lessonId: "m1-video", text: "zebracorn unique words", segments: [] })); + const idx = await buildCourseIndex(course, data); + expect(idx.search("zebracorn", 3)).toEqual([]); + } finally { + rmSync(data, { recursive: true, force: true }); + } +});