feat(ai): read the importer's deck and transcript cache; studio DraftWriter fits DraftSink as is
Retrieval now reads <dataDir>/cache/decks/<courseId>/<deckId>/text.json and
<dataDir>/cache/transcripts/<courseId>/<lessonId>.json (skipped when the lesson
has a transcript file). DraftSink returns { location?, branch?, commit? }, so
the studio's createDraftWriter(ctx) needs no adapter; generateDraft computes
paths itself and reports the draft branch.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Lucas Winther
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
08d3e3f642
commit
6fb0019cd9
@@ -4,11 +4,25 @@ import { dirname, join, resolve, sep } from "node:path";
|
||||
export interface DraftFile { path: string; content: string }
|
||||
|
||||
/**
|
||||
* Where generated files go. Same signature as the studio's `createDraftWriter(ctx).writeDraftFiles`, which
|
||||
* commits onto the author's draft branch; Phase 2 passes that in. Drafts are never published by the AI package.
|
||||
* What a sink reports back. The studio's writer returns `{ branch, commit }` (a commit on `draft/<authorId>`);
|
||||
* the filesystem fallback returns `{ location }` (a folder under `<dataDir>/drafts`).
|
||||
*/
|
||||
export interface DraftWriteResult { location?: string; branch?: string; commit?: string }
|
||||
|
||||
/**
|
||||
* Where generated files go. Same method as the studio's `DraftWriter` (`createDraftWriter(ctx, { identity? })` in
|
||||
* @stc/feature-studio), so that writer is a `DraftSink` as is, with no adapter: Phase 2 passes it in as
|
||||
* `aiFeature(ctx, { draftSink: createDraftWriter(ctx) })`. Drafts are never published by the AI package.
|
||||
*/
|
||||
export interface DraftSink {
|
||||
writeDraftFiles(courseId: string, authorId: string, files: DraftFile[], message: string): Promise<{ location: string; paths: string[] }>;
|
||||
writeDraftFiles(courseId: string, authorId: string, files: DraftFile[], message: string): Promise<DraftWriteResult>;
|
||||
}
|
||||
|
||||
/** Human-readable place of a draft: the folder, or `draft/<author> @ <short commit>`. */
|
||||
export function draftLocation(r: DraftWriteResult): string {
|
||||
if (r.location) return r.location;
|
||||
if (r.branch) return r.commit ? `${r.branch} @ ${r.commit.slice(0, 7)}` : r.branch;
|
||||
return "draft";
|
||||
}
|
||||
|
||||
export function safeRelPath(p: string): string | null {
|
||||
@@ -40,7 +54,7 @@ export function createFsDraftSink(dataDir: string): DraftSink {
|
||||
join(location, "DRAFT.json"),
|
||||
JSON.stringify({ courseId, authorId, message, createdAt: new Date().toISOString(), published: false, files: paths }, null, 2),
|
||||
);
|
||||
return { location, paths };
|
||||
return { location };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ import { stat } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import type { AiProvider } from "@stc/contracts";
|
||||
import { resolveInside, type CourseRepository, type LoadedCourse } from "@stc/course-format";
|
||||
import { safeRelPath, type DraftFile, type DraftSink } from "./draft-sink";
|
||||
import { draftLocation, safeRelPath, type DraftFile, type DraftSink } from "./draft-sink";
|
||||
import { OUTPUT_FORMAT, REQUIRED, VOICE, styleTemplate, type Kind, type Style } from "./templates";
|
||||
|
||||
export type GenerateSource =
|
||||
@@ -20,7 +20,8 @@ export interface GenerateRequest {
|
||||
instructions?: string;
|
||||
}
|
||||
|
||||
export interface GenerateResult { location: string; paths: string[]; warnings: string[] }
|
||||
/** `location` is a folder (filesystem sink) or `draft/<author> @ <commit>` (studio writer). */
|
||||
export interface GenerateResult { location: string; branch?: string; commit?: string; paths: string[]; warnings: string[] }
|
||||
|
||||
const MAX_MATERIAL = 400_000;
|
||||
|
||||
@@ -165,5 +166,11 @@ export async function generateDraft(o: {
|
||||
const files = parsed.files.map((f) => (/\.(md|markdown)$/i.test(f.path) ? { ...f, content: marker(o.req) + f.content } : f));
|
||||
await o.onProgress?.(0.9, "Writing drafts");
|
||||
const written = await o.sink.writeDraftFiles(course.id, o.authorId, files, `AI draft: ${o.req.kind} "${o.req.topic}" (${o.req.style} style)`);
|
||||
return { ...written, warnings: [...material.warnings, ...parsed.warnings, ...checkHouseStyle(o.req, parsed.files)] };
|
||||
return {
|
||||
location: draftLocation(written),
|
||||
...(written.branch ? { branch: written.branch } : {}),
|
||||
...(written.commit ? { commit: written.commit } : {}),
|
||||
paths: files.map((f) => f.path),
|
||||
warnings: [...material.warnings, ...parsed.warnings, ...checkHouseStyle(o.req, parsed.files)],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -128,6 +128,18 @@ function lessonTitleOf(course: LoadedCourse, lessonId: string | undefined): stri
|
||||
return lessonId ? course.lessonIndex.get(lessonId)?.lesson.title : undefined;
|
||||
}
|
||||
|
||||
async function readJson<T>(path: string): Promise<T | null> {
|
||||
try {
|
||||
const f = Bun.file(path);
|
||||
if (!(await f.exists()) || f.size > MAX_FILE * 25) return null;
|
||||
return (await f.json()) as T;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const clock = (s: number) => `${Math.floor(s / 60)}:${String(Math.floor(s % 60)).padStart(2, "0")}`;
|
||||
|
||||
export async function buildCourseIndex(course: LoadedCourse, dataDir: string): Promise<CourseIndex> {
|
||||
const bm = new Bm25<Chunk>();
|
||||
const seen = new Set<string>();
|
||||
@@ -179,8 +191,50 @@ export async function buildCourseIndex(course: LoadedCourse, dataDir: string): P
|
||||
});
|
||||
}
|
||||
|
||||
// Derived text from the importer and transcription jobs: <dataDir>/cache/<courseId>/**.
|
||||
const decks = new Map((course.manifest?.decks ?? []).map((d) => [d.id, d]));
|
||||
|
||||
// The importer's deck cache: <dataDir>/cache/decks/<courseId>/<deckId>/text.json = [{ page, text }].
|
||||
for (const deck of decks.values()) {
|
||||
const pages = await readJson<{ page?: number; text?: string }[]>(join(dataDir, "cache", "decks", course.id, deck.id, "text.json"));
|
||||
if (!Array.isArray(pages)) continue;
|
||||
const lessonId = course.lessonIndex.has(deck.attachTo) ? deck.attachTo : undefined;
|
||||
for (const p of pages) {
|
||||
const text = typeof p?.text === "string" ? p.text.trim() : "";
|
||||
if (!text) continue;
|
||||
for (const part of splitLong(text)) {
|
||||
add({ path: posix(deck.file), kind: "deck", text: part, heading: p.page ? `Slide ${p.page}` : deck.title, lessonId, lessonTitle: lessonTitleOf(course, lessonId) });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The importer's transcription cache: <dataDir>/cache/transcripts/<courseId>/<lessonId>.json =
|
||||
// { lessonId, text, segments: [{ start, end, text }] }. Skipped when the lesson has a transcript file in the
|
||||
// course, which the transcription job writes alongside and which is indexed above.
|
||||
const trDir = join(dataDir, "cache", "transcripts", course.id);
|
||||
for (const name of await readdir(trDir).catch(() => [] as string[])) {
|
||||
if (extname(name) !== ".json") continue;
|
||||
const lessonId = name.slice(0, -5);
|
||||
const entry = course.lessonIndex.get(lessonId);
|
||||
if (!entry || entry.lesson.transcript) continue;
|
||||
const tr = await readJson<{ text?: string; segments?: { start?: number; text?: string }[] }>(join(trDir, name));
|
||||
if (!tr) continue;
|
||||
const segs = Array.isArray(tr.segments) ? tr.segments.filter((s) => typeof s?.text === "string" && s.text.trim()) : [];
|
||||
const blocks: { at: number; text: string }[] = [];
|
||||
if (segs.length) {
|
||||
for (const s of segs) {
|
||||
const last = blocks[blocks.length - 1];
|
||||
if (last && last.text.length + s.text!.length < MAX_CHUNK) last.text += ` ${s.text!.trim()}`;
|
||||
else blocks.push({ at: Number(s.start) || 0, text: s.text!.trim() });
|
||||
}
|
||||
} else if (typeof tr.text === "string" && tr.text.trim()) {
|
||||
blocks.push(...splitLong(tr.text.trim()).map((t) => ({ at: -1, text: t })));
|
||||
}
|
||||
for (const b of blocks) {
|
||||
add({ path: entry.lesson.file ? posix(entry.lesson.file) : `transcripts/${lessonId}`, kind: "transcript", text: b.text, heading: b.at >= 0 ? `at ${clock(b.at)}` : undefined, lessonId, lessonTitle: entry.lesson.title });
|
||||
}
|
||||
}
|
||||
|
||||
// Older layout: plain text under <dataDir>/cache/<courseId>/**.
|
||||
const cacheRoot = join(dataDir, "cache", course.id);
|
||||
await walk(cacheRoot, async (abs, rel) => {
|
||||
if (!CACHE_TEXT.has(extname(rel).toLowerCase())) return true;
|
||||
|
||||
@@ -5,7 +5,7 @@ import { Field, PageHead } from "./components";
|
||||
|
||||
interface CourseOpt { id: string; title: string }
|
||||
interface Sources { lessons: { id: string; title: string; type: string; module: string; hasText: boolean }[]; files: string[]; cache: string[] }
|
||||
interface Job { id: string; status: "queued" | "running" | "done" | "failed"; progress: number; message: string | null; error: string | null; result: { location: string; paths: string[]; warnings: string[] } | null }
|
||||
interface Job { id: string; status: "queued" | "running" | "done" | "failed"; progress: number; message: string | null; error: string | null; result: { location: string; branch?: string; commit?: string; paths: string[]; warnings: string[] } | null }
|
||||
|
||||
const KINDS = [
|
||||
{ value: "document", label: "Document" },
|
||||
@@ -178,7 +178,7 @@ export default function GeneratePage() {
|
||||
{job.status === "failed" ? <p className="ai-error">{job.error}</p> : null}
|
||||
{job.result ? (
|
||||
<>
|
||||
<span className="ai-hint">Saved as a draft at <span className="ai-mono">{job.result.location}</span>. Review it in the studio before publishing.</span>
|
||||
<span className="ai-hint">{job.result.branch ? "Committed to your draft branch " : "Saved as a draft at "}<span className="ai-mono">{job.result.location}</span>. Review it in the studio before publishing.</span>
|
||||
<ul className="ai-rows">{job.result.paths.map((p) => <li key={p}><span className="ai-path">{p}</span><span className="ai-meta">draft</span></li>)}</ul>
|
||||
{job.result.warnings.length ? (
|
||||
<div className="ai-section">
|
||||
|
||||
@@ -4,7 +4,7 @@ import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { createCourseRepository, type CourseRepository } from "@stc/course-format";
|
||||
import { buildGeneratorPrompt, gatherMaterial, generateDraft, parseFiles, checkHouseStyle, type GenerateRequest } from "../src/generator/generate";
|
||||
import { createFsDraftSink } from "../src/generator/draft-sink";
|
||||
import { createFsDraftSink, type DraftSink } from "../src/generator/draft-sink";
|
||||
import { fakeProvider } from "./helpers";
|
||||
|
||||
const FIXTURES = join(import.meta.dir, "../../../fixtures/courses");
|
||||
@@ -88,3 +88,25 @@ test("generation writes drafts through the sink and never touches the course", a
|
||||
expect(readdirSync(join(FIXTURES, "sample"), { recursive: true }).length).toBe(before);
|
||||
expect(existsSync(join(FIXTURES, "sample", "course"))).toBe(false);
|
||||
});
|
||||
|
||||
// The studio's DraftWriter (`createDraftWriter(ctx)` in @stc/feature-studio), reproduced structurally: it commits
|
||||
// to `draft/<authorId>` and returns { branch, commit }. It must be usable as a DraftSink without an adapter.
|
||||
interface StudioDraftWriterShape {
|
||||
writeDraftFiles(courseId: string, authorId: string, files: { path: string; content: string }[], message: string): Promise<{ branch: string; commit: string }>;
|
||||
}
|
||||
|
||||
test("a studio DraftWriter is a DraftSink as is, and the result names the draft branch", async () => {
|
||||
const calls: { courseId: string; authorId: string; paths: string[]; message: string }[] = [];
|
||||
const writer: StudioDraftWriterShape = {
|
||||
async writeDraftFiles(courseId, authorId, files, message) {
|
||||
calls.push({ courseId, authorId, paths: files.map((f) => f.path), message });
|
||||
return { branch: `draft/${authorId}`, commit: "0123456789abcdef0123456789abcdef01234567" };
|
||||
},
|
||||
};
|
||||
const sink: DraftSink = writer;
|
||||
const { provider } = fakeProvider([() => ({ text: "=== FILE: questions/adding.md ===\n## Check your understanding\n=== END FILE ===\n" })]);
|
||||
const r = await generateDraft({ provider, repo, req: req({ kind: "questions" }), sink, authorId: "author-1", dataDir });
|
||||
expect(calls).toEqual([{ courseId: "sample", authorId: "author-1", paths: ["questions/adding.md"], message: expect.stringContaining("AI draft") }]);
|
||||
expect(r).toMatchObject({ branch: "draft/author-1", commit: "0123456789abcdef0123456789abcdef01234567", paths: ["questions/adding.md"] });
|
||||
expect(r.location).toBe("draft/author-1 @ 0123456");
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { afterAll, beforeAll, expect, test } from "bun:test";
|
||||
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { cpSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { createCourseRepository, type LoadedCourse } from "@stc/course-format";
|
||||
@@ -81,3 +81,50 @@ test("notes are searched alongside the course", async () => {
|
||||
const hits = idx.search("zebras striped", 3, extra);
|
||||
expect(hits[0]).toMatchObject({ kind: "note", lessonId: "m1-intro", lessonTitle: "Introduction" });
|
||||
});
|
||||
|
||||
// The importer's layout (feature-import): <dataDir>/cache/decks/<courseId>/<deckId>/text.json ([{ page, text }])
|
||||
// and <dataDir>/cache/transcripts/<courseId>/<lessonId>.json ({ lessonId, text, segments: [{ start, end, text }] }).
|
||||
test("reads the importer's deck text.json and transcript cache", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "stc-ai-imp-"));
|
||||
try {
|
||||
const courses = join(dir, "courses");
|
||||
cpSync(join(FIXTURES, "sample"), join(courses, "sample"), { recursive: true });
|
||||
const yaml = join(courses, "sample/stcourse.yaml");
|
||||
writeFileSync(yaml, readFileSync(yaml, "utf8").replace(", transcript: lessons/02-video.md", ""));
|
||||
const c = (await createCourseRepository([courses])).get("sample")!;
|
||||
expect(c.status).toBe("ok");
|
||||
const data = join(dir, "data");
|
||||
mkdirSync(join(data, "cache/decks/sample/d1"), { recursive: true });
|
||||
writeFileSync(join(data, "cache/decks/sample/d1/text.json"), JSON.stringify([
|
||||
{ page: 1, text: "Title slide" },
|
||||
{ page: 2, text: "Orchestration with Kubernetes pods and replica sets." },
|
||||
{ page: 3, text: "" },
|
||||
]));
|
||||
mkdirSync(join(data, "cache/transcripts/sample"), { recursive: true });
|
||||
writeFileSync(join(data, "cache/transcripts/sample/m1-video.json"), JSON.stringify({
|
||||
lessonId: "m1-video",
|
||||
text: "Welcome. Today the presenter shows a waveform monitor.",
|
||||
segments: [{ start: 0, end: 4, text: "Welcome." }, { start: 65, end: 70, text: "Today the presenter shows a waveform monitor." }],
|
||||
}));
|
||||
const idx = await buildCourseIndex(c, data);
|
||||
const deck = idx.search("Kubernetes replica sets", 3)[0]!;
|
||||
expect(deck).toMatchObject({ kind: "deck", path: "media/deck.pdf", heading: "Slide 2", lessonId: "m1-video" });
|
||||
const tr = idx.search("waveform monitor", 3)[0]!;
|
||||
expect(tr).toMatchObject({ kind: "transcript", lessonId: "m1-video", lessonTitle: "Watch the clip" });
|
||||
expect(tr.text).toContain("waveform monitor");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("a cached transcript is skipped when the lesson already has a transcript file", async () => {
|
||||
const data = mkdtempSync(join(tmpdir(), "stc-ai-tr-"));
|
||||
try {
|
||||
mkdirSync(join(data, "cache/transcripts/sample"), { recursive: true });
|
||||
writeFileSync(join(data, "cache/transcripts/sample/m1-video.json"), JSON.stringify({ lessonId: "m1-video", text: "zebracorn unique words", segments: [] }));
|
||||
const idx = await buildCourseIndex(course, data);
|
||||
expect(idx.search("zebracorn", 3)).toEqual([]);
|
||||
} finally {
|
||||
rmSync(data, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user