Files
STCourses/packages/feature-studio/src/server/feature.ts
T
Lucas WintherandClaude Opus 5.5 f02e30450c fix(studio): review round 2 - a conversion that fails after its commit recovers without a restart
- The convert job reloads the course on success and on failure, so learners
  get the intact import (rolled back) or the published studio course, never a
  course whose folder moved away.
- POST /convert on a course with an incomplete conversion (.converting-<id>,
  staging or retired folders) queues a job that recovers and finishes it
  instead of answering 409; the studio page shows "Conversion incomplete" with
  Retry for the owner.
- The 409 for pending import jobs names their kinds and ids; the dead withLock
  in git.ts is gone; the dot-folder discovery rule is documented in
  docs/operations.md and spec section 5.
- Route tests (SQLite and Postgres) inject a failed cross-filesystem copy, a
  checksum mismatch and a pre-commit failure.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-30 21:00:45 +01:00

446 lines
23 KiB
TypeScript

import { existsSync, realpathSync } from "node:fs";
import { basename, extname, resolve, sep } from "node:path";
import type { AuthUser, Feature, JobHandler, RequestContext, RouteHandler, ServerContext } from "@stc/contracts";
import { canView, serveMedia } from "@stc/core";
import { ManifestSchema, parseManifest, resolveInside, type LoadedCourse } from "@stc/course-format";
import { aclRepo } from "@stc/db";
import { createDraftWriter, type DraftWriter } from "./draft-writer";
import { storeDirs, studioWorktreesDir } from "./paths";
import { checkRelPath, createStudioStore, MANIFEST, ownerRequiredFor, StudioError, type FileWrite, type StudioStore, type StudioStoreOptions } from "./store";
import { createPublishedWatcher, type PublishedWatcher } from "./watch";
export const OUTSIDE_MESSAGE = "This course is managed outside STCourses";
export const IMPORTED_MESSAGE = "Imported course: not editable in the studio until the owner makes it editable";
export const CONVERT_JOB = "studio.convert-import";
/** Upload cap in bytes from AppConfig.uploadMaxMb (STC_UPLOAD_MAX_MB, default 100; review I2). */
export function uploadMaxBytes(mb: number | undefined): number {
return (typeof mb === "number" && Number.isFinite(mb) && mb > 0 ? mb : 100) * 1024 * 1024;
}
/** Server-side allowlist by extension for uploads: text, documents, images, media, archives and code. */
export const UPLOAD_EXTENSIONS = new Set([
"md", "markdown", "txt", "pdf", "png", "jpg", "jpeg", "gif", "svg", "webp", "mp4", "webm", "mp3", "vtt", "srt", "zip",
"js", "mjs", "cjs", "jsx", "ts", "tsx", "json", "yaml", "yml", "py", "sh", "css", "html", "csv", "toml", "sql",
"scala", "sc", "fs", "fsx", "fsproj", "rs", "go", "java", "kt", "c", "h", "cpp", "hpp", "cs", "rb", "php", "swift", "hs", "ex", "exs", "dockerfile",
]);
/** Reads at most `max` bytes of the body; null when it is larger (checked while streaming, not after buffering). */
async function readCapped(req: Request, max: number): Promise<Uint8Array<ArrayBuffer> | null> {
const reader = req.body?.getReader();
if (!reader) return new Uint8Array();
const chunks: Uint8Array[] = [];
let total = 0;
for (;;) {
const { done, value } = await reader.read();
if (done) break;
total += value.byteLength;
if (total > max) { await reader.cancel(); return null; }
chunks.push(value);
}
const out = new Uint8Array(total);
let off = 0;
for (const c of chunks) { out.set(c, off); off += c.byteLength; }
return out;
}
export interface StudioFeature extends Feature {
jobs: Record<string, JobHandler>;
store: StudioStore;
watcher: PublishedWatcher;
draftWriter: DraftWriter;
/** Resolves once interrupted conversions are recovered (every studio route waits for it). */
ready: Promise<void>;
stop(): void;
}
export interface StudioFeatureOptions {
/** Poll the published worktrees for external commits (default true). */
watch?: boolean;
watchIntervalMs?: number;
/** Tests only: conversion failure injection (see StudioStoreOptions.testHooks). */
storeTestHooks?: StudioStoreOptions["testHooks"];
}
const json = (body: unknown, status = 200) => Response.json(body, { status });
const fail = (status: number, error: string) => json({ error }, status);
async function body<T>(req: Request): Promise<T> {
try {
return (await req.json()) as T;
} catch {
throw new StudioError("request body must be JSON");
}
}
function manifestToYaml(m: unknown): string {
return Bun.YAML.stringify(m, null, 2) + "\n";
}
export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = {}): StudioFeature {
const store = createStudioStore({ ...storeDirs(ctx.config), testHooks: opts.storeTestHooks });
const watcher = createPublishedWatcher(store, ctx.courses, { intervalMs: opts.watchIntervalMs });
const acl = aclRepo(ctx.db);
const wtDir = resolve(ctx.config.studioWorktreesDir ?? studioWorktreesDir(ctx.config.dataDir));
const draftWriter = createDraftWriter(ctx);
const hasRole = (u: AuthUser) => u.roles.includes("owner") || u.roles.includes("author");
const authoredCourses = async (u: AuthUser): Promise<string[]> =>
(await ctx.db.sql`SELECT course_id FROM course_authors WHERE user_id = ${u.id}`).map((r: { course_id: string }) => r.course_id);
/**
* An authoring right (owner, the `author` role, or a course_authors row) and, for a course the server has loaded,
* core `canView` (the one visibility rule, R14). A studio course not loaded yet (e.g. its published manifest is
* broken) has no visibility to check.
*/
const canEdit = async (u: AuthUser, courseId: string) => {
const isAuthor = await acl.isAuthor(courseId, u.id);
if (!hasRole(u) && !isAuthor) return false;
const c = ctx.courses.get(courseId);
return !c || canView(u, c, await acl.listForCourse(courseId), isAuthor);
};
const isStudioCourse = (c: LoadedCourse) => resolve(c.manifestPath).startsWith(wtDir + sep);
const who = (u: AuthUser) => ({ id: u.id, name: u.name || u.email || u.id, email: u.email || `${u.id}@users.stcourses.invalid`, owner: u.roles.includes("owner") });
const importedRoot = (() => { const d = resolve(ctx.config.importedCoursesDir ?? resolve(ctx.config.dataDir, "courses")); return existsSync(d) ? realpathSync(d) : d; })();
/** Why a course that is not studio-managed is read-only here. */
const readOnlyMessage = (c: LoadedCourse): string => {
const kind = c.manifest?.source?.kind;
const underImports = c.root === importedRoot || c.root.startsWith(importedRoot + sep);
if (kind && kind !== "authored" && underImports) return IMPORTED_MESSAGE;
return OUTSIDE_MESSAGE;
};
const isOwner = (u: AuthUser) => u.roles.includes("owner");
/** v1.1 S: only the owner may turn an imported course into a studio course ("Make editable in the studio"). */
const convertible = (u: AuthUser, courseId: string) => isOwner(u) && store.isConvertibleImport(courseId);
/** Importer jobs (queued or running) that would write into this course's import folder. */
const pendingImports = async (courseId: string) =>
(await ctx.jobs.list()).filter((j) => {
if (!j.kind.startsWith("import.") || (j.status !== "queued" && j.status !== "running")) return false;
const p = (j.payload ?? {}) as { courseId?: unknown; options?: { courseId?: unknown } };
return p.courseId === courseId || p.options?.courseId === courseId;
});
/** 409 text naming the blocking importer jobs (id and kind), so the owner can find them on the import page. */
const importRunning = (jobs: { id: string; kind: string }[]) =>
`an import job for this course is still running (${jobs.map((j) => `${j.kind} ${j.id}`).join(", ")}); make it editable once it has finished`;
/** The conversion job: payload `{ courseId, actor }`, checked again by the store under the course lock. */
const convertJob: JobHandler = async (payload, jc) => {
await ready;
const p = (payload ?? {}) as { courseId?: unknown; actor?: { id?: unknown; name?: unknown; email?: unknown } };
const courseId = typeof p.courseId === "string" ? p.courseId : "";
const blocking = await pendingImports(courseId);
if (blocking.length) throw new StudioError(importRunning(blocking), 409);
const actor = { name: String(p.actor?.name ?? "STCourses"), email: String(p.actor?.email ?? "[email protected]") };
try {
const r = await store.convertImported(courseId, actor, { onProgress: (f, m) => jc.reportProgress(f, m) });
return { courseId, ...r };
} finally {
// Success or failure, learners must get whatever is on disk now: the intact import (failure before the commit,
// rolled back) or the published worktree (after it), never a course whose folder has moved away.
await ctx.courses.reload(courseId).catch((e) => console.error("[studio] reload after conversion", e));
if (store.isManaged(courseId)) await store.publishedHead(courseId).then((h) => watcher.note(courseId, h), () => {});
}
};
const afterPublishedChange = async (courseId: string) => {
await ctx.courses.reload(courseId);
watcher.note(courseId, await store.publishedHead(courseId));
};
// Crash recovery for conversions interrupted by a stop (v1.1 S); learners never see a course twice meanwhile.
const ready: Promise<void> = store.recoverConversions()
.then(async (ids) => { if (ids.length) await ctx.courses.reload(); })
.catch((e) => console.error("[studio] recovery", e));
/** Wraps a handler: signed-in author/owner (or per-course author), error mapping. */
const guard = (h: (req: Request, rc: RequestContext, user: AuthUser) => Promise<Response>): RouteHandler => async (req, rc) => {
const user = rc.user;
if (!user) return fail(401, "sign in to use the studio");
try {
await ready;
if (!hasRole(user) && (await authoredCourses(user)).length === 0) return fail(403, "the studio is for authors");
return await h(req, rc, user);
} catch (e) {
if (e instanceof StudioError) return fail(e.status, e.message);
const msg = e instanceof Error ? e.message : String(e);
if (/^invalid path/.test(msg)) return fail(400, msg);
console.error("[studio]", e);
return fail(500, msg);
}
};
/** For per-course routes: the course must be studio-managed and editable by the user. */
const course = (h: (req: Request, rc: RequestContext, user: AuthUser, courseId: string) => Promise<Response>) =>
guard(async (req, rc, user) => {
const courseId = rc.params.courseId ?? "";
if (!(await canEdit(user, courseId))) return fail(403, "you are not an author of this course");
if (!store.isManaged(courseId)) {
const c = ctx.courses.get(courseId);
return c ? fail(403, readOnlyMessage(c)) : fail(404, "no such course");
}
return h(req, rc, user, courseId);
});
async function courseState(courseId: string, user: AuthUser) {
const status = await store.status(courseId, user.id);
const m = await store.readFile(courseId, user.id, MANIFEST);
let manifestRaw: unknown = null;
let manifestErrors: string[] = [];
if (m) {
try { manifestRaw = Bun.YAML.parse(m.content); } catch { /* reported by parseManifest */ }
const parsed = parseManifest(m.content);
manifestErrors = parsed.ok ? [] : parsed.errors;
} else {
manifestErrors = [`${MANIFEST} is missing`];
}
return {
readOnly: false,
conversionIncomplete: store.conversionIncomplete(courseId),
canRetryConversion: isOwner(user),
course: { id: courseId, title: (manifestRaw as { title?: string } | null)?.title ?? courseId, managed: true },
status,
manifestText: m?.content ?? "",
manifestHash: m?.hash ?? null,
manifestRaw,
manifestErrors,
diagnostics: status.merging ? [] : await store.validateDraft(courseId, user.id),
files: await store.listFiles(courseId, user.id),
};
}
const routes: Feature["routes"] = {
"/api/studio/courses": {
GET: guard(async (_req, _rc, user) => {
const mine = new Set(await authoredCourses(user));
const all = hasRole(user);
const managed = await store.listManaged();
const courses: unknown[] = [];
for (const id of managed) {
if (!all && !mine.has(id)) continue;
const loaded = ctx.courses.get(id);
const st = await store.summary(id, user.id).catch(() => null); // read-only: no worktree is created here
courses.push({
id, managed: true,
title: loaded?.manifest?.title ?? id,
pitch: loaded?.manifest?.pitch ?? null,
accent: loaded?.manifest?.accent ?? null,
publishedStatus: loaded?.status ?? "error",
ahead: st?.ahead ?? 0, behind: st?.behind ?? 0, merging: st?.merging ?? false, remote: st?.remote ?? null,
});
}
if (all) {
for (const c of ctx.courses.list()) {
if (isStudioCourse(c) || managed.includes(c.id)) continue;
const message = readOnlyMessage(c);
courses.push({
id: c.id, managed: false, title: c.manifest?.title ?? c.id, pitch: c.manifest?.pitch ?? null, accent: c.manifest?.accent ?? null, publishedStatus: c.status, message,
imported: message === IMPORTED_MESSAGE, convertible: convertible(user, c.id),
});
}
}
return json({ courses, canCreate: all });
}),
POST: guard(async (req, _rc, user) => {
if (!hasRole(user)) return fail(403, "only authors can create courses");
const b = await body<{ id?: unknown; title?: unknown; pitch?: unknown }>(req);
if (typeof b.id !== "string" || typeof b.title !== "string" || !b.title.trim()) return fail(400, "id and title are required");
if (ctx.courses.get(b.id) || store.isManaged(b.id)) return fail(409, `course "${b.id}" already exists`);
await store.createCourse({ id: b.id, title: b.title, pitch: typeof b.pitch === "string" ? b.pitch : undefined }, who(user));
await ctx.db.sql`INSERT INTO course_authors (course_id, user_id) VALUES (${b.id}, ${user.id})`;
await afterPublishedChange(b.id);
return json({ id: b.id }, 201);
}),
},
"/api/studio/courses/:courseId": {
GET: guard(async (_req, rc, user) => {
const courseId = rc.params.courseId ?? "";
if (!(await canEdit(user, courseId))) return fail(403, "you are not an author of this course");
if (!store.isManaged(courseId)) {
const c = ctx.courses.get(courseId);
if (!c) return fail(404, "no such course");
const message = readOnlyMessage(c);
return json({ readOnly: true, message, imported: message === IMPORTED_MESSAGE, convertible: convertible(user, c.id), course: { id: c.id, title: c.manifest?.title ?? c.id, managed: false, root: c.root }, diagnostics: c.diagnostics });
}
return json(await courseState(courseId, user));
}),
},
"/api/studio/courses/:courseId/convert": {
POST: guard(async (_req, rc, user) => {
const courseId = rc.params.courseId ?? "";
if (!isOwner(user)) return fail(403, "only the owner can make an imported course editable in the studio");
// An incomplete conversion (failed after its commit) is finished by a new job: it recovers first.
const incomplete = store.conversionIncomplete(courseId);
if (store.isManaged(courseId) && !incomplete) return fail(409, `course "${courseId}" is already a studio course`);
if (!incomplete && !store.isConvertibleImport(courseId)) {
const c = ctx.courses.get(courseId);
return c ? fail(409, `${readOnlyMessage(c) === IMPORTED_MESSAGE ? "this imported course" : OUTSIDE_MESSAGE}: only imported courses in ${importedRoot} can be made editable`) : fail(404, "no such imported course");
}
const pending = (await ctx.jobs.list(CONVERT_JOB)).find((j) => (j.payload as { courseId?: string } | null)?.courseId === courseId && (j.status === "queued" || j.status === "running"));
if (pending) return fail(409, "this course is already being made editable");
const blocking = await pendingImports(courseId);
if (blocking.length) return fail(409, importRunning(blocking));
const jobId = await ctx.jobs.enqueue(CONVERT_JOB, { courseId, actor: { id: user.id, name: who(user).name, email: who(user).email } });
return json({ jobId }, 202);
}),
},
"/api/studio/jobs/:jobId": {
GET: guard(async (_req, rc, user) => {
if (!isOwner(user)) return fail(403, "only the owner can follow studio jobs");
const j = await ctx.jobs.get(rc.params.jobId ?? "");
if (!j || !j.kind.startsWith("studio.")) return fail(404, "no such job");
return json({ id: j.id, kind: j.kind, status: j.status, progress: j.progress, message: j.message, error: j.error, result: j.result });
}),
},
"/api/studio/courses/:courseId/file": {
GET: course(async (_req, rc, user, courseId) => {
const path = rc.url.searchParams.get("path") ?? "";
const f = await store.readFile(courseId, user.id, path);
return f ? json({ path, ...f }) : fail(404, "no such file");
}),
},
"/api/studio/courses/:courseId/files": {
PUT: course(async (req, _rc, user, courseId) => {
const b = await body<{ message?: string; files?: FileWrite[]; manifest?: unknown; manifestBaseHash?: string | null }>(req);
const files: FileWrite[] = Array.isArray(b.files) ? b.files.map((f) => ({ path: String(f.path), content: f.delete ? undefined : String(f.content ?? ""), baseHash: f.baseHash, delete: !!f.delete })) : [];
if (b.manifest !== undefined) {
files.unshift({ path: MANIFEST, content: manifestToYaml(b.manifest), baseHash: b.manifestBaseHash });
}
const mf = files.find((f) => checkRelPath(f.path) === MANIFEST);
if (mf && typeof mf.content === "string" && !user.roles.includes("owner")) {
// Early feedback only; publish, sync and resolve enforce the same rule on the merged result.
const needOwner = ownerRequiredFor(await store.publishedManifest(courseId), mf.content);
if (needOwner) return fail(403, needOwner);
}
const r = await store.saveFiles(courseId, who(user), files, b.message ?? "");
if (!r.ok) return json({ ok: false, reason: "conflict", conflicts: r.conflicts }, 409);
return json({ ok: true, commit: r.commit });
}),
},
"/api/studio/courses/:courseId/validate": {
POST: course(async (req) => {
const b = await body<{ manifest?: unknown; manifestText?: string }>(req);
if (typeof b.manifestText === "string") {
const r = parseManifest(b.manifestText);
return json({ errors: r.ok ? [] : r.errors });
}
const r = ManifestSchema.safeParse(b.manifest);
return json({ errors: r.success ? [] : r.error.issues.map((i) => `${i.path.join(".") || "(root)"}: ${i.message}`) });
}),
},
"/api/studio/courses/:courseId/upload": {
POST: course(async (req, _rc, user, courseId) => {
const max = uploadMaxBytes(ctx.config.uploadMaxMb);
const declared = Number(req.headers.get("content-length") ?? "0");
if (declared > max) return fail(413, `file too large (limit ${Math.round(max / 1048576)} MB)`);
const bytes = await readCapped(req, max);
if (!bytes) return fail(413, `file too large (limit ${Math.round(max / 1048576)} MB)`);
let form: FormData;
try { form = await new Response(bytes, { headers: { "content-type": req.headers.get("content-type") ?? "" } }).formData(); } catch { return fail(400, "expected multipart form data"); }
const file = form.get("file");
if (!(file instanceof Blob)) return fail(400, "file is required");
const explicit = form.get("path");
const dir = String(form.get("dir") ?? "").replace(/^\/+|\/+$/g, "");
const name = basename((file as File).name || "upload.bin");
const path = checkRelPath(typeof explicit === "string" && explicit ? explicit : dir ? `${dir}/${name}` : name);
if (path === MANIFEST) return fail(400, "edit stcourse.yaml in the manifest editor, not by upload");
const ext = (extname(path).slice(1) || basename(path)).toLowerCase();
if (!UPLOAD_EXTENSIONS.has(ext)) return fail(415, `file type .${ext} is not accepted`);
const overwrite = String(form.get("overwrite") ?? "") === "true";
const message = String(form.get("message") ?? "") || `Upload ${path}`;
// baseHash null = the file must not exist yet; replacing needs an explicit overwrite=true
const r = await store.saveFiles(courseId, who(user), [{ path, content: new Uint8Array(await file.arrayBuffer()), ...(overwrite ? {} : { baseHash: null }) }], message);
if (!r.ok) return json({ ok: false, reason: "exists", error: `${path} already exists; upload again with overwrite to replace it`, conflicts: r.conflicts }, 409);
return json({ ok: true, path, commit: r.commit });
}),
},
"/api/studio/courses/:courseId/raw/*": {
GET: course(async (req, rc, user, courseId) => {
const m = /^\/api\/studio\/courses\/[^/]+\/raw\/(.*)$/s.exec(rc.url.pathname);
let rel: string;
try { rel = decodeURIComponent(m?.[1] ?? ""); } catch { return fail(404, "not found"); }
const dir = await store.draftDir(courseId, user.id);
try { checkRelPath(rel); } catch { return fail(404, "not found"); }
const abs = resolveInside(dir, rel);
if (!abs) return fail(404, "not found");
const res = await serveMedia(req, abs, { download: rc.url.searchParams.has("download") });
// Draft files are author content served from the app's origin: never let them run script (SVG, HTML).
res.headers.set("Content-Security-Policy", "default-src 'none'; img-src 'self' data:; style-src 'unsafe-inline'; media-src 'self'; sandbox");
res.headers.set("X-Content-Type-Options", "nosniff");
return res;
}),
},
"/api/studio/courses/:courseId/publish": {
POST: course(async (_req, _rc, user, courseId) => {
const r = await store.publish(courseId, who(user));
if (r.ok) {
await afterPublishedChange(courseId);
return json(r);
}
return json(r, r.reason === "invalid" ? 422 : r.reason === "forbidden" ? 403 : 409);
}),
},
"/api/studio/courses/:courseId/update-draft": {
POST: course(async (req, _rc, user, courseId) => {
const b = req.headers.get("content-type")?.includes("json") ? await body<{ from?: string }>(req) : {};
const r = await store.updateDraft(courseId, who(user), b.from === "remote" ? "remote" : "published");
return json(r, r.ok ? 200 : r.reason === "no-remote" ? 400 : 409);
}),
},
"/api/studio/courses/:courseId/conflicts": {
GET: course(async (_req, _rc, user, courseId) => json({ files: await store.conflicts(courseId, user.id) })),
},
"/api/studio/courses/:courseId/resolve": {
POST: course(async (req, _rc, user, courseId) => {
const b = await body<{ files?: { path: string; content: string | null }[] }>(req);
if (!Array.isArray(b.files) || !b.files.length) return fail(400, "files are required");
return json(await store.resolve(courseId, who(user), b.files.map((f) => ({ path: String(f.path), content: f.content === null ? null : String(f.content) }))));
}),
},
"/api/studio/courses/:courseId/abort": {
POST: course(async (_req, _rc, user, courseId) => {
await store.abortMerge(courseId, user.id);
return json({ ok: true });
}),
},
"/api/studio/courses/:courseId/remote": {
PUT: course(async (req, _rc, _user, courseId) => {
const b = await body<{ url?: string | null }>(req);
await store.setRemote(courseId, b.url ? String(b.url) : null);
return json({ ok: true, remote: await store.remote(courseId) });
}),
},
"/api/studio/courses/:courseId/sync": {
POST: course(async (_req, _rc, user, courseId) => {
const r = await store.sync(courseId, who(user));
if (r.ok) {
await afterPublishedChange(courseId);
return json(r);
}
return json(r, r.reason === "no-remote" ? 400 : r.reason === "network" ? 502 : r.reason === "forbidden" ? 403 : 409);
}),
},
};
if (opts.watch !== false) {
watcher.check().catch((e) => console.error("[studio] watcher", e));
watcher.start();
}
return { name: "studio", routes, jobs: { [CONVERT_JOB]: convertJob }, multipartRoutes: ["/api/studio/courses/:courseId/upload"], store, watcher, draftWriter, ready, stop: () => watcher.stop() };
}