Files
STCourses/packages/feature-import/src/context.ts
T
Lucas WintherandClaude Opus 5.5 771d755da4 fix(studio): review round 1 for imported-course editing
- Media exclusions move from the editable root .gitignore to the repo's
  info/exclude, and dirtyFiles never counts files listed in media.json, so an
  author's or remote's .gitignore cannot block publish or Sync. Publish and
  Sync refuse commits that track a file over referenced media.
- Importer jobs (code-repo, pdf-deck, transcribe) and the cover route run
  under the course lock now shared with the studio (core withCourseLock) and
  re-check the course inside it; after conversion they fail with "this course
  is now edited in the studio". Convert answers 409 while an import job for
  the course is queued or running, and the job re-checks.
- Crash-safe conversion: the import folder is hidden (.converting-<id>) before
  the published worktree exists; cross-filesystem moves copy into a staging
  dir, fsync, verify size and sha256, then rename; the source is retired as
  .converted-<id>.<n>. Recovery runs at server start and before every convert;
  a re-run of a finished conversion reports success. The loader skips
  dot-folders.
- Minor: no raw NUL bytes in source, link fallback only on EXDEV/EPERM/EMLINK,
  referenced media are not read whole by the editor, missing media are logged,
  mediaDir JSDoc, docs on per-author copies across filesystems.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-30 20:53:50 +01:00

126 lines
6.5 KiB
TypeScript

import { mkdir, readdir, rename, stat } from "node:fs/promises";
import { join, relative, resolve } from "node:path";
import type { ServerContext } from "@stc/contracts";
import { withCourseLock } from "@stc/core";
import { readRawManifest, writeManifest, type RawManifest } from "./util/manifest";
import { slugify, uniqueId } from "./util/text";
import { extractFrame, type Tools } from "./media";
export interface ImportEnv { [k: string]: string | undefined }
/** `env` supplies only tool paths and transcription credentials (STC_FFMPEG, STC_FFPROBE, STC_TRANSCRIBE_KEY, _MODEL). */
export interface ImportOptions { env?: ImportEnv }
export interface Deps {
ctx: ServerContext;
env: ImportEnv;
tools: Tools;
/** AppConfig.python (STC_PYTHON). */
python: string;
/** AppConfig.importRoots (STC_IMPORT_ROOTS). */
importRoots: string[];
/** AppConfig.transcribeUrl (STC_TRANSCRIBE_URL). */
transcribeUrl: string | null;
}
export const coursesDir = (d: Deps) => resolve(d.ctx.config.importedCoursesDir ?? resolve(d.ctx.config.dataDir, "courses"));
export const cacheDir = (d: Deps) => resolve(d.ctx.config.cacheDir ?? resolve(d.ctx.config.dataDir, "cache"));
const studioRepo = (d: Deps, id: string) => join(d.ctx.config.studioReposDir ?? resolve(d.ctx.config.dataDir, "studio", "repos"), `${id}.git`);
export const deckCacheDir = (d: Deps, courseId: string, deckId: string) => join(cacheDir(d), "decks", courseId, deckId);
export const coverCacheDir = (d: Deps, courseId: string) => join(cacheDir(d), "covers", courseId);
export const transcriptCacheDir = (d: Deps, courseId: string) => join(cacheDir(d), "transcripts", courseId);
export const isPlainId = (s: string) => /^[a-z0-9][a-z0-9-]*$/.test(s);
/** A fresh course id from a title, unique among managed folders and every course the server already knows. */
export async function freshCourseId(d: Deps, wanted: string | undefined, title: string): Promise<string> {
if (wanted !== undefined) {
if (!isPlainId(wanted)) throw new Error(`invalid courseId "${wanted}" (use lowercase letters, digits and dashes)`);
// Studio repos share the id space (ruling R9): refuse an id the studio already uses, even before it is published.
if (await exists(join(coursesDir(d), wanted)) || await exists(studioRepo(d, wanted)) || d.ctx.courses.get(wanted)) throw new Error(`a course with id "${wanted}" already exists`);
return wanted;
}
const taken = new Set<string>(d.ctx.courses.list().map((c) => c.id));
for (const e of await readdir(coursesDir(d)).catch(() => [] as string[])) taken.add(e);
const repos = d.ctx.config.studioReposDir ?? resolve(d.ctx.config.dataDir, "studio", "repos");
for (const e of await readdir(repos).catch(() => [] as string[])) if (e.endsWith(".git")) taken.add(e.slice(0, -4));
return uniqueId(slugify(title), (x) => taken.has(x));
}
export async function exists(p: string): Promise<boolean> {
return stat(p).then(() => true, () => false);
}
/** Scratch folder on the same filesystem as the courses, so finished work is moved in with one rename. */
export async function stageDir(d: Deps, label: string): Promise<string> {
const dir = resolve(d.ctx.config.dataDir, "tmp", `${label}-${Bun.randomUUIDv7()}`);
await mkdir(dir, { recursive: true });
return dir;
}
export async function publishCourse(d: Deps, stage: string, id: string): Promise<string> {
await mkdir(coursesDir(d), { recursive: true });
const dest = join(coursesDir(d), id);
await rename(stage, dest);
return dest;
}
export interface ManagedCourse { id: string; dir: string; manifest: RawManifest }
export const STUDIO_OWNED = "this course is now edited in the studio; add files there instead";
/**
* Runs importer work that changes an existing course under the course lock shared with the studio (core
* `withCourseLock`), after checking, inside the lock, that the course is still an importer folder. A conversion to a
* studio course therefore either waits for the job, or the job fails cleanly with STUDIO_OWNED and writes nothing.
*/
export function withManagedCourse<T>(d: Deps, courseId: string | undefined, fn: (c: ManagedCourse) => Promise<T>): Promise<T> {
return withCourseLock(courseId ?? "", async () => fn(await managedCourse(d, courseId)));
}
/** An existing course the importer may add to. Only folders directly under <dataDir>/courses qualify: overlays and authored repos are never edited. */
export async function managedCourse(d: Deps, courseId: string | undefined): Promise<ManagedCourse> {
if (!courseId) throw new Error("options.courseId is required: choose the course to add to");
if (isPlainId(courseId) && (await exists(studioRepo(d, courseId)))) throw new Error(STUDIO_OWNED);
const dir = join(coursesDir(d), courseId);
const rel = relative(coursesDir(d), dir);
if (!isPlainId(courseId) || rel.startsWith("..") || !(await exists(join(dir, "stcourse.yaml")))) {
throw new Error(`course "${courseId}" is not an imported course under ${coursesDir(d)}; the importer only edits courses it created`);
}
const manifest = await readRawManifest(dir);
if (manifest.id !== courseId) throw new Error(`course folder "${courseId}" has manifest id "${manifest.id}"`);
return { id: courseId, dir, manifest };
}
/** Reloads the repository and turns load problems into warnings. */
export async function finish(d: Deps, id: string, warnings: string[]): Promise<void> {
await d.ctx.courses.reload(id);
const c = d.ctx.courses.get(id);
if (!c) {
warnings.push(`course "${id}" is not visible to the server: add ${coursesDir(d)} to STC_COURSE_DIRS`);
return;
}
for (const diag of c.diagnostics) warnings.push(`${diag.level}: ${diag.message}${diag.path ? ` (${diag.path})` : ""}`);
}
export async function saveManifest(dir: string, manifest: RawManifest): Promise<void> {
await writeManifest(dir, manifest);
}
export function sourceBlock(kind: string, origin: string) {
return { kind, origin, importedAt: new Date().toISOString() };
}
/** Writes up to `count` frame candidates from evenly spaced clips; returns the candidate file names. Failures are skipped. */
export async function frameCandidates(d: Deps, courseId: string, clips: string[], count = 3): Promise<string[]> {
const out: string[] = [];
const n = Math.min(count, clips.length);
for (let i = 0; i < n; i++) {
const name = `frame-${i + 1}.jpg`;
try {
await extractFrame(d.tools, clips[Math.floor((i * clips.length) / n)]!, join(coverCacheDir(d, courseId), name), 0.2);
out.push(name);
} catch { /* a bad clip must not fail the import */ }
}
return out;
}