feat(studio): convert an imported course into a studio repo, media kept outside git
Small files are committed unchanged (visibility and source.kind stay); large media are referenced from .stcourses/media.json, ignored, and hard-linked into every worktree from <dataDir>/media/<id>, where the import folder is moved. Studio writes to referenced media are refused and never write through a shared inode. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Lucas Winther
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
60179407f4
commit
df6d235070
@@ -0,0 +1,64 @@
|
||||
import { copyFile, link, mkdir, readdir, rename, rm, cp, stat } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { $ } from "bun";
|
||||
import { gitignoreLine, isLargeMedia } from "./media-refs";
|
||||
import { checkRelPath } from "./relpath";
|
||||
|
||||
export interface ScannedImport {
|
||||
/** Committed into git. */
|
||||
small: string[];
|
||||
/** Kept out of git, referenced by path, size and checksum. */
|
||||
large: { path: string; size: number }[];
|
||||
/** Skipped and never followed: symlinks, sockets, `.git` folders. */
|
||||
skipped: string[];
|
||||
}
|
||||
|
||||
/** Walks an import folder with lstat semantics: symlinks and special files are skipped, never followed. */
|
||||
export async function scanImport(root: string): Promise<ScannedImport> {
|
||||
const out: ScannedImport = { small: [], large: [], skipped: [] };
|
||||
const walk = async (rel: string) => {
|
||||
const entries = await readdir(join(root, rel), { withFileTypes: true });
|
||||
entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
||||
for (const e of entries) {
|
||||
const r = rel ? `${rel}/${e.name}` : e.name;
|
||||
if (e.name.toLowerCase() === ".git") { out.skipped.push(r); continue; }
|
||||
if (e.isDirectory()) await walk(r);
|
||||
else if (e.isFile()) {
|
||||
const size = (await stat(join(root, r))).size;
|
||||
if (isLargeMedia(r, size)) out.large.push({ path: r, size });
|
||||
else out.small.push(r);
|
||||
} else out.skipped.push(r);
|
||||
}
|
||||
};
|
||||
await walk("");
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Problems that make a large file impossible to reference: a name git ignore files or the studio cannot express. */
|
||||
export function unreferencable(large: { path: string }[]): string[] {
|
||||
const bad: string[] = [];
|
||||
for (const f of large) {
|
||||
try { gitignoreLine(f.path); if (checkRelPath(f.path) !== f.path) throw new Error("not normal"); } catch { bad.push(f.path); }
|
||||
}
|
||||
return bad;
|
||||
}
|
||||
|
||||
/** A hard link (no extra space), else a reflink or plain copy (another filesystem). The source is only read. */
|
||||
export async function linkOrCopy(src: string, dest: string): Promise<void> {
|
||||
await mkdir(dirname(dest), { recursive: true });
|
||||
try { await link(src, dest); return; } catch { /* cross-device or unsupported */ }
|
||||
const r = await $`cp --reflink=auto -- ${src} ${dest}`.quiet().nothrow();
|
||||
if (r.exitCode !== 0) await copyFile(src, dest);
|
||||
}
|
||||
|
||||
/** Moves the import folder into the media store; across filesystems it copies (keeping links) and then removes. */
|
||||
export async function moveDir(src: string, dest: string): Promise<void> {
|
||||
await mkdir(dirname(dest), { recursive: true });
|
||||
try {
|
||||
await rename(src, dest);
|
||||
} catch (e) {
|
||||
if ((e as NodeJS.ErrnoException).code !== "EXDEV") throw e;
|
||||
await cp(src, dest, { recursive: true, verbatimSymlinks: true, errorOnExist: true, force: false });
|
||||
await rm(src, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,10 @@
|
||||
import { constants, existsSync, lstatSync, mkdirSync, readdirSync, realpathSync, rmSync, statSync } from "node:fs";
|
||||
import { mkdir, open, rm, unlink } from "node:fs/promises";
|
||||
import { constants, existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs";
|
||||
import { copyFile, mkdir, open, rm, unlink, writeFile } from "node:fs/promises";
|
||||
import { dirname, join, sep } from "node:path";
|
||||
import { loadCourse, type Diagnostic } from "@stc/course-format";
|
||||
import { loadCourse, parseManifest, type Diagnostic } from "@stc/course-format";
|
||||
import { linkOrCopy, moveDir, scanImport, unreferencable } from "./convert";
|
||||
import { git, gitOk, splitZ, withLock, type Identity } from "./git";
|
||||
import { MEDIA_REFS, mediaGitignore, parseMediaRefs, sha256File, stringifyMediaRefs, type MediaRef } from "./media-refs";
|
||||
import { checkRelPath } from "./relpath";
|
||||
|
||||
export { checkRelPath } from "./relpath";
|
||||
@@ -68,6 +70,20 @@ export type SyncResult =
|
||||
| { ok: false; reason: "conflict"; conflicts: string[] }
|
||||
| { ok: false; reason: "forbidden"; message: string };
|
||||
|
||||
export interface ConvertResult {
|
||||
commit: string;
|
||||
/** Files kept out of git and referenced from MEDIA_REFS. */
|
||||
media: number;
|
||||
/** Files committed. */
|
||||
files: number;
|
||||
/** Paths skipped in the import folder (symlinks, special files, .git folders). */
|
||||
skipped: string[];
|
||||
/** True when an interrupted conversion was only finished (the repo already existed). */
|
||||
resumed: boolean;
|
||||
}
|
||||
|
||||
export type ConvertProgress = (fraction: number, message: string) => void | Promise<void>;
|
||||
|
||||
export class StudioError extends Error {
|
||||
constructor(message: string, public status = 400) { super(message); }
|
||||
}
|
||||
@@ -121,6 +137,8 @@ export interface StudioStoreOptions {
|
||||
importedDir?: string;
|
||||
/** AppConfig.studioWorktreesDir (default `<dataDir>/worktrees`). */
|
||||
worktreesDir?: string;
|
||||
/** Media stores of converted imports (default `<dataDir>/media`): `<mediaDir>/<id>` is the former import folder. */
|
||||
mediaDir?: string;
|
||||
publishedBranch?: string;
|
||||
/** Tests only: allow local-path remotes (the HTTP feature never sets this). */
|
||||
allowLocalRemotes?: boolean;
|
||||
@@ -133,6 +151,7 @@ export function createStudioStore(opts: StudioStoreOptions) {
|
||||
const reposDir = opts.reposDir ?? join(dataDir, "studio", "repos");
|
||||
const importedDir = opts.importedDir ?? join(dataDir, "courses");
|
||||
const worktreesDir = opts.worktreesDir ?? join(dataDir, "worktrees");
|
||||
const mediaDir = opts.mediaDir ?? join(dataDir, "media");
|
||||
const remoteGuard = opts.allowLocalRemotes ? ["-c", "protocol.ext.allow=never", "-c", "protocol.fd.allow=never"] : REMOTE_GUARD;
|
||||
|
||||
const paths = (courseId: string) => {
|
||||
@@ -144,6 +163,8 @@ export function createStudioStore(opts: StudioStoreOptions) {
|
||||
published: join(root, "published"),
|
||||
draftsDir: join(root, "drafts"),
|
||||
draft: (authorId: string) => join(root, "drafts", safeAuthor(authorId)),
|
||||
media: join(mediaDir, courseId),
|
||||
imported: join(importedDir, courseId),
|
||||
};
|
||||
};
|
||||
const branchOf = (authorId: string) => `draft/${safeAuthor(authorId)}`;
|
||||
@@ -233,9 +254,53 @@ export function createStudioStore(opts: StudioStoreOptions) {
|
||||
const branch = branchOf(authorId);
|
||||
const exists = (await git(p.bare, ["rev-parse", "-q", "--verify", `refs/heads/${branch}`])).code === 0;
|
||||
await gitOk(p.bare, exists ? ["worktree", "add", dir, branch] : ["worktree", "add", "-b", branch, dir, main]);
|
||||
await materializeMedia(dir, p.media);
|
||||
return dir;
|
||||
}
|
||||
|
||||
/** The draft's or worktree's media references (empty for courses that never were imports). */
|
||||
async function mediaRefsIn(dir: string): Promise<MediaRef[]> {
|
||||
const f = await readFileIn(dir, MEDIA_REFS);
|
||||
return f ? parseMediaRefs(f.content) ?? [] : [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Puts every referenced file into `dir` as a hard link to `fromRoot/<path>` (a copy across filesystems).
|
||||
* Idempotent; existing destinations are left alone. A reference must name a regular file whose real path is inside
|
||||
* `fromRoot`, and the destination path may not contain a symlink. Returns the references it could not satisfy.
|
||||
*/
|
||||
async function materializeMedia(dir: string, fromRoot: string): Promise<{ linked: number; missing: string[] }> {
|
||||
const refs = await mediaRefsIn(dir);
|
||||
const missing: string[] = [];
|
||||
let linked = 0;
|
||||
let realFrom: string;
|
||||
try { realFrom = realpathSync(fromRoot); } catch { return { linked, missing: refs.map((r) => r.path) }; }
|
||||
for (const r of refs) {
|
||||
try {
|
||||
const src = join(realFrom, r.path);
|
||||
const st = lstatSync(src);
|
||||
const real = realpathSync(src);
|
||||
if (!st.isFile() || !real.startsWith(realFrom + sep)) throw new Error("not a regular file inside the media store");
|
||||
const dest = await safeWritePath(dir, r.path);
|
||||
if (lexists(dest)) continue;
|
||||
await linkOrCopy(real, dest);
|
||||
linked++;
|
||||
} catch {
|
||||
missing.push(r.path);
|
||||
}
|
||||
}
|
||||
return { linked, missing };
|
||||
}
|
||||
|
||||
/** Referenced media and the reference file are not editable in the studio (they are not in git). */
|
||||
async function refuseMediaWrites(dir: string, paths: string[]): Promise<void> {
|
||||
const locked = new Set([MEDIA_REFS, ...(await mediaRefsIn(dir)).map((r) => r.path)]);
|
||||
const hit = paths.map((x) => checkRelPath(x)).filter((x) => locked.has(x));
|
||||
if (hit.length) {
|
||||
throw new StudioError(`${hit.join(", ")}: imported media are kept outside git and cannot be changed in the studio; upload a file under a new name instead`, 409);
|
||||
}
|
||||
}
|
||||
|
||||
async function readFileIn(dir: string, rel: string): Promise<{ content: string; hash: string } | null> {
|
||||
let abs: string;
|
||||
try { abs = inside(dir, rel); } catch { return null; }
|
||||
@@ -260,6 +325,8 @@ export function createStudioStore(opts: StudioStoreOptions) {
|
||||
if (lexists(abs)) await unlink(abs);
|
||||
} else {
|
||||
const abs = await safeWritePath(dir, f.path);
|
||||
// never write into an inode shared with another path (imported media, an owner's original): replace the file
|
||||
if (lexists(abs) && lstatSync(abs).nlink > 1) await unlink(abs);
|
||||
// O_NOFOLLOW: a symlink swapped in after the check makes the open fail instead of writing through it
|
||||
const fh = await open(abs, constants.O_WRONLY | constants.O_CREAT | constants.O_TRUNC | constants.O_NOFOLLOW, 0o644);
|
||||
try { await fh.writeFile(f.content ?? ""); } finally { await fh.close(); }
|
||||
@@ -315,12 +382,130 @@ export function createStudioStore(opts: StudioStoreOptions) {
|
||||
});
|
||||
},
|
||||
|
||||
/** An importer folder `<importedDir>/<id>` holding an imported (not authored) course the owner may convert. */
|
||||
isConvertibleImport(courseId: string): boolean {
|
||||
if (!ID_RE.test(courseId) || isManaged(courseId)) return false;
|
||||
const p = paths(courseId);
|
||||
try {
|
||||
if (!lstatSync(p.imported).isDirectory()) return false;
|
||||
const m = parseLoose(readFileSync(join(p.imported, MANIFEST), "utf8"));
|
||||
return !!m?.source?.kind && m.source.kind !== "authored";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
},
|
||||
|
||||
/** Re-links referenced media into the published worktree (after an outside checkout removed them, say). */
|
||||
repairMedia: (courseId: string) => lock(courseId, async () => {
|
||||
const p = requireManaged(courseId);
|
||||
return materializeMedia(p.published, p.media);
|
||||
}),
|
||||
|
||||
/**
|
||||
* Turns the imported course `<importedDir>/<id>` into a studio course with the same id (v1.1 S, ruling R9):
|
||||
* small files are committed as they are (manifest unchanged: visibility and source.kind stay), large media are
|
||||
* referenced from MEDIA_REFS and hard-linked into each worktree, and the import folder moves to `<mediaDir>/<id>`.
|
||||
* Owner-only at the HTTP layer. Restart-safe: leftovers of an attempt that never produced a published worktree are
|
||||
* removed and the conversion redone; a converted course whose import folder is still there only finishes the move.
|
||||
*/
|
||||
convertImported(courseId: string, who: Identity, opts: { onProgress?: ConvertProgress } = {}): Promise<ConvertResult> {
|
||||
if (!ID_RE.test(courseId)) return Promise.reject(new StudioError(`invalid course id: ${courseId}`, 404));
|
||||
const progress = opts.onProgress ?? (() => {});
|
||||
return lock(courseId, async () => {
|
||||
const p = paths(courseId);
|
||||
const srcManifest = join(p.imported, MANIFEST);
|
||||
const hasImport = lexists(srcManifest);
|
||||
if (isManaged(courseId)) {
|
||||
const refs = await readFileIn(p.published, MEDIA_REFS);
|
||||
if (!hasImport || !refs) throw new StudioError(`course "${courseId}" is already a studio course`, 409);
|
||||
if (lexists(p.media)) throw new StudioError(`the media store ${p.media} already exists`, 409);
|
||||
await moveDir(p.imported, p.media);
|
||||
await materializeMedia(p.published, p.media);
|
||||
return { commit: await rev(p.published, "HEAD"), media: (parseMediaRefs(refs.content) ?? []).length, files: 0, skipped: [], resumed: true };
|
||||
}
|
||||
if (!hasImport) throw new StudioError(`there is no imported course "${courseId}" in ${importedDir}`, 404);
|
||||
const top = lstatSync(p.imported);
|
||||
if (top.isSymbolicLink() || !top.isDirectory() || lstatSync(srcManifest).isSymbolicLink()) throw new StudioError("the import folder and its manifest must not be symlinks", 400);
|
||||
if (lexists(p.media)) throw new StudioError(`the media store ${p.media} already exists`, 409);
|
||||
const parsed = parseManifest(await Bun.file(srcManifest).text());
|
||||
if (!parsed.ok) throw new StudioError(`the imported manifest is invalid: ${parsed.errors.join("; ")}`, 400);
|
||||
const m = parsed.manifest;
|
||||
if (m.id !== courseId) throw new StudioError(`the imported manifest has id "${m.id}", not "${courseId}"`, 400);
|
||||
if (m.root) throw new StudioError("an imported course with root: cannot become a studio course", 400);
|
||||
if (!m.source?.kind || m.source.kind === "authored") throw new StudioError("only imported courses can be made editable this way", 400);
|
||||
|
||||
await progress(0.02, "Reading the import folder");
|
||||
const scan = await scanImport(p.imported);
|
||||
const bad = unreferencable(scan.large);
|
||||
if (bad.length) throw new StudioError(`these large files have names the studio cannot keep outside git; rename them first: ${bad.map((b) => JSON.stringify(b)).join(", ")}`, 400);
|
||||
|
||||
// A previous attempt that never got as far as the published worktree: start again.
|
||||
rmSync(p.bare, { recursive: true, force: true });
|
||||
rmSync(p.root, { recursive: true, force: true });
|
||||
|
||||
const total = scan.large.reduce((n, f) => n + f.size, 0) || 1;
|
||||
let done = 0;
|
||||
const refs: MediaRef[] = [];
|
||||
for (const f of scan.large) {
|
||||
await progress(0.05 + 0.75 * (done / total), `Checksumming ${f.path}`);
|
||||
refs.push({ path: f.path, size: f.size, sha256: await sha256File(join(p.imported, f.path)) });
|
||||
done += f.size;
|
||||
}
|
||||
|
||||
const work = join(dataDir, "tmp", `convert-${courseId}-${Bun.randomUUIDv7()}`);
|
||||
const stage = join(work, "tree");
|
||||
try {
|
||||
await progress(0.82, "Committing the course files");
|
||||
for (const f of scan.small) {
|
||||
await mkdir(dirname(join(stage, f)), { recursive: true });
|
||||
await copyFile(join(p.imported, f), join(stage, f));
|
||||
}
|
||||
const ownIgnore = scan.small.includes(".gitignore") ? await Bun.file(join(stage, ".gitignore")).text() : null;
|
||||
await mkdir(join(stage, dirname(MEDIA_REFS)), { recursive: true });
|
||||
await writeFile(join(stage, MEDIA_REFS), stringifyMediaRefs(refs));
|
||||
await writeFile(join(stage, ".gitignore"), mediaGitignore(refs.map((r) => r.path), ownIgnore));
|
||||
const list = [...new Set([...scan.small, ".gitignore", MEDIA_REFS])];
|
||||
await writeFile(join(work, "paths"), list.join(" ") + " ");
|
||||
|
||||
mkdirSync(reposDir, { recursive: true });
|
||||
await gitOk(dataDir, ["init", "-q", "--bare", "-b", main, p.bare]);
|
||||
const gd = ["--literal-pathspecs", "--git-dir", p.bare, "--work-tree", stage];
|
||||
// -f: a nested .gitignore from the course's own code must not drop course files
|
||||
await gitOk(stage, [...gd, "add", "-f", `--pathspec-from-file=${join(work, "paths")}`, "--pathspec-file-nul"]);
|
||||
await gitOk(stage, [...gd, "commit", "-q", "-m", `Make ${courseId} editable in the studio
|
||||
|
||||
Converted from the import in ${p.imported}. ${refs.length} large media files stay outside git (${MEDIA_REFS}).`], who);
|
||||
mkdirSync(p.root, { recursive: true });
|
||||
await gitOk(p.bare, ["worktree", "add", p.published, main]);
|
||||
|
||||
await progress(0.92, "Linking media into the published course");
|
||||
const mat = await materializeMedia(p.published, p.imported);
|
||||
if (mat.missing.length) throw new Error(`could not link media: ${mat.missing.join(", ")}`);
|
||||
const dirty = await dirtyFiles(p.published);
|
||||
if (dirty.length) throw new Error(`the converted course is not clean after checkout: ${dirty.join(", ")}`);
|
||||
|
||||
await progress(0.97, "Moving the import folder to the media store");
|
||||
await moveDir(p.imported, p.media);
|
||||
return { commit: await rev(p.published, "HEAD"), media: refs.length, files: list.length, skipped: scan.skipped, resumed: false };
|
||||
} catch (e) {
|
||||
if (lexists(p.imported)) {
|
||||
rmSync(p.bare, { recursive: true, force: true });
|
||||
rmSync(p.root, { recursive: true, force: true });
|
||||
}
|
||||
throw e;
|
||||
} finally {
|
||||
await rm(work, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
ensureDraft: (courseId: string, authorId: string) => lock(courseId, () => ensureDraftUnlocked(courseId, authorId)),
|
||||
|
||||
async listFiles(courseId: string, authorId: string): Promise<string[]> {
|
||||
const dir = await api.ensureDraft(courseId, authorId);
|
||||
const tracked = splitZ(await gitOk(dir, ["ls-files", "-z", "--cached", "--others", "--exclude-standard"]));
|
||||
return [...new Set(tracked)].filter((f) => existsSync(join(dir, f))).sort();
|
||||
const media = (await mediaRefsIn(dir)).map((r) => r.path);
|
||||
return [...new Set([...tracked, ...media])].filter((f) => existsSync(join(dir, f))).sort();
|
||||
},
|
||||
|
||||
async readFile(courseId: string, authorId: string, rel: string) {
|
||||
@@ -340,6 +525,7 @@ export function createStudioStore(opts: StudioStoreOptions) {
|
||||
return lock(courseId, async () => {
|
||||
const dir = await ensureDraftUnlocked(courseId, who.id);
|
||||
if (await isMerging(dir)) throw new StudioError("the draft has a merge in progress: resolve or abort it first", 409);
|
||||
await refuseMediaWrites(dir, files.map((f) => f.path));
|
||||
const conflicts: FileConflict[] = [];
|
||||
for (const f of files) {
|
||||
if (f.baseHash === undefined) continue;
|
||||
@@ -432,6 +618,7 @@ export function createStudioStore(opts: StudioStoreOptions) {
|
||||
return { ok: false, reason: "invalid", diagnostics: mergedErrors };
|
||||
}
|
||||
await gitOk(p.published, ["commit", "-q", "-m", `Publish ${branch}`], who);
|
||||
await materializeMedia(p.published, p.media);
|
||||
// Bring the draft level with what was just published, so "behind" stays meaningful.
|
||||
await git(dir, ["merge", "-q", "--ff-only", main]);
|
||||
return { ok: true, commit: await rev(p.published, "HEAD"), published: true };
|
||||
@@ -493,6 +680,7 @@ export function createStudioStore(opts: StudioStoreOptions) {
|
||||
return lock(courseId, async () => {
|
||||
const dir = await ensureDraftUnlocked(courseId, who.id);
|
||||
if (!(await isMerging(dir))) throw new StudioError("no merge in progress", 409);
|
||||
await refuseMediaWrites(dir, files.map((f) => f.path));
|
||||
const m = files.find((f) => checkRelPath(f.path) === MANIFEST);
|
||||
if (m && !who.owner) {
|
||||
const needOwner = ownerRequiredFor(await showText(requireManaged(courseId).bare, `${main}:${MANIFEST}`), m.content);
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { loadCourse } from "@stc/course-format";
|
||||
import { MEDIA_REFS, parseMediaRefs } from "../src/server/media-refs";
|
||||
import { BIG_PDF, makeImportedCourse, VIDEO, VIDEO2, videoBytes } from "./import-fixture";
|
||||
import { ada, extGit, newStore } from "./helpers";
|
||||
|
||||
const ino = (p: string) => statSync(p).ino;
|
||||
const tracked = async (dir: string) => (await extGit(dir, "-c", "core.quotepath=false", "ls-files")).split("\n").filter(Boolean).sort();
|
||||
|
||||
describe("convertImported", () => {
|
||||
test("builds the repo and published worktree; large media stay out of git as hard links", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
const src = makeImportedCourse(dataDir, { hostile: true });
|
||||
const srcVideoIno = ino(join(src, VIDEO));
|
||||
const r = await store.convertImported("imp", ada);
|
||||
expect(r.media).toBe(3);
|
||||
expect(store.isManaged("imp")).toBe(true);
|
||||
expect(existsSync(join(dataDir, "studio/repos/imp.git/HEAD"))).toBe(true);
|
||||
expect(existsSync(src)).toBe(false);
|
||||
const mediaRoot = join(dataDir, "media/imp");
|
||||
expect(existsSync(join(mediaRoot, "stcourse.yaml"))).toBe(true);
|
||||
|
||||
const pub = store.paths("imp").published;
|
||||
expect(await tracked(pub)).toEqual([
|
||||
".gitignore", MEDIA_REFS, "code/S01 - Basics/.gitignore", "code/S01 - Basics/README.md", "media/cover.jpg", "stcourse.yaml", "transcripts/l1.md",
|
||||
]);
|
||||
// the symlink was skipped, never followed; .git contents never copied
|
||||
expect(existsSync(join(pub, "code/leak.txt"))).toBe(false);
|
||||
expect(existsSync(join(pub, "code/.git"))).toBe(false);
|
||||
expect(await extGit(pub, "status", "--porcelain", "--untracked-files=all")).toBe("");
|
||||
|
||||
const refs = parseMediaRefs(readFileSync(join(pub, MEDIA_REFS), "utf8"))!;
|
||||
expect(refs.map((x) => x.path).sort()).toEqual([BIG_PDF, VIDEO2, VIDEO].sort());
|
||||
const v = refs.find((x) => x.path === VIDEO)!;
|
||||
expect(v.size).toBe(4096);
|
||||
expect(v.sha256).toBe(new Bun.CryptoHasher("sha256").update(videoBytes(4096, 7)).digest("hex"));
|
||||
for (const p of [VIDEO, VIDEO2, BIG_PDF]) expect(ino(join(pub, p))).toBe(ino(join(mediaRoot, p)));
|
||||
expect(ino(join(mediaRoot, VIDEO))).toBe(srcVideoIno);
|
||||
|
||||
// the course loads from the published worktree exactly as before
|
||||
const c = await loadCourse(join(pub, "stcourse.yaml"));
|
||||
expect(c.status).toBe("ok");
|
||||
expect(c.manifest?.visibility).toBe("restricted");
|
||||
expect(c.manifest?.source?.kind).toBe("video-folder");
|
||||
// the commit is attributed to the converting owner
|
||||
expect(await extGit(pub, "log", "-1", "--format=%an <%ae>")).toBe(`${ada.name} <${ada.email}>`);
|
||||
});
|
||||
|
||||
test("a draft gets the media; listFiles shows them; the media and reference file are read-only", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
makeImportedCourse(dataDir);
|
||||
await store.convertImported("imp", ada);
|
||||
const draft = await store.ensureDraft("imp", ada.id);
|
||||
expect(ino(join(draft, VIDEO))).toBe(ino(join(dataDir, "media/imp", VIDEO)));
|
||||
const files = await store.listFiles("imp", ada.id);
|
||||
expect(files).toContain(VIDEO);
|
||||
expect(files).toContain(BIG_PDF);
|
||||
for (const path of [VIDEO, MEDIA_REFS]) {
|
||||
await expect(store.saveFiles("imp", ada, [{ path, content: "x" }], "x")).rejects.toMatchObject({ status: 409 });
|
||||
await expect(store.saveFiles("imp", ada, [{ path, delete: true }], "x")).rejects.toMatchObject({ status: 409 });
|
||||
}
|
||||
expect(readFileSync(join(dataDir, "media/imp", VIDEO))).toEqual(Buffer.from(videoBytes(4096, 7)));
|
||||
});
|
||||
|
||||
test("a studio write to a hard-linked file breaks the link instead of writing through it", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
makeImportedCourse(dataDir);
|
||||
await store.convertImported("imp", ada);
|
||||
const draft = await store.ensureDraft("imp", ada.id);
|
||||
// a planted hard link at an ordinary (non-referenced) path
|
||||
const outside = join(dataDir, "media/imp/transcripts/l1.md");
|
||||
rmSync(join(draft, "transcripts/l1.md"));
|
||||
const { linkSync } = await import("node:fs");
|
||||
linkSync(outside, join(draft, "transcripts/l1.md"));
|
||||
expect((await store.saveFiles("imp", ada, [{ path: "transcripts/l1.md", content: "# changed\n" }], "edit")).ok).toBe(true);
|
||||
expect(readFileSync(outside, "utf8")).toBe("# Intro\n\n[0:00] Hello.\n");
|
||||
expect(readFileSync(join(draft, "transcripts/l1.md"), "utf8")).toBe("# changed\n");
|
||||
});
|
||||
|
||||
test("edit and publish: the published worktree keeps its media and stays clean", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
makeImportedCourse(dataDir);
|
||||
await store.convertImported("imp", ada);
|
||||
const m = (await store.readFile("imp", ada.id, "stcourse.yaml"))!;
|
||||
const next = m.content.replace("title: Intro", "title: Welcome aboard");
|
||||
expect((await store.saveFiles("imp", ada, [{ path: "stcourse.yaml", content: next, baseHash: m.hash }], "Rename lesson")).ok).toBe(true);
|
||||
const r = await store.publish("imp", ada);
|
||||
expect(r).toMatchObject({ ok: true, published: true });
|
||||
const pub = store.paths("imp").published;
|
||||
expect(readFileSync(join(pub, "stcourse.yaml"), "utf8")).toContain("title: Welcome aboard");
|
||||
expect(ino(join(pub, VIDEO))).toBe(ino(join(dataDir, "media/imp", VIDEO)));
|
||||
expect(await extGit(pub, "status", "--porcelain", "--untracked-files=all")).toBe("");
|
||||
});
|
||||
|
||||
test("a non-owner cannot publish the converted course as public; the owner can", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
makeImportedCourse(dataDir);
|
||||
await store.convertImported("imp", ada);
|
||||
const m = (await store.readFile("imp", ada.id, "stcourse.yaml"))!;
|
||||
await store.saveFiles("imp", ada, [{ path: "stcourse.yaml", content: m.content.replace("visibility: restricted", "visibility: public"), baseHash: m.hash }], "go public");
|
||||
expect(await store.publish("imp", ada)).toMatchObject({ ok: false, reason: "forbidden" });
|
||||
expect(await store.publish("imp", { ...ada, owner: true })).toMatchObject({ ok: true, published: true });
|
||||
});
|
||||
|
||||
describe("refusals leave the import untouched and no repo behind", () => {
|
||||
const cases: [string, (dataDir: string) => void, number][] = [
|
||||
["no such import", () => {}, 404],
|
||||
["an authored course", (d) => makeImportedCourse(d, { kind: "authored" }), 400],
|
||||
["a manifest id mismatch", (d) => makeImportedCourse(d, { manifestId: "other" }), 400],
|
||||
["a root: field", (d) => makeImportedCourse(d, { root: "." }), 400],
|
||||
["a media store already there", (d) => { makeImportedCourse(d); mkdirSync(join(d, "media/imp"), { recursive: true }); }, 409],
|
||||
];
|
||||
test.each(cases)("%s", async (_name, prepare, status) => {
|
||||
const { dataDir, store } = await newStore();
|
||||
prepare(dataDir);
|
||||
const before = existsSync(join(dataDir, "courses/imp/stcourse.yaml")) ? readFileSync(join(dataDir, "courses/imp/stcourse.yaml"), "utf8") : null;
|
||||
await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status });
|
||||
expect(existsSync(join(dataDir, "studio/repos/imp.git"))).toBe(false);
|
||||
expect(existsSync(join(dataDir, "worktrees/imp"))).toBe(false);
|
||||
if (before !== null) expect(readFileSync(join(dataDir, "courses/imp/stcourse.yaml"), "utf8")).toBe(before);
|
||||
});
|
||||
|
||||
test("a studio course with the same id", async () => {
|
||||
const { store } = await newStore();
|
||||
await store.createCourse({ id: "imp", title: "Studio" }, ada);
|
||||
await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status: 409 });
|
||||
expect(store.isManaged("imp")).toBe(true);
|
||||
});
|
||||
|
||||
test("a large file whose name has a line break", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
const src = makeImportedCourse(dataDir);
|
||||
writeFileSync(join(src, "media/bad\nname.mp4"), "x");
|
||||
await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status: 400 });
|
||||
expect(existsSync(join(dataDir, "studio/repos/imp.git"))).toBe(false);
|
||||
expect(existsSync(join(src, "stcourse.yaml"))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("restart safety", () => {
|
||||
test("a leftover partial repo (no published worktree) is rebuilt", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
makeImportedCourse(dataDir);
|
||||
mkdirSync(join(dataDir, "studio/repos/imp.git"), { recursive: true });
|
||||
writeFileSync(join(dataDir, "studio/repos/imp.git/junk"), "x");
|
||||
await store.convertImported("imp", ada);
|
||||
expect(store.isManaged("imp")).toBe(true);
|
||||
expect(existsSync(join(dataDir, "studio/repos/imp.git/junk"))).toBe(false);
|
||||
});
|
||||
|
||||
test("a managed course whose import folder is still there only finishes the move", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
makeImportedCourse(dataDir);
|
||||
await store.convertImported("imp", ada);
|
||||
const head = await store.publishedHead("imp");
|
||||
// simulate a crash between the checkout and the move
|
||||
const { renameSync } = await import("node:fs");
|
||||
renameSync(join(dataDir, "media/imp"), join(dataDir, "courses/imp"));
|
||||
const r = await store.convertImported("imp", ada);
|
||||
expect(r.resumed).toBe(true);
|
||||
expect(await store.publishedHead("imp")).toBe(head);
|
||||
expect(existsSync(join(dataDir, "courses/imp"))).toBe(false);
|
||||
expect(existsSync(join(dataDir, "media/imp", VIDEO))).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,86 @@
|
||||
// A synthetic imported course, laid out as the importer leaves it in <dataDir>/courses/<id>. Nothing here is real material.
|
||||
import { mkdirSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { LARGE_FILE_BYTES } from "../src/server/media-refs";
|
||||
import { tempDir } from "./helpers";
|
||||
|
||||
export const VIDEO = "media/01 Intro – Übersicht.mp4";
|
||||
export const VIDEO2 = "media/[2] Next step.mp4";
|
||||
export const BIG_PDF = "decks/big deck.pdf";
|
||||
/** Fake MP4 bytes: the server only streams them, nothing decodes them. */
|
||||
export const videoBytes = (n = 4096, seed = 7) => new Uint8Array(n).map((_, i) => (i * seed) % 256);
|
||||
|
||||
export interface ImportFixtureOpts {
|
||||
id?: string;
|
||||
kind?: string;
|
||||
manifestId?: string;
|
||||
root?: string;
|
||||
visibility?: string;
|
||||
/** Extra hostile bits: a symlink to outside, a .git dir, a nested .gitignore that hides a lesson. */
|
||||
hostile?: boolean;
|
||||
}
|
||||
|
||||
function put(dir: string, rel: string, content: string | Uint8Array) {
|
||||
mkdirSync(dirname(join(dir, rel)), { recursive: true });
|
||||
writeFileSync(join(dir, rel), content);
|
||||
}
|
||||
|
||||
export function manifestYaml(o: ImportFixtureOpts = {}): string {
|
||||
const id = o.manifestId ?? o.id ?? "imp";
|
||||
return [
|
||||
"schema: 1",
|
||||
`id: ${id}`,
|
||||
"title: Imported Course",
|
||||
`visibility: ${o.visibility ?? "restricted"}`,
|
||||
...(o.root ? [`root: ${o.root}`] : []),
|
||||
"cover: media/cover.jpg",
|
||||
"source:",
|
||||
` kind: ${o.kind ?? "video-folder"}`,
|
||||
" origin: /import/synthetic",
|
||||
"decks:",
|
||||
" - id: d1",
|
||||
` file: ${BIG_PDF}`,
|
||||
" attachTo: l1",
|
||||
"modules:",
|
||||
" - id: m1",
|
||||
" title: Videos",
|
||||
" lessons:",
|
||||
" - id: l1",
|
||||
" title: Intro",
|
||||
" type: video",
|
||||
` file: "${VIDEO}"`,
|
||||
" transcript: transcripts/l1.md",
|
||||
" - id: l2",
|
||||
" title: Next step",
|
||||
" type: video",
|
||||
` file: "${VIDEO2}"`,
|
||||
" - id: notes",
|
||||
" title: Reading",
|
||||
" type: text",
|
||||
" file: code/S01 - Basics/README.md",
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
/** Writes the course into `<dataDir>/courses/<id>` and returns that folder. */
|
||||
export function makeImportedCourse(dataDir: string, o: ImportFixtureOpts = {}): string {
|
||||
const id = o.id ?? "imp";
|
||||
const dir = join(dataDir, "courses", id);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
put(dir, "stcourse.yaml", manifestYaml(o));
|
||||
put(dir, VIDEO, videoBytes(4096, 7));
|
||||
put(dir, VIDEO2, videoBytes(2048, 11));
|
||||
put(dir, "media/cover.jpg", new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0, 0x10, 0x4a, 0x46, 0x49, 0x46, 0, 1, 0xff, 0xd9]));
|
||||
put(dir, BIG_PDF, new Uint8Array(LARGE_FILE_BYTES + 100).fill(0x25));
|
||||
put(dir, "transcripts/l1.md", "# Intro\n\n[0:00] Hello.\n");
|
||||
put(dir, "code/S01 - Basics/README.md", "# Basics\n\nRead me.\n");
|
||||
if (o.hostile) {
|
||||
// the companion code repo's own .gitignore hides Markdown: the lesson must still be committed
|
||||
put(dir, "code/S01 - Basics/.gitignore", "*.md\n");
|
||||
put(dir, "code/.git/config", "[core]\n");
|
||||
const outside = join(tempDir(), "secret.txt");
|
||||
writeFileSync(outside, "outside\n");
|
||||
symlinkSync(outside, join(dir, "code/leak.txt"));
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
Reference in New Issue
Block a user