feat(studio): convert an imported course into a studio repo, media kept outside git
Small files are committed unchanged (visibility and source.kind stay); large media are referenced from .stcourses/media.json, ignored, and hard-linked into every worktree from <dataDir>/media/<id>, where the import folder is moved. Studio writes to referenced media are refused and never write through a shared inode. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Lucas WintherandClaude Opus 5.5 committed
1 parent
60179407f4
commit
df6d235070
4 files changed
+510
-4
No files matched your search
@@ -0,0 +1,168 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { loadCourse } from "@stc/course-format";
|
||||
import { MEDIA_REFS, parseMediaRefs } from "../src/server/media-refs";
|
||||
import { BIG_PDF, makeImportedCourse, VIDEO, VIDEO2, videoBytes } from "./import-fixture";
|
||||
import { ada, extGit, newStore } from "./helpers";
|
||||
|
||||
const ino = (p: string) => statSync(p).ino;
|
||||
const tracked = async (dir: string) => (await extGit(dir, "-c", "core.quotepath=false", "ls-files")).split("\n").filter(Boolean).sort();
|
||||
|
||||
describe("convertImported", () => {
|
||||
test("builds the repo and published worktree; large media stay out of git as hard links", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
const src = makeImportedCourse(dataDir, { hostile: true });
|
||||
const srcVideoIno = ino(join(src, VIDEO));
|
||||
const r = await store.convertImported("imp", ada);
|
||||
expect(r.media).toBe(3);
|
||||
expect(store.isManaged("imp")).toBe(true);
|
||||
expect(existsSync(join(dataDir, "studio/repos/imp.git/HEAD"))).toBe(true);
|
||||
expect(existsSync(src)).toBe(false);
|
||||
const mediaRoot = join(dataDir, "media/imp");
|
||||
expect(existsSync(join(mediaRoot, "stcourse.yaml"))).toBe(true);
|
||||
|
||||
const pub = store.paths("imp").published;
|
||||
expect(await tracked(pub)).toEqual([
|
||||
".gitignore", MEDIA_REFS, "code/S01 - Basics/.gitignore", "code/S01 - Basics/README.md", "media/cover.jpg", "stcourse.yaml", "transcripts/l1.md",
|
||||
]);
|
||||
// the symlink was skipped, never followed; .git contents never copied
|
||||
expect(existsSync(join(pub, "code/leak.txt"))).toBe(false);
|
||||
expect(existsSync(join(pub, "code/.git"))).toBe(false);
|
||||
expect(await extGit(pub, "status", "--porcelain", "--untracked-files=all")).toBe("");
|
||||
|
||||
const refs = parseMediaRefs(readFileSync(join(pub, MEDIA_REFS), "utf8"))!;
|
||||
expect(refs.map((x) => x.path).sort()).toEqual([BIG_PDF, VIDEO2, VIDEO].sort());
|
||||
const v = refs.find((x) => x.path === VIDEO)!;
|
||||
expect(v.size).toBe(4096);
|
||||
expect(v.sha256).toBe(new Bun.CryptoHasher("sha256").update(videoBytes(4096, 7)).digest("hex"));
|
||||
for (const p of [VIDEO, VIDEO2, BIG_PDF]) expect(ino(join(pub, p))).toBe(ino(join(mediaRoot, p)));
|
||||
expect(ino(join(mediaRoot, VIDEO))).toBe(srcVideoIno);
|
||||
|
||||
// the course loads from the published worktree exactly as before
|
||||
const c = await loadCourse(join(pub, "stcourse.yaml"));
|
||||
expect(c.status).toBe("ok");
|
||||
expect(c.manifest?.visibility).toBe("restricted");
|
||||
expect(c.manifest?.source?.kind).toBe("video-folder");
|
||||
// the commit is attributed to the converting owner
|
||||
expect(await extGit(pub, "log", "-1", "--format=%an <%ae>")).toBe(`${ada.name} <${ada.email}>`);
|
||||
});
|
||||
|
||||
test("a draft gets the media; listFiles shows them; the media and reference file are read-only", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
makeImportedCourse(dataDir);
|
||||
await store.convertImported("imp", ada);
|
||||
const draft = await store.ensureDraft("imp", ada.id);
|
||||
expect(ino(join(draft, VIDEO))).toBe(ino(join(dataDir, "media/imp", VIDEO)));
|
||||
const files = await store.listFiles("imp", ada.id);
|
||||
expect(files).toContain(VIDEO);
|
||||
expect(files).toContain(BIG_PDF);
|
||||
for (const path of [VIDEO, MEDIA_REFS]) {
|
||||
await expect(store.saveFiles("imp", ada, [{ path, content: "x" }], "x")).rejects.toMatchObject({ status: 409 });
|
||||
await expect(store.saveFiles("imp", ada, [{ path, delete: true }], "x")).rejects.toMatchObject({ status: 409 });
|
||||
}
|
||||
expect(readFileSync(join(dataDir, "media/imp", VIDEO))).toEqual(Buffer.from(videoBytes(4096, 7)));
|
||||
});
|
||||
|
||||
test("a studio write to a hard-linked file breaks the link instead of writing through it", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
makeImportedCourse(dataDir);
|
||||
await store.convertImported("imp", ada);
|
||||
const draft = await store.ensureDraft("imp", ada.id);
|
||||
// a planted hard link at an ordinary (non-referenced) path
|
||||
const outside = join(dataDir, "media/imp/transcripts/l1.md");
|
||||
rmSync(join(draft, "transcripts/l1.md"));
|
||||
const { linkSync } = await import("node:fs");
|
||||
linkSync(outside, join(draft, "transcripts/l1.md"));
|
||||
expect((await store.saveFiles("imp", ada, [{ path: "transcripts/l1.md", content: "# changed\n" }], "edit")).ok).toBe(true);
|
||||
expect(readFileSync(outside, "utf8")).toBe("# Intro\n\n[0:00] Hello.\n");
|
||||
expect(readFileSync(join(draft, "transcripts/l1.md"), "utf8")).toBe("# changed\n");
|
||||
});
|
||||
|
||||
test("edit and publish: the published worktree keeps its media and stays clean", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
makeImportedCourse(dataDir);
|
||||
await store.convertImported("imp", ada);
|
||||
const m = (await store.readFile("imp", ada.id, "stcourse.yaml"))!;
|
||||
const next = m.content.replace("title: Intro", "title: Welcome aboard");
|
||||
expect((await store.saveFiles("imp", ada, [{ path: "stcourse.yaml", content: next, baseHash: m.hash }], "Rename lesson")).ok).toBe(true);
|
||||
const r = await store.publish("imp", ada);
|
||||
expect(r).toMatchObject({ ok: true, published: true });
|
||||
const pub = store.paths("imp").published;
|
||||
expect(readFileSync(join(pub, "stcourse.yaml"), "utf8")).toContain("title: Welcome aboard");
|
||||
expect(ino(join(pub, VIDEO))).toBe(ino(join(dataDir, "media/imp", VIDEO)));
|
||||
expect(await extGit(pub, "status", "--porcelain", "--untracked-files=all")).toBe("");
|
||||
});
|
||||
|
||||
test("a non-owner cannot publish the converted course as public; the owner can", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
makeImportedCourse(dataDir);
|
||||
await store.convertImported("imp", ada);
|
||||
const m = (await store.readFile("imp", ada.id, "stcourse.yaml"))!;
|
||||
await store.saveFiles("imp", ada, [{ path: "stcourse.yaml", content: m.content.replace("visibility: restricted", "visibility: public"), baseHash: m.hash }], "go public");
|
||||
expect(await store.publish("imp", ada)).toMatchObject({ ok: false, reason: "forbidden" });
|
||||
expect(await store.publish("imp", { ...ada, owner: true })).toMatchObject({ ok: true, published: true });
|
||||
});
|
||||
|
||||
describe("refusals leave the import untouched and no repo behind", () => {
|
||||
const cases: [string, (dataDir: string) => void, number][] = [
|
||||
["no such import", () => {}, 404],
|
||||
["an authored course", (d) => makeImportedCourse(d, { kind: "authored" }), 400],
|
||||
["a manifest id mismatch", (d) => makeImportedCourse(d, { manifestId: "other" }), 400],
|
||||
["a root: field", (d) => makeImportedCourse(d, { root: "." }), 400],
|
||||
["a media store already there", (d) => { makeImportedCourse(d); mkdirSync(join(d, "media/imp"), { recursive: true }); }, 409],
|
||||
];
|
||||
test.each(cases)("%s", async (_name, prepare, status) => {
|
||||
const { dataDir, store } = await newStore();
|
||||
prepare(dataDir);
|
||||
const before = existsSync(join(dataDir, "courses/imp/stcourse.yaml")) ? readFileSync(join(dataDir, "courses/imp/stcourse.yaml"), "utf8") : null;
|
||||
await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status });
|
||||
expect(existsSync(join(dataDir, "studio/repos/imp.git"))).toBe(false);
|
||||
expect(existsSync(join(dataDir, "worktrees/imp"))).toBe(false);
|
||||
if (before !== null) expect(readFileSync(join(dataDir, "courses/imp/stcourse.yaml"), "utf8")).toBe(before);
|
||||
});
|
||||
|
||||
test("a studio course with the same id", async () => {
|
||||
const { store } = await newStore();
|
||||
await store.createCourse({ id: "imp", title: "Studio" }, ada);
|
||||
await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status: 409 });
|
||||
expect(store.isManaged("imp")).toBe(true);
|
||||
});
|
||||
|
||||
test("a large file whose name has a line break", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
const src = makeImportedCourse(dataDir);
|
||||
writeFileSync(join(src, "media/bad\nname.mp4"), "x");
|
||||
await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status: 400 });
|
||||
expect(existsSync(join(dataDir, "studio/repos/imp.git"))).toBe(false);
|
||||
expect(existsSync(join(src, "stcourse.yaml"))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("restart safety", () => {
|
||||
test("a leftover partial repo (no published worktree) is rebuilt", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
makeImportedCourse(dataDir);
|
||||
mkdirSync(join(dataDir, "studio/repos/imp.git"), { recursive: true });
|
||||
writeFileSync(join(dataDir, "studio/repos/imp.git/junk"), "x");
|
||||
await store.convertImported("imp", ada);
|
||||
expect(store.isManaged("imp")).toBe(true);
|
||||
expect(existsSync(join(dataDir, "studio/repos/imp.git/junk"))).toBe(false);
|
||||
});
|
||||
|
||||
test("a managed course whose import folder is still there only finishes the move", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
makeImportedCourse(dataDir);
|
||||
await store.convertImported("imp", ada);
|
||||
const head = await store.publishedHead("imp");
|
||||
// simulate a crash between the checkout and the move
|
||||
const { renameSync } = await import("node:fs");
|
||||
renameSync(join(dataDir, "media/imp"), join(dataDir, "courses/imp"));
|
||||
const r = await store.convertImported("imp", ada);
|
||||
expect(r.resumed).toBe(true);
|
||||
expect(await store.publishedHead("imp")).toBe(head);
|
||||
expect(existsSync(join(dataDir, "courses/imp"))).toBe(false);
|
||||
expect(existsSync(join(dataDir, "media/imp", VIDEO))).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,86 @@
|
||||
// A synthetic imported course, laid out as the importer leaves it in <dataDir>/courses/<id>. Nothing here is real material.
|
||||
import { mkdirSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { LARGE_FILE_BYTES } from "../src/server/media-refs";
|
||||
import { tempDir } from "./helpers";
|
||||
|
||||
export const VIDEO = "media/01 Intro – Übersicht.mp4";
|
||||
export const VIDEO2 = "media/[2] Next step.mp4";
|
||||
export const BIG_PDF = "decks/big deck.pdf";
|
||||
/** Fake MP4 bytes: the server only streams them, nothing decodes them. */
|
||||
export const videoBytes = (n = 4096, seed = 7) => new Uint8Array(n).map((_, i) => (i * seed) % 256);
|
||||
|
||||
export interface ImportFixtureOpts {
|
||||
id?: string;
|
||||
kind?: string;
|
||||
manifestId?: string;
|
||||
root?: string;
|
||||
visibility?: string;
|
||||
/** Extra hostile bits: a symlink to outside, a .git dir, a nested .gitignore that hides a lesson. */
|
||||
hostile?: boolean;
|
||||
}
|
||||
|
||||
function put(dir: string, rel: string, content: string | Uint8Array) {
|
||||
mkdirSync(dirname(join(dir, rel)), { recursive: true });
|
||||
writeFileSync(join(dir, rel), content);
|
||||
}
|
||||
|
||||
export function manifestYaml(o: ImportFixtureOpts = {}): string {
|
||||
const id = o.manifestId ?? o.id ?? "imp";
|
||||
return [
|
||||
"schema: 1",
|
||||
`id: ${id}`,
|
||||
"title: Imported Course",
|
||||
`visibility: ${o.visibility ?? "restricted"}`,
|
||||
...(o.root ? [`root: ${o.root}`] : []),
|
||||
"cover: media/cover.jpg",
|
||||
"source:",
|
||||
` kind: ${o.kind ?? "video-folder"}`,
|
||||
" origin: /import/synthetic",
|
||||
"decks:",
|
||||
" - id: d1",
|
||||
` file: ${BIG_PDF}`,
|
||||
" attachTo: l1",
|
||||
"modules:",
|
||||
" - id: m1",
|
||||
" title: Videos",
|
||||
" lessons:",
|
||||
" - id: l1",
|
||||
" title: Intro",
|
||||
" type: video",
|
||||
` file: "${VIDEO}"`,
|
||||
" transcript: transcripts/l1.md",
|
||||
" - id: l2",
|
||||
" title: Next step",
|
||||
" type: video",
|
||||
` file: "${VIDEO2}"`,
|
||||
" - id: notes",
|
||||
" title: Reading",
|
||||
" type: text",
|
||||
" file: code/S01 - Basics/README.md",
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
/** Writes the course into `<dataDir>/courses/<id>` and returns that folder. */
|
||||
export function makeImportedCourse(dataDir: string, o: ImportFixtureOpts = {}): string {
|
||||
const id = o.id ?? "imp";
|
||||
const dir = join(dataDir, "courses", id);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
put(dir, "stcourse.yaml", manifestYaml(o));
|
||||
put(dir, VIDEO, videoBytes(4096, 7));
|
||||
put(dir, VIDEO2, videoBytes(2048, 11));
|
||||
put(dir, "media/cover.jpg", new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0, 0x10, 0x4a, 0x46, 0x49, 0x46, 0, 1, 0xff, 0xd9]));
|
||||
put(dir, BIG_PDF, new Uint8Array(LARGE_FILE_BYTES + 100).fill(0x25));
|
||||
put(dir, "transcripts/l1.md", "# Intro\n\n[0:00] Hello.\n");
|
||||
put(dir, "code/S01 - Basics/README.md", "# Basics\n\nRead me.\n");
|
||||
if (o.hostile) {
|
||||
// the companion code repo's own .gitignore hides Markdown: the lesson must still be committed
|
||||
put(dir, "code/S01 - Basics/.gitignore", "*.md\n");
|
||||
put(dir, "code/.git/config", "[core]\n");
|
||||
const outside = join(tempDir(), "secret.txt");
|
||||
writeFileSync(outside, "outside\n");
|
||||
symlinkSync(outside, join(dir, "code/leak.txt"));
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
Reference in new issue
Block a user