diff --git a/packages/feature-studio/src/server/convert.ts b/packages/feature-studio/src/server/convert.ts new file mode 100644 index 0000000..6e384ef --- /dev/null +++ b/packages/feature-studio/src/server/convert.ts @@ -0,0 +1,64 @@ +import { copyFile, link, mkdir, readdir, rename, rm, cp, stat } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { $ } from "bun"; +import { gitignoreLine, isLargeMedia } from "./media-refs"; +import { checkRelPath } from "./relpath"; + +export interface ScannedImport { + /** Committed into git. */ + small: string[]; + /** Kept out of git, referenced by path, size and checksum. */ + large: { path: string; size: number }[]; + /** Skipped and never followed: symlinks, sockets, `.git` folders. */ + skipped: string[]; +} + +/** Walks an import folder with lstat semantics: symlinks and special files are skipped, never followed. */ +export async function scanImport(root: string): Promise { + const out: ScannedImport = { small: [], large: [], skipped: [] }; + const walk = async (rel: string) => { + const entries = await readdir(join(root, rel), { withFileTypes: true }); + entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + for (const e of entries) { + const r = rel ? `${rel}/${e.name}` : e.name; + if (e.name.toLowerCase() === ".git") { out.skipped.push(r); continue; } + if (e.isDirectory()) await walk(r); + else if (e.isFile()) { + const size = (await stat(join(root, r))).size; + if (isLargeMedia(r, size)) out.large.push({ path: r, size }); + else out.small.push(r); + } else out.skipped.push(r); + } + }; + await walk(""); + return out; +} + +/** Problems that make a large file impossible to reference: a name git ignore files or the studio cannot express. */ +export function unreferencable(large: { path: string }[]): string[] { + const bad: string[] = []; + for (const f of large) { + try { gitignoreLine(f.path); if (checkRelPath(f.path) !== f.path) throw new Error("not normal"); } catch { bad.push(f.path); } + } + return bad; +} + +/** A hard link (no extra space), else a reflink or plain copy (another filesystem). The source is only read. */ +export async function linkOrCopy(src: string, dest: string): Promise { + await mkdir(dirname(dest), { recursive: true }); + try { await link(src, dest); return; } catch { /* cross-device or unsupported */ } + const r = await $`cp --reflink=auto -- ${src} ${dest}`.quiet().nothrow(); + if (r.exitCode !== 0) await copyFile(src, dest); +} + +/** Moves the import folder into the media store; across filesystems it copies (keeping links) and then removes. */ +export async function moveDir(src: string, dest: string): Promise { + await mkdir(dirname(dest), { recursive: true }); + try { + await rename(src, dest); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "EXDEV") throw e; + await cp(src, dest, { recursive: true, verbatimSymlinks: true, errorOnExist: true, force: false }); + await rm(src, { recursive: true, force: true }); + } +} diff --git a/packages/feature-studio/src/server/store.ts b/packages/feature-studio/src/server/store.ts index 7ebed29..54be0a5 100644 --- a/packages/feature-studio/src/server/store.ts +++ b/packages/feature-studio/src/server/store.ts @@ -1,8 +1,10 @@ -import { constants, existsSync, lstatSync, mkdirSync, readdirSync, realpathSync, rmSync, statSync } from "node:fs"; -import { mkdir, open, rm, unlink } from "node:fs/promises"; +import { constants, existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs"; +import { copyFile, mkdir, open, rm, unlink, writeFile } from "node:fs/promises"; import { dirname, join, sep } from "node:path"; -import { loadCourse, type Diagnostic } from "@stc/course-format"; +import { loadCourse, parseManifest, type Diagnostic } from "@stc/course-format"; +import { linkOrCopy, moveDir, scanImport, unreferencable } from "./convert"; import { git, gitOk, splitZ, withLock, type Identity } from "./git"; +import { MEDIA_REFS, mediaGitignore, parseMediaRefs, sha256File, stringifyMediaRefs, type MediaRef } from "./media-refs"; import { checkRelPath } from "./relpath"; export { checkRelPath } from "./relpath"; @@ -68,6 +70,20 @@ export type SyncResult = | { ok: false; reason: "conflict"; conflicts: string[] } | { ok: false; reason: "forbidden"; message: string }; +export interface ConvertResult { + commit: string; + /** Files kept out of git and referenced from MEDIA_REFS. */ + media: number; + /** Files committed. */ + files: number; + /** Paths skipped in the import folder (symlinks, special files, .git folders). */ + skipped: string[]; + /** True when an interrupted conversion was only finished (the repo already existed). */ + resumed: boolean; +} + +export type ConvertProgress = (fraction: number, message: string) => void | Promise; + export class StudioError extends Error { constructor(message: string, public status = 400) { super(message); } } @@ -121,6 +137,8 @@ export interface StudioStoreOptions { importedDir?: string; /** AppConfig.studioWorktreesDir (default `/worktrees`). */ worktreesDir?: string; + /** Media stores of converted imports (default `/media`): `/` is the former import folder. */ + mediaDir?: string; publishedBranch?: string; /** Tests only: allow local-path remotes (the HTTP feature never sets this). */ allowLocalRemotes?: boolean; @@ -133,6 +151,7 @@ export function createStudioStore(opts: StudioStoreOptions) { const reposDir = opts.reposDir ?? join(dataDir, "studio", "repos"); const importedDir = opts.importedDir ?? join(dataDir, "courses"); const worktreesDir = opts.worktreesDir ?? join(dataDir, "worktrees"); + const mediaDir = opts.mediaDir ?? join(dataDir, "media"); const remoteGuard = opts.allowLocalRemotes ? ["-c", "protocol.ext.allow=never", "-c", "protocol.fd.allow=never"] : REMOTE_GUARD; const paths = (courseId: string) => { @@ -144,6 +163,8 @@ export function createStudioStore(opts: StudioStoreOptions) { published: join(root, "published"), draftsDir: join(root, "drafts"), draft: (authorId: string) => join(root, "drafts", safeAuthor(authorId)), + media: join(mediaDir, courseId), + imported: join(importedDir, courseId), }; }; const branchOf = (authorId: string) => `draft/${safeAuthor(authorId)}`; @@ -233,9 +254,53 @@ export function createStudioStore(opts: StudioStoreOptions) { const branch = branchOf(authorId); const exists = (await git(p.bare, ["rev-parse", "-q", "--verify", `refs/heads/${branch}`])).code === 0; await gitOk(p.bare, exists ? ["worktree", "add", dir, branch] : ["worktree", "add", "-b", branch, dir, main]); + await materializeMedia(dir, p.media); return dir; } + /** The draft's or worktree's media references (empty for courses that never were imports). */ + async function mediaRefsIn(dir: string): Promise { + const f = await readFileIn(dir, MEDIA_REFS); + return f ? parseMediaRefs(f.content) ?? [] : []; + } + + /** + * Puts every referenced file into `dir` as a hard link to `fromRoot/` (a copy across filesystems). + * Idempotent; existing destinations are left alone. A reference must name a regular file whose real path is inside + * `fromRoot`, and the destination path may not contain a symlink. Returns the references it could not satisfy. + */ + async function materializeMedia(dir: string, fromRoot: string): Promise<{ linked: number; missing: string[] }> { + const refs = await mediaRefsIn(dir); + const missing: string[] = []; + let linked = 0; + let realFrom: string; + try { realFrom = realpathSync(fromRoot); } catch { return { linked, missing: refs.map((r) => r.path) }; } + for (const r of refs) { + try { + const src = join(realFrom, r.path); + const st = lstatSync(src); + const real = realpathSync(src); + if (!st.isFile() || !real.startsWith(realFrom + sep)) throw new Error("not a regular file inside the media store"); + const dest = await safeWritePath(dir, r.path); + if (lexists(dest)) continue; + await linkOrCopy(real, dest); + linked++; + } catch { + missing.push(r.path); + } + } + return { linked, missing }; + } + + /** Referenced media and the reference file are not editable in the studio (they are not in git). */ + async function refuseMediaWrites(dir: string, paths: string[]): Promise { + const locked = new Set([MEDIA_REFS, ...(await mediaRefsIn(dir)).map((r) => r.path)]); + const hit = paths.map((x) => checkRelPath(x)).filter((x) => locked.has(x)); + if (hit.length) { + throw new StudioError(`${hit.join(", ")}: imported media are kept outside git and cannot be changed in the studio; upload a file under a new name instead`, 409); + } + } + async function readFileIn(dir: string, rel: string): Promise<{ content: string; hash: string } | null> { let abs: string; try { abs = inside(dir, rel); } catch { return null; } @@ -260,6 +325,8 @@ export function createStudioStore(opts: StudioStoreOptions) { if (lexists(abs)) await unlink(abs); } else { const abs = await safeWritePath(dir, f.path); + // never write into an inode shared with another path (imported media, an owner's original): replace the file + if (lexists(abs) && lstatSync(abs).nlink > 1) await unlink(abs); // O_NOFOLLOW: a symlink swapped in after the check makes the open fail instead of writing through it const fh = await open(abs, constants.O_WRONLY | constants.O_CREAT | constants.O_TRUNC | constants.O_NOFOLLOW, 0o644); try { await fh.writeFile(f.content ?? ""); } finally { await fh.close(); } @@ -315,12 +382,130 @@ export function createStudioStore(opts: StudioStoreOptions) { }); }, + /** An importer folder `/` holding an imported (not authored) course the owner may convert. */ + isConvertibleImport(courseId: string): boolean { + if (!ID_RE.test(courseId) || isManaged(courseId)) return false; + const p = paths(courseId); + try { + if (!lstatSync(p.imported).isDirectory()) return false; + const m = parseLoose(readFileSync(join(p.imported, MANIFEST), "utf8")); + return !!m?.source?.kind && m.source.kind !== "authored"; + } catch { + return false; + } + }, + + /** Re-links referenced media into the published worktree (after an outside checkout removed them, say). */ + repairMedia: (courseId: string) => lock(courseId, async () => { + const p = requireManaged(courseId); + return materializeMedia(p.published, p.media); + }), + + /** + * Turns the imported course `/` into a studio course with the same id (v1.1 S, ruling R9): + * small files are committed as they are (manifest unchanged: visibility and source.kind stay), large media are + * referenced from MEDIA_REFS and hard-linked into each worktree, and the import folder moves to `/`. + * Owner-only at the HTTP layer. Restart-safe: leftovers of an attempt that never produced a published worktree are + * removed and the conversion redone; a converted course whose import folder is still there only finishes the move. + */ + convertImported(courseId: string, who: Identity, opts: { onProgress?: ConvertProgress } = {}): Promise { + if (!ID_RE.test(courseId)) return Promise.reject(new StudioError(`invalid course id: ${courseId}`, 404)); + const progress = opts.onProgress ?? (() => {}); + return lock(courseId, async () => { + const p = paths(courseId); + const srcManifest = join(p.imported, MANIFEST); + const hasImport = lexists(srcManifest); + if (isManaged(courseId)) { + const refs = await readFileIn(p.published, MEDIA_REFS); + if (!hasImport || !refs) throw new StudioError(`course "${courseId}" is already a studio course`, 409); + if (lexists(p.media)) throw new StudioError(`the media store ${p.media} already exists`, 409); + await moveDir(p.imported, p.media); + await materializeMedia(p.published, p.media); + return { commit: await rev(p.published, "HEAD"), media: (parseMediaRefs(refs.content) ?? []).length, files: 0, skipped: [], resumed: true }; + } + if (!hasImport) throw new StudioError(`there is no imported course "${courseId}" in ${importedDir}`, 404); + const top = lstatSync(p.imported); + if (top.isSymbolicLink() || !top.isDirectory() || lstatSync(srcManifest).isSymbolicLink()) throw new StudioError("the import folder and its manifest must not be symlinks", 400); + if (lexists(p.media)) throw new StudioError(`the media store ${p.media} already exists`, 409); + const parsed = parseManifest(await Bun.file(srcManifest).text()); + if (!parsed.ok) throw new StudioError(`the imported manifest is invalid: ${parsed.errors.join("; ")}`, 400); + const m = parsed.manifest; + if (m.id !== courseId) throw new StudioError(`the imported manifest has id "${m.id}", not "${courseId}"`, 400); + if (m.root) throw new StudioError("an imported course with root: cannot become a studio course", 400); + if (!m.source?.kind || m.source.kind === "authored") throw new StudioError("only imported courses can be made editable this way", 400); + + await progress(0.02, "Reading the import folder"); + const scan = await scanImport(p.imported); + const bad = unreferencable(scan.large); + if (bad.length) throw new StudioError(`these large files have names the studio cannot keep outside git; rename them first: ${bad.map((b) => JSON.stringify(b)).join(", ")}`, 400); + + // A previous attempt that never got as far as the published worktree: start again. + rmSync(p.bare, { recursive: true, force: true }); + rmSync(p.root, { recursive: true, force: true }); + + const total = scan.large.reduce((n, f) => n + f.size, 0) || 1; + let done = 0; + const refs: MediaRef[] = []; + for (const f of scan.large) { + await progress(0.05 + 0.75 * (done / total), `Checksumming ${f.path}`); + refs.push({ path: f.path, size: f.size, sha256: await sha256File(join(p.imported, f.path)) }); + done += f.size; + } + + const work = join(dataDir, "tmp", `convert-${courseId}-${Bun.randomUUIDv7()}`); + const stage = join(work, "tree"); + try { + await progress(0.82, "Committing the course files"); + for (const f of scan.small) { + await mkdir(dirname(join(stage, f)), { recursive: true }); + await copyFile(join(p.imported, f), join(stage, f)); + } + const ownIgnore = scan.small.includes(".gitignore") ? await Bun.file(join(stage, ".gitignore")).text() : null; + await mkdir(join(stage, dirname(MEDIA_REFS)), { recursive: true }); + await writeFile(join(stage, MEDIA_REFS), stringifyMediaRefs(refs)); + await writeFile(join(stage, ".gitignore"), mediaGitignore(refs.map((r) => r.path), ownIgnore)); + const list = [...new Set([...scan.small, ".gitignore", MEDIA_REFS])]; + await writeFile(join(work, "paths"), list.join("") + ""); + + mkdirSync(reposDir, { recursive: true }); + await gitOk(dataDir, ["init", "-q", "--bare", "-b", main, p.bare]); + const gd = ["--literal-pathspecs", "--git-dir", p.bare, "--work-tree", stage]; + // -f: a nested .gitignore from the course's own code must not drop course files + await gitOk(stage, [...gd, "add", "-f", `--pathspec-from-file=${join(work, "paths")}`, "--pathspec-file-nul"]); + await gitOk(stage, [...gd, "commit", "-q", "-m", `Make ${courseId} editable in the studio + +Converted from the import in ${p.imported}. ${refs.length} large media files stay outside git (${MEDIA_REFS}).`], who); + mkdirSync(p.root, { recursive: true }); + await gitOk(p.bare, ["worktree", "add", p.published, main]); + + await progress(0.92, "Linking media into the published course"); + const mat = await materializeMedia(p.published, p.imported); + if (mat.missing.length) throw new Error(`could not link media: ${mat.missing.join(", ")}`); + const dirty = await dirtyFiles(p.published); + if (dirty.length) throw new Error(`the converted course is not clean after checkout: ${dirty.join(", ")}`); + + await progress(0.97, "Moving the import folder to the media store"); + await moveDir(p.imported, p.media); + return { commit: await rev(p.published, "HEAD"), media: refs.length, files: list.length, skipped: scan.skipped, resumed: false }; + } catch (e) { + if (lexists(p.imported)) { + rmSync(p.bare, { recursive: true, force: true }); + rmSync(p.root, { recursive: true, force: true }); + } + throw e; + } finally { + await rm(work, { recursive: true, force: true }); + } + }); + }, + ensureDraft: (courseId: string, authorId: string) => lock(courseId, () => ensureDraftUnlocked(courseId, authorId)), async listFiles(courseId: string, authorId: string): Promise { const dir = await api.ensureDraft(courseId, authorId); const tracked = splitZ(await gitOk(dir, ["ls-files", "-z", "--cached", "--others", "--exclude-standard"])); - return [...new Set(tracked)].filter((f) => existsSync(join(dir, f))).sort(); + const media = (await mediaRefsIn(dir)).map((r) => r.path); + return [...new Set([...tracked, ...media])].filter((f) => existsSync(join(dir, f))).sort(); }, async readFile(courseId: string, authorId: string, rel: string) { @@ -340,6 +525,7 @@ export function createStudioStore(opts: StudioStoreOptions) { return lock(courseId, async () => { const dir = await ensureDraftUnlocked(courseId, who.id); if (await isMerging(dir)) throw new StudioError("the draft has a merge in progress: resolve or abort it first", 409); + await refuseMediaWrites(dir, files.map((f) => f.path)); const conflicts: FileConflict[] = []; for (const f of files) { if (f.baseHash === undefined) continue; @@ -432,6 +618,7 @@ export function createStudioStore(opts: StudioStoreOptions) { return { ok: false, reason: "invalid", diagnostics: mergedErrors }; } await gitOk(p.published, ["commit", "-q", "-m", `Publish ${branch}`], who); + await materializeMedia(p.published, p.media); // Bring the draft level with what was just published, so "behind" stays meaningful. await git(dir, ["merge", "-q", "--ff-only", main]); return { ok: true, commit: await rev(p.published, "HEAD"), published: true }; @@ -493,6 +680,7 @@ export function createStudioStore(opts: StudioStoreOptions) { return lock(courseId, async () => { const dir = await ensureDraftUnlocked(courseId, who.id); if (!(await isMerging(dir))) throw new StudioError("no merge in progress", 409); + await refuseMediaWrites(dir, files.map((f) => f.path)); const m = files.find((f) => checkRelPath(f.path) === MANIFEST); if (m && !who.owner) { const needOwner = ownerRequiredFor(await showText(requireManaged(courseId).bare, `${main}:${MANIFEST}`), m.content); diff --git a/packages/feature-studio/test/convert.test.ts b/packages/feature-studio/test/convert.test.ts new file mode 100644 index 0000000..f604b64 --- /dev/null +++ b/packages/feature-studio/test/convert.test.ts @@ -0,0 +1,168 @@ +import { describe, expect, test } from "bun:test"; +import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { loadCourse } from "@stc/course-format"; +import { MEDIA_REFS, parseMediaRefs } from "../src/server/media-refs"; +import { BIG_PDF, makeImportedCourse, VIDEO, VIDEO2, videoBytes } from "./import-fixture"; +import { ada, extGit, newStore } from "./helpers"; + +const ino = (p: string) => statSync(p).ino; +const tracked = async (dir: string) => (await extGit(dir, "-c", "core.quotepath=false", "ls-files")).split("\n").filter(Boolean).sort(); + +describe("convertImported", () => { + test("builds the repo and published worktree; large media stay out of git as hard links", async () => { + const { dataDir, store } = await newStore(); + const src = makeImportedCourse(dataDir, { hostile: true }); + const srcVideoIno = ino(join(src, VIDEO)); + const r = await store.convertImported("imp", ada); + expect(r.media).toBe(3); + expect(store.isManaged("imp")).toBe(true); + expect(existsSync(join(dataDir, "studio/repos/imp.git/HEAD"))).toBe(true); + expect(existsSync(src)).toBe(false); + const mediaRoot = join(dataDir, "media/imp"); + expect(existsSync(join(mediaRoot, "stcourse.yaml"))).toBe(true); + + const pub = store.paths("imp").published; + expect(await tracked(pub)).toEqual([ + ".gitignore", MEDIA_REFS, "code/S01 - Basics/.gitignore", "code/S01 - Basics/README.md", "media/cover.jpg", "stcourse.yaml", "transcripts/l1.md", + ]); + // the symlink was skipped, never followed; .git contents never copied + expect(existsSync(join(pub, "code/leak.txt"))).toBe(false); + expect(existsSync(join(pub, "code/.git"))).toBe(false); + expect(await extGit(pub, "status", "--porcelain", "--untracked-files=all")).toBe(""); + + const refs = parseMediaRefs(readFileSync(join(pub, MEDIA_REFS), "utf8"))!; + expect(refs.map((x) => x.path).sort()).toEqual([BIG_PDF, VIDEO2, VIDEO].sort()); + const v = refs.find((x) => x.path === VIDEO)!; + expect(v.size).toBe(4096); + expect(v.sha256).toBe(new Bun.CryptoHasher("sha256").update(videoBytes(4096, 7)).digest("hex")); + for (const p of [VIDEO, VIDEO2, BIG_PDF]) expect(ino(join(pub, p))).toBe(ino(join(mediaRoot, p))); + expect(ino(join(mediaRoot, VIDEO))).toBe(srcVideoIno); + + // the course loads from the published worktree exactly as before + const c = await loadCourse(join(pub, "stcourse.yaml")); + expect(c.status).toBe("ok"); + expect(c.manifest?.visibility).toBe("restricted"); + expect(c.manifest?.source?.kind).toBe("video-folder"); + // the commit is attributed to the converting owner + expect(await extGit(pub, "log", "-1", "--format=%an <%ae>")).toBe(`${ada.name} <${ada.email}>`); + }); + + test("a draft gets the media; listFiles shows them; the media and reference file are read-only", async () => { + const { dataDir, store } = await newStore(); + makeImportedCourse(dataDir); + await store.convertImported("imp", ada); + const draft = await store.ensureDraft("imp", ada.id); + expect(ino(join(draft, VIDEO))).toBe(ino(join(dataDir, "media/imp", VIDEO))); + const files = await store.listFiles("imp", ada.id); + expect(files).toContain(VIDEO); + expect(files).toContain(BIG_PDF); + for (const path of [VIDEO, MEDIA_REFS]) { + await expect(store.saveFiles("imp", ada, [{ path, content: "x" }], "x")).rejects.toMatchObject({ status: 409 }); + await expect(store.saveFiles("imp", ada, [{ path, delete: true }], "x")).rejects.toMatchObject({ status: 409 }); + } + expect(readFileSync(join(dataDir, "media/imp", VIDEO))).toEqual(Buffer.from(videoBytes(4096, 7))); + }); + + test("a studio write to a hard-linked file breaks the link instead of writing through it", async () => { + const { dataDir, store } = await newStore(); + makeImportedCourse(dataDir); + await store.convertImported("imp", ada); + const draft = await store.ensureDraft("imp", ada.id); + // a planted hard link at an ordinary (non-referenced) path + const outside = join(dataDir, "media/imp/transcripts/l1.md"); + rmSync(join(draft, "transcripts/l1.md")); + const { linkSync } = await import("node:fs"); + linkSync(outside, join(draft, "transcripts/l1.md")); + expect((await store.saveFiles("imp", ada, [{ path: "transcripts/l1.md", content: "# changed\n" }], "edit")).ok).toBe(true); + expect(readFileSync(outside, "utf8")).toBe("# Intro\n\n[0:00] Hello.\n"); + expect(readFileSync(join(draft, "transcripts/l1.md"), "utf8")).toBe("# changed\n"); + }); + + test("edit and publish: the published worktree keeps its media and stays clean", async () => { + const { dataDir, store } = await newStore(); + makeImportedCourse(dataDir); + await store.convertImported("imp", ada); + const m = (await store.readFile("imp", ada.id, "stcourse.yaml"))!; + const next = m.content.replace("title: Intro", "title: Welcome aboard"); + expect((await store.saveFiles("imp", ada, [{ path: "stcourse.yaml", content: next, baseHash: m.hash }], "Rename lesson")).ok).toBe(true); + const r = await store.publish("imp", ada); + expect(r).toMatchObject({ ok: true, published: true }); + const pub = store.paths("imp").published; + expect(readFileSync(join(pub, "stcourse.yaml"), "utf8")).toContain("title: Welcome aboard"); + expect(ino(join(pub, VIDEO))).toBe(ino(join(dataDir, "media/imp", VIDEO))); + expect(await extGit(pub, "status", "--porcelain", "--untracked-files=all")).toBe(""); + }); + + test("a non-owner cannot publish the converted course as public; the owner can", async () => { + const { dataDir, store } = await newStore(); + makeImportedCourse(dataDir); + await store.convertImported("imp", ada); + const m = (await store.readFile("imp", ada.id, "stcourse.yaml"))!; + await store.saveFiles("imp", ada, [{ path: "stcourse.yaml", content: m.content.replace("visibility: restricted", "visibility: public"), baseHash: m.hash }], "go public"); + expect(await store.publish("imp", ada)).toMatchObject({ ok: false, reason: "forbidden" }); + expect(await store.publish("imp", { ...ada, owner: true })).toMatchObject({ ok: true, published: true }); + }); + + describe("refusals leave the import untouched and no repo behind", () => { + const cases: [string, (dataDir: string) => void, number][] = [ + ["no such import", () => {}, 404], + ["an authored course", (d) => makeImportedCourse(d, { kind: "authored" }), 400], + ["a manifest id mismatch", (d) => makeImportedCourse(d, { manifestId: "other" }), 400], + ["a root: field", (d) => makeImportedCourse(d, { root: "." }), 400], + ["a media store already there", (d) => { makeImportedCourse(d); mkdirSync(join(d, "media/imp"), { recursive: true }); }, 409], + ]; + test.each(cases)("%s", async (_name, prepare, status) => { + const { dataDir, store } = await newStore(); + prepare(dataDir); + const before = existsSync(join(dataDir, "courses/imp/stcourse.yaml")) ? readFileSync(join(dataDir, "courses/imp/stcourse.yaml"), "utf8") : null; + await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status }); + expect(existsSync(join(dataDir, "studio/repos/imp.git"))).toBe(false); + expect(existsSync(join(dataDir, "worktrees/imp"))).toBe(false); + if (before !== null) expect(readFileSync(join(dataDir, "courses/imp/stcourse.yaml"), "utf8")).toBe(before); + }); + + test("a studio course with the same id", async () => { + const { store } = await newStore(); + await store.createCourse({ id: "imp", title: "Studio" }, ada); + await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status: 409 }); + expect(store.isManaged("imp")).toBe(true); + }); + + test("a large file whose name has a line break", async () => { + const { dataDir, store } = await newStore(); + const src = makeImportedCourse(dataDir); + writeFileSync(join(src, "media/bad\nname.mp4"), "x"); + await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status: 400 }); + expect(existsSync(join(dataDir, "studio/repos/imp.git"))).toBe(false); + expect(existsSync(join(src, "stcourse.yaml"))).toBe(true); + }); + }); + + describe("restart safety", () => { + test("a leftover partial repo (no published worktree) is rebuilt", async () => { + const { dataDir, store } = await newStore(); + makeImportedCourse(dataDir); + mkdirSync(join(dataDir, "studio/repos/imp.git"), { recursive: true }); + writeFileSync(join(dataDir, "studio/repos/imp.git/junk"), "x"); + await store.convertImported("imp", ada); + expect(store.isManaged("imp")).toBe(true); + expect(existsSync(join(dataDir, "studio/repos/imp.git/junk"))).toBe(false); + }); + + test("a managed course whose import folder is still there only finishes the move", async () => { + const { dataDir, store } = await newStore(); + makeImportedCourse(dataDir); + await store.convertImported("imp", ada); + const head = await store.publishedHead("imp"); + // simulate a crash between the checkout and the move + const { renameSync } = await import("node:fs"); + renameSync(join(dataDir, "media/imp"), join(dataDir, "courses/imp")); + const r = await store.convertImported("imp", ada); + expect(r.resumed).toBe(true); + expect(await store.publishedHead("imp")).toBe(head); + expect(existsSync(join(dataDir, "courses/imp"))).toBe(false); + expect(existsSync(join(dataDir, "media/imp", VIDEO))).toBe(true); + }); + }); +}); diff --git a/packages/feature-studio/test/import-fixture.ts b/packages/feature-studio/test/import-fixture.ts new file mode 100644 index 0000000..007c77f --- /dev/null +++ b/packages/feature-studio/test/import-fixture.ts @@ -0,0 +1,86 @@ +// A synthetic imported course, laid out as the importer leaves it in /courses/. Nothing here is real material. +import { mkdirSync, symlinkSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { LARGE_FILE_BYTES } from "../src/server/media-refs"; +import { tempDir } from "./helpers"; + +export const VIDEO = "media/01 Intro – Übersicht.mp4"; +export const VIDEO2 = "media/[2] Next step.mp4"; +export const BIG_PDF = "decks/big deck.pdf"; +/** Fake MP4 bytes: the server only streams them, nothing decodes them. */ +export const videoBytes = (n = 4096, seed = 7) => new Uint8Array(n).map((_, i) => (i * seed) % 256); + +export interface ImportFixtureOpts { + id?: string; + kind?: string; + manifestId?: string; + root?: string; + visibility?: string; + /** Extra hostile bits: a symlink to outside, a .git dir, a nested .gitignore that hides a lesson. */ + hostile?: boolean; +} + +function put(dir: string, rel: string, content: string | Uint8Array) { + mkdirSync(dirname(join(dir, rel)), { recursive: true }); + writeFileSync(join(dir, rel), content); +} + +export function manifestYaml(o: ImportFixtureOpts = {}): string { + const id = o.manifestId ?? o.id ?? "imp"; + return [ + "schema: 1", + `id: ${id}`, + "title: Imported Course", + `visibility: ${o.visibility ?? "restricted"}`, + ...(o.root ? [`root: ${o.root}`] : []), + "cover: media/cover.jpg", + "source:", + ` kind: ${o.kind ?? "video-folder"}`, + " origin: /import/synthetic", + "decks:", + " - id: d1", + ` file: ${BIG_PDF}`, + " attachTo: l1", + "modules:", + " - id: m1", + " title: Videos", + " lessons:", + " - id: l1", + " title: Intro", + " type: video", + ` file: "${VIDEO}"`, + " transcript: transcripts/l1.md", + " - id: l2", + " title: Next step", + " type: video", + ` file: "${VIDEO2}"`, + " - id: notes", + " title: Reading", + " type: text", + " file: code/S01 - Basics/README.md", + "", + ].join("\n"); +} + +/** Writes the course into `/courses/` and returns that folder. */ +export function makeImportedCourse(dataDir: string, o: ImportFixtureOpts = {}): string { + const id = o.id ?? "imp"; + const dir = join(dataDir, "courses", id); + mkdirSync(dir, { recursive: true }); + put(dir, "stcourse.yaml", manifestYaml(o)); + put(dir, VIDEO, videoBytes(4096, 7)); + put(dir, VIDEO2, videoBytes(2048, 11)); + put(dir, "media/cover.jpg", new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0, 0x10, 0x4a, 0x46, 0x49, 0x46, 0, 1, 0xff, 0xd9])); + put(dir, BIG_PDF, new Uint8Array(LARGE_FILE_BYTES + 100).fill(0x25)); + put(dir, "transcripts/l1.md", "# Intro\n\n[0:00] Hello.\n"); + put(dir, "code/S01 - Basics/README.md", "# Basics\n\nRead me.\n"); + if (o.hostile) { + // the companion code repo's own .gitignore hides Markdown: the lesson must still be committed + put(dir, "code/S01 - Basics/.gitignore", "*.md\n"); + put(dir, "code/.git/config", "[core]\n"); + const outside = join(tempDir(), "secret.txt"); + writeFileSync(outside, "outside\n"); + symlinkSync(outside, join(dir, "code/leak.txt")); + } + return dir; +}