STCourses/packages/feature-studio/test/convert.test.ts

import { describe, expect, test } from "bun:test";
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { loadAll, loadCourse } from "@stc/course-format";
import { createStudioStore } from "../src/server/store";
import { SimulatedCrash } from "../src/server/convert";
import { gitignoreLine, MEDIA_REFS, parseMediaRefs } from "../src/server/media-refs";
import { BIG_PDF, makeImportedCourse, VIDEO, VIDEO2, videoBytes } from "./import-fixture";
import { ada, extGit, newStore, tempDir } from "./helpers";

const ino = (p: string) => statSync(p).ino;
const tracked = async (dir: string) => (await extGit(dir, "-c", "core.quotepath=false", "ls-files")).split("\n").filter(Boolean).sort();

describe("convertImported", () => {
  test("builds the repo and published worktree; large media stay out of git as hard links", async () => {
    const { dataDir, store } = await newStore();
    const src = makeImportedCourse(dataDir, { hostile: true });
    const srcVideoIno = ino(join(src, VIDEO));
    const r = await store.convertImported("imp", ada);
    expect(r.media).toBe(3);
    expect(store.isManaged("imp")).toBe(true);
    expect(existsSync(join(dataDir, "studio/repos/imp.git/HEAD"))).toBe(true);
    expect(existsSync(src)).toBe(false);
    const mediaRoot = join(dataDir, "media/imp");
    expect(existsSync(join(mediaRoot, "stcourse.yaml"))).toBe(true);

    const pub = store.paths("imp").published;
    expect(await tracked(pub)).toEqual([
      MEDIA_REFS, "code/S01 - Basics/.gitignore", "code/S01 - Basics/README.md", "media/cover.jpg", "stcourse.yaml", "transcripts/l1.md",
    ]);
    // the symlink was skipped, never followed; .git contents never copied
    expect(existsSync(join(pub, "code/leak.txt"))).toBe(false);
    expect(existsSync(join(pub, "code/.git"))).toBe(false);
    expect(await extGit(pub, "status", "--porcelain", "--untracked-files=all")).toBe("");
    // the exclusions live in the repo's info/exclude, which authors and remotes cannot change
    const exclude = readFileSync(join(dataDir, "studio/repos/imp.git/info/exclude"), "utf8");
    for (const p of [VIDEO, VIDEO2, BIG_PDF]) expect(exclude).toContain(gitignoreLine(p));

    const refs = parseMediaRefs(readFileSync(join(pub, MEDIA_REFS), "utf8"))!;
    expect(refs.map((x) => x.path).sort()).toEqual([BIG_PDF, VIDEO2, VIDEO].sort());
    const v = refs.find((x) => x.path === VIDEO)!;
    expect(v.size).toBe(4096);
    expect(v.sha256).toBe(new Bun.CryptoHasher("sha256").update(videoBytes(4096, 7)).digest("hex"));
    for (const p of [VIDEO, VIDEO2, BIG_PDF]) expect(ino(join(pub, p))).toBe(ino(join(mediaRoot, p)));
    expect(ino(join(mediaRoot, VIDEO))).toBe(srcVideoIno);

    // the course loads from the published worktree exactly as before
    const c = await loadCourse(join(pub, "stcourse.yaml"));
    expect(c.status).toBe("ok");
    expect(c.manifest?.visibility).toBe("restricted");
    expect(c.manifest?.source?.kind).toBe("video-folder");
    // the commit is attributed to the converting owner
    expect(await extGit(pub, "log", "-1", "--format=%an <%ae>")).toBe(`${ada.name} <${ada.email}>`);
  });

  test("a draft gets the media; listFiles shows them; the media and reference file are read-only", async () => {
    const { dataDir, store } = await newStore();
    makeImportedCourse(dataDir);
    await store.convertImported("imp", ada);
    const draft = await store.ensureDraft("imp", ada.id);
    expect(ino(join(draft, VIDEO))).toBe(ino(join(dataDir, "media/imp", VIDEO)));
    const files = await store.listFiles("imp", ada.id);
    expect(files).toContain(VIDEO);
    expect(files).toContain(BIG_PDF);
    for (const path of [VIDEO, MEDIA_REFS]) {
      await expect(store.saveFiles("imp", ada, [{ path, content: "x" }], "x")).rejects.toMatchObject({ status: 409 });
      await expect(store.saveFiles("imp", ada, [{ path, delete: true }], "x")).rejects.toMatchObject({ status: 409 });
    }
    expect(readFileSync(join(dataDir, "media/imp", VIDEO))).toEqual(Buffer.from(videoBytes(4096, 7)));
  });

  test("a studio write to a hard-linked file breaks the link instead of writing through it", async () => {
    const { dataDir, store } = await newStore();
    makeImportedCourse(dataDir);
    await store.convertImported("imp", ada);
    const draft = await store.ensureDraft("imp", ada.id);
    // a planted hard link at an ordinary (non-referenced) path
    const outside = join(dataDir, "media/imp/transcripts/l1.md");
    rmSync(join(draft, "transcripts/l1.md"));
    const { linkSync } = await import("node:fs");
    linkSync(outside, join(draft, "transcripts/l1.md"));
    expect((await store.saveFiles("imp", ada, [{ path: "transcripts/l1.md", content: "# changed\n" }], "edit")).ok).toBe(true);
    expect(readFileSync(outside, "utf8")).toBe("# Intro\n\n[0:00] Hello.\n");
    expect(readFileSync(join(draft, "transcripts/l1.md"), "utf8")).toBe("# changed\n");
  });

  test("edit and publish: the published worktree keeps its media and stays clean", async () => {
    const { dataDir, store } = await newStore();
    makeImportedCourse(dataDir);
    await store.convertImported("imp", ada);
    const m = (await store.readFile("imp", ada.id, "stcourse.yaml"))!;
    const next = m.content.replace("title: Intro", "title: Welcome aboard");
    expect((await store.saveFiles("imp", ada, [{ path: "stcourse.yaml", content: next, baseHash: m.hash }], "Rename lesson")).ok).toBe(true);
    const r = await store.publish("imp", ada);
    expect(r).toMatchObject({ ok: true, published: true });
    const pub = store.paths("imp").published;
    expect(readFileSync(join(pub, "stcourse.yaml"), "utf8")).toContain("title: Welcome aboard");
    expect(ino(join(pub, VIDEO))).toBe(ino(join(dataDir, "media/imp", VIDEO)));
    expect(await extGit(pub, "status", "--porcelain", "--untracked-files=all")).toBe("");
  });

  test("a non-owner cannot publish the converted course as public; the owner can", async () => {
    const { dataDir, store } = await newStore();
    makeImportedCourse(dataDir);
    await store.convertImported("imp", ada);
    const m = (await store.readFile("imp", ada.id, "stcourse.yaml"))!;
    await store.saveFiles("imp", ada, [{ path: "stcourse.yaml", content: m.content.replace("visibility: restricted", "visibility: public"), baseHash: m.hash }], "go public");
    expect(await store.publish("imp", ada)).toMatchObject({ ok: false, reason: "forbidden" });
    expect(await store.publish("imp", { ...ada, owner: true })).toMatchObject({ ok: true, published: true });
  });

  describe("refusals leave the import untouched and no repo behind", () => {
    const cases: [string, (dataDir: string) => void, number][] = [
      ["no such import", () => {}, 404],
      ["an authored course", (d) => makeImportedCourse(d, { kind: "authored" }), 400],
      ["a manifest id mismatch", (d) => makeImportedCourse(d, { manifestId: "other" }), 400],
      ["a root: field", (d) => makeImportedCourse(d, { root: "." }), 400],
      ["a media store already there", (d) => { makeImportedCourse(d); mkdirSync(join(d, "media/imp"), { recursive: true }); }, 409],
    ];
    test.each(cases)("%s", async (_name, prepare, status) => {
      const { dataDir, store } = await newStore();
      prepare(dataDir);
      const before = existsSync(join(dataDir, "courses/imp/stcourse.yaml")) ? readFileSync(join(dataDir, "courses/imp/stcourse.yaml"), "utf8") : null;
      await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status });
      expect(existsSync(join(dataDir, "studio/repos/imp.git"))).toBe(false);
      expect(existsSync(join(dataDir, "worktrees/imp"))).toBe(false);
      if (before !== null) expect(readFileSync(join(dataDir, "courses/imp/stcourse.yaml"), "utf8")).toBe(before);
    });

    test("a studio course with the same id", async () => {
      const { store } = await newStore();
      await store.createCourse({ id: "imp", title: "Studio" }, ada);
      await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status: 409 });
      expect(store.isManaged("imp")).toBe(true);
    });

    test("a large file whose name has a line break", async () => {
      const { dataDir, store } = await newStore();
      const src = makeImportedCourse(dataDir);
      writeFileSync(join(src, "media/bad\nname.mp4"), "x");
      await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status: 400 });
      expect(existsSync(join(dataDir, "studio/repos/imp.git"))).toBe(false);
      expect(existsSync(join(src, "stcourse.yaml"))).toBe(true);
    });
  });

  describe("restart safety", () => {
    test("a leftover partial repo (no published worktree) is rebuilt", async () => {
      const { dataDir, store } = await newStore();
      makeImportedCourse(dataDir);
      mkdirSync(join(dataDir, "studio/repos/imp.git"), { recursive: true });
      writeFileSync(join(dataDir, "studio/repos/imp.git/junk"), "x");
      await store.convertImported("imp", ada);
      expect(store.isManaged("imp")).toBe(true);
      expect(existsSync(join(dataDir, "studio/repos/imp.git/junk"))).toBe(false);
    });

    test("a managed course whose import folder is still there only finishes the move", async () => {
      const { dataDir, store } = await newStore();
      makeImportedCourse(dataDir);
      await store.convertImported("imp", ada);
      const head = await store.publishedHead("imp");
      // simulate a crash between the checkout and the move
      const { renameSync } = await import("node:fs");
      renameSync(join(dataDir, "media/imp"), join(dataDir, "courses/imp"));
      const r = await store.convertImported("imp", ada);
      expect(r.resumed).toBe(true);
      expect(await store.publishedHead("imp")).toBe(head);
      expect(existsSync(join(dataDir, "courses/imp"))).toBe(false);
      expect(existsSync(join(dataDir, "media/imp", VIDEO))).toBe(true);
    });
  });
});

describe("an author's or a remote's .gitignore never blocks the course", () => {
  const editManifest = async (store: Awaited<ReturnType<typeof newStore>>["store"], title: string) => {
    const m = (await store.readFile("imp", ada.id, "stcourse.yaml"))!;
    const text = m.content.replace(/title: (Intro|Welcome \d+)\n/, `title: ${title}\n`);
    expect((await store.saveFiles("imp", ada, [{ path: "stcourse.yaml", content: text, baseHash: m.hash }], title)).ok).toBe(true);
  };

  test("un-ignoring the media in .gitignore: publish keeps working, again and again", async () => {
    const { dataDir, store } = await newStore();
    makeImportedCourse(dataDir);
    await store.convertImported("imp", ada);
    expect((await store.saveFiles("imp", ada, [{ path: ".gitignore", content: "!*.mp4\n!*.pdf\n!/media/**\n" }], "tidy .gitignore")).ok).toBe(true);
    expect(await store.publish("imp", ada)).toMatchObject({ ok: true, published: true });
    const pub = store.paths("imp").published;
    // git itself now sees the media as untracked; the studio does not count them
    expect(await extGit(pub, "-c", "core.quotepath=false", "status", "--porcelain")).toContain("media/");
    expect((await store.status("imp", ada.id)).publishedDirty).toEqual([]);
    await editManifest(store, "Welcome 2");
    expect(await store.publish("imp", ada)).toMatchObject({ ok: true, published: true });
    await editManifest(store, "Welcome 3");
    expect(await store.publish("imp", ada)).toMatchObject({ ok: true, published: true });
    expect(readFileSync(join(pub, "stcourse.yaml"), "utf8")).toContain("title: Welcome 3");
  });

  test("a remote commit that rewrites .gitignore: Sync and publish keep working", async () => {
    const { dataDir, store } = await newStore({ allowLocalRemotes: true });
    makeImportedCourse(dataDir);
    await store.convertImported("imp", ada);
    const remote = join(tempDir(), "remote.git");
    await extGit(tempDir(), "init", "-q", "--bare", "-b", "main", remote);
    await store.setRemote("imp", remote);
    expect(await store.sync("imp", ada)).toMatchObject({ ok: true, pushed: true });
    const other = join(tempDir(), "other");
    await extGit(tempDir(), "clone", "-q", remote, other);
    writeFileSync(join(other, ".gitignore"), "!*\n");
    await extGit(other, "add", ".gitignore");
    await extGit(other, "commit", "-qm", "unignore everything");
    await extGit(other, "push", "-q", "origin", "main");
    expect(await store.sync("imp", ada)).toMatchObject({ ok: true, pulled: true });
    expect(await store.updateDraft("imp", ada)).toMatchObject({ ok: true });
    await editManifest(store, "Welcome 2");
    expect(await store.publish("imp", ada)).toMatchObject({ ok: true, published: true });
    expect(await store.sync("imp", ada)).toMatchObject({ ok: true, pushed: true });
  });

  test("a remote commit that puts a file over referenced media is refused at Sync; the media stay", async () => {
    const { dataDir, store } = await newStore({ allowLocalRemotes: true });
    makeImportedCourse(dataDir);
    await store.convertImported("imp", ada);
    const remote = join(tempDir(), "remote.git");
    await extGit(tempDir(), "init", "-q", "--bare", "-b", "main", remote);
    await store.setRemote("imp", remote);
    await store.sync("imp", ada);
    const other = join(tempDir(), "other");
    await extGit(tempDir(), "clone", "-q", remote, other);
    mkdirSync(join(other, "media"), { recursive: true });
    writeFileSync(join(other, VIDEO), "not the video");
    await extGit(other, "add", "-f", VIDEO);
    await extGit(other, "commit", "-qm", "replace media");
    await extGit(other, "push", "-q", "origin", "main");
    expect(await store.sync("imp", ada)).toMatchObject({ ok: false, reason: "forbidden" });
    expect(ino(join(store.paths("imp").published, VIDEO))).toBe(ino(join(dataDir, "media/imp", VIDEO)));
  });

  test("referenced media are not read whole into the editor", async () => {
    const { dataDir, store } = await newStore();
    makeImportedCourse(dataDir);
    await store.convertImported("imp", ada);
    expect(await store.readFile("imp", ada.id, VIDEO)).toBeNull();
    expect(await store.readFile("imp", ada.id, "transcripts/l1.md")).not.toBeNull();
  });
});

describe("crash safety: a retry after any crash point succeeds and the id is never loaded twice", () => {
  const loaded = async (dataDir: string) => (await loadAll([join(dataDir, "courses"), join(dataDir, "worktrees")])).filter((c) => c.id === "imp");
  const sameFs = ["committed", "hidden", "worktree", "before-move"];
  const crossFs = [...sameFs, "staging-copied", "staging-placed", "source-retired"];
  const cases: [string, boolean][] = [...sameFs.map((s) => [s, false] as [string, boolean]), ...crossFs.map((s) => [s, true] as [string, boolean])];

  test.each(cases)("crash at %s (cross-filesystem: %p)", async (step, exdev) => {
    const dataDir = tempDir();
    makeImportedCourse(dataDir);
    const crashing = createStudioStore({ dataDir, testHooks: { crashAt: step, forceExdev: exdev } });
    await expect(crashing.convertImported("imp", ada)).rejects.toBeInstanceOf(SimulatedCrash);
    expect((await loaded(dataDir)).length).toBeLessThanOrEqual(1);

    // the process restarts: recovery first, then the owner (or the requeued job) retries
    const fresh = createStudioStore({ dataDir, testHooks: { forceExdev: exdev } });
    await fresh.recoverConversions();
    expect((await loaded(dataDir)).length).toBeLessThanOrEqual(1);
    await fresh.convertImported("imp", ada);
    // and once more: a re-run of a finished conversion reports success
    expect(await fresh.convertImported("imp", ada)).toMatchObject({ resumed: true });

    expect(fresh.isManaged("imp")).toBe(true);
    const all = await loaded(dataDir);
    expect(all).toHaveLength(1);
    expect(all[0]!.status).toBe("ok");
    expect(all[0]!.root).toBe(fresh.paths("imp").published);
    expect(readdirSync(join(dataDir, "courses"))).toEqual([]);
    expect(readdirSync(join(dataDir, "media"))).toEqual(["imp"]);
    const pub = fresh.paths("imp").published;
    for (const p of [VIDEO, VIDEO2, BIG_PDF]) {
      expect(statSync(join(pub, p)).size).toBe(statSync(join(dataDir, "media/imp", p)).size);
      if (!exdev) expect(ino(join(pub, p))).toBe(ino(join(dataDir, "media/imp", p)));
    }
    expect(readFileSync(join(dataDir, "media/imp", VIDEO))).toEqual(Buffer.from(videoBytes(4096, 7)));
    expect(await extGit(pub, "status", "--porcelain", "--untracked-files=all")).toBe("");
  });

  test("a failed cross-filesystem copy leaves no staging dir and a retry finishes", async () => {
    const dataDir = tempDir();
    makeImportedCourse(dataDir);
    const store = createStudioStore({ dataDir, testHooks: { forceExdev: true } });
    // the media store's parent is a file: the copy fails in-process
    writeFileSync(join(dataDir, "media"), "not a dir");
    await expect(store.convertImported("imp", ada)).rejects.toThrow();
    expect((await loaded(dataDir)).length).toBeLessThanOrEqual(1);
    rmSync(join(dataDir, "media"));
    await store.convertImported("imp", ada);
    expect(readdirSync(join(dataDir, "media"))).toEqual(["imp"]);
    expect(readdirSync(join(dataDir, "courses"))).toEqual([]);
    expect(await loaded(dataDir)).toHaveLength(1);
  });
});