diff --git a/packages/feature-studio/src/server/media-refs.ts b/packages/feature-studio/src/server/media-refs.ts new file mode 100644 index 0000000..12ba06f --- /dev/null +++ b/packages/feature-studio/src/server/media-refs.ts @@ -0,0 +1,61 @@ +import { extname } from "node:path"; +import { checkRelPath } from "./relpath"; + +/** + * Large media of a converted import stay out of git (v1.1 S, docs/operations.md "Imported courses in the studio"). + * The repo commits `MEDIA_REFS` (path, size and checksum per file, spec 5) and a `.gitignore` line per path; + * every studio worktree gets the files as hard links to `/media//`. + */ +export const MEDIA_REFS = ".stcourses/media.json"; +export const LARGE_FILE_BYTES = 8 * 1024 * 1024; +const MEDIA_EXT = new Set(["mp4", "m4v", "webm", "mov", "mkv", "avi", "mp3", "m4a", "wav", "ogg", "oga", "opus", "flac"]); + +export interface MediaRef { path: string; size: number; sha256: string } + +/** Video and audio always; anything else only when it is over `LARGE_FILE_BYTES`. */ +export function isLargeMedia(rel: string, size: number): boolean { + return MEDIA_EXT.has(extname(rel).slice(1).toLowerCase()) || size > LARGE_FILE_BYTES; +} + +/** One anchored `.gitignore` pattern matching exactly `rel`. Throws for names a `.gitignore` line cannot hold. */ +export function gitignoreLine(rel: string): string { + if (/[\n\r]/.test(rel)) throw new Error(`a file name with a line break cannot be kept out of git: ${JSON.stringify(rel)}`); + let s = rel.replace(/[\\*?[\]]/g, (c) => `\\${c}`); + const trail = /( +)$/.exec(s)?.[1]?.length ?? 0; + if (trail) s = s.slice(0, -trail) + "\\ ".repeat(trail); + return `/${s}`; +} + +const HEADER = "# Imported media kept outside git: see .stcourses/media.json (STCourses)"; + +/** The repo's root `.gitignore`: the course's own one (if any) first, then one line per referenced file. */ +export function mediaGitignore(paths: string[], existing: string | null): string { + const own = existing ? (existing.endsWith("\n") ? existing : existing + "\n") + "\n" : ""; + return `${own}${HEADER}\n${paths.map(gitignoreLine).join("\n")}\n`; +} + +export async function sha256File(abs: string): Promise { + const h = new Bun.CryptoHasher("sha256"); + for await (const chunk of Bun.file(abs).stream()) h.update(chunk); + return h.digest("hex"); +} + +export function stringifyMediaRefs(files: MediaRef[]): string { + return JSON.stringify({ schema: 1, files }, null, 2) + "\n"; +} + +/** The reference list, or null when the text is not a valid one (every path must pass `checkRelPath`). */ +export function parseMediaRefs(text: string): MediaRef[] | null { + let raw: unknown; + try { raw = JSON.parse(text); } catch { return null; } + const r = raw as { schema?: unknown; files?: unknown } | null; + if (!r || r.schema !== 1 || !Array.isArray(r.files)) return null; + const out: MediaRef[] = []; + for (const f of r.files as Partial[]) { + if (!f || typeof f.path !== "string" || typeof f.size !== "number" || !Number.isInteger(f.size) || f.size < 0) return null; + if (typeof f.sha256 !== "string" || !/^[0-9a-f]{64}$/.test(f.sha256)) return null; + try { if (checkRelPath(f.path) !== f.path) return null; } catch { return null; } + out.push({ path: f.path, size: f.size, sha256: f.sha256 }); + } + return out; +} diff --git a/packages/feature-studio/src/server/relpath.ts b/packages/feature-studio/src/server/relpath.ts new file mode 100644 index 0000000..c6a914b --- /dev/null +++ b/packages/feature-studio/src/server/relpath.ts @@ -0,0 +1,12 @@ +import { isAbsolute, normalize, sep } from "node:path"; + +/** Normalised relative path, or throws. Rejects absolute paths, `..`, `.git` and NUL. */ +export function checkRelPath(p: string): string { + if (typeof p !== "string" || p === "" || p.includes("\0") || isAbsolute(p) || p.includes("\\")) throw new Error(`invalid path: ${JSON.stringify(p)}`); + const n = normalize(p); + const parts = n.split(sep); + if (n === "." || parts[0] === ".." || parts.some((x) => x === ".." || x.toLowerCase() === ".git") || n.endsWith(sep)) { + throw new Error(`invalid path: ${JSON.stringify(p)}`); + } + return n; +} diff --git a/packages/feature-studio/src/server/store.ts b/packages/feature-studio/src/server/store.ts index 638822a..7ebed29 100644 --- a/packages/feature-studio/src/server/store.ts +++ b/packages/feature-studio/src/server/store.ts @@ -1,8 +1,11 @@ import { constants, existsSync, lstatSync, mkdirSync, readdirSync, realpathSync, rmSync, statSync } from "node:fs"; import { mkdir, open, rm, unlink } from "node:fs/promises"; -import { dirname, isAbsolute, join, normalize, sep } from "node:path"; +import { dirname, join, sep } from "node:path"; import { loadCourse, type Diagnostic } from "@stc/course-format"; import { git, gitOk, splitZ, withLock, type Identity } from "./git"; +import { checkRelPath } from "./relpath"; + +export { checkRelPath } from "./relpath"; export type { Identity } from "./git"; @@ -19,17 +22,6 @@ export function safeAuthor(authorId: string): string { return s === "" ? "_" : s; } -/** Normalised relative path, or throws. Rejects absolute paths, `..`, `.git` and NUL. */ -export function checkRelPath(p: string): string { - if (typeof p !== "string" || p === "" || p.includes("\0") || isAbsolute(p) || p.includes("\\")) throw new Error(`invalid path: ${JSON.stringify(p)}`); - const n = normalize(p); - const parts = n.split(sep); - if (n === "." || parts[0] === ".." || parts.some((x) => x === ".." || x.toLowerCase() === ".git") || n.endsWith(sep)) { - throw new Error(`invalid path: ${JSON.stringify(p)}`); - } - return n; -} - export interface FileWrite { path: string; content?: string | Uint8Array; diff --git a/packages/feature-studio/test/media-refs.test.ts b/packages/feature-studio/test/media-refs.test.ts new file mode 100644 index 0000000..03d7e3a --- /dev/null +++ b/packages/feature-studio/test/media-refs.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, test } from "bun:test"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { $ } from "bun"; +import { gitignoreLine, isLargeMedia, LARGE_FILE_BYTES, mediaGitignore, parseMediaRefs, sha256File } from "../src/server/media-refs"; +import { tempDir } from "./helpers"; + +describe("isLargeMedia", () => { + test("video and audio by extension, whatever the size", () => { + for (const p of ["a.mp4", "x/B.MOV", "c.webm", "d.mkv", "e.mp3", "f.m4a", "g.wav", "h.flac", "i.avi", "j.m4v"]) expect(isLargeMedia(p, 10)).toBe(true); + }); + test("other files only above the size limit", () => { + expect(isLargeMedia("deck.pdf", 1000)).toBe(false); + expect(isLargeMedia("deck.pdf", LARGE_FILE_BYTES + 1)).toBe(true); + expect(isLargeMedia("lesson.md", LARGE_FILE_BYTES)).toBe(false); + }); +}); + +describe("gitignoreLine", () => { + test("anchors and escapes", () => { + expect(gitignoreLine("media/a.mp4")).toBe("/media/a.mp4"); + expect(gitignoreLine("media/[1] *?.mp4")).toBe("/media/\\[1\\] \\*\\?.mp4"); + expect(gitignoreLine("a\\b.mp4")).toBe("/a\\\\b.mp4"); + expect(gitignoreLine("trail.mp4 ")).toBe("/trail.mp4\\ \\ "); + }); + test("refuses names a .gitignore cannot express", () => { + expect(() => gitignoreLine("a\nb.mp4")).toThrow(); + expect(() => gitignoreLine("a\rb.mp4")).toThrow(); + }); + + test("git ignores exactly the listed paths", async () => { + const dir = tempDir(); + await $`git init -q ${dir}`.quiet(); + const listed = ["media/01 Intro – Übersicht.mp4", "media/[draft] *star?.mp4", "#hash.mp4", "!bang.mp4", "media/trailing.mp4 ", "deep/a b/c.mkv"]; + const siblings = ["media/01 Intro – Übersicht.mp4.txt", "media/d.mp4", "hash.mp4", "bang.mp4", "media/trailing.mp4", "deep/a b/c.mkv2", "sub/#hash.mp4"]; + for (const p of [...listed, ...siblings]) { + mkdirSync(join(dir, p, ".."), { recursive: true }); + writeFileSync(join(dir, p), "x"); + } + writeFileSync(join(dir, ".gitignore"), mediaGitignore(listed, null)); + const input = Buffer.from([...listed, ...siblings].join("\0") + "\0"); + const r = await $`git -c core.quotepath=false check-ignore -z --no-index --stdin < ${input}`.cwd(dir).quiet().nothrow(); + const ignored = r.stdout.toString().split("\0").filter(Boolean).sort(); + expect(ignored).toEqual([...listed].sort()); + }); + + test("an existing .gitignore is kept above the media block", () => { + const out = mediaGitignore(["a.mp4"], "node_modules/\n"); + expect(out.startsWith("node_modules/\n")).toBe(true); + expect(out).toContain("/a.mp4\n"); + expect(mediaGitignore(["a.mp4"], "dist")).toStartWith("dist\n"); + }); +}); + +test("sha256File streams the file", async () => { + const dir = tempDir(); + const bytes = new Uint8Array(3 * 1024 * 1024 + 7).map((_, i) => i % 251); + writeFileSync(join(dir, "f.bin"), bytes); + expect(await sha256File(join(dir, "f.bin"))).toBe(new Bun.CryptoHasher("sha256").update(bytes).digest("hex")); +}); + +describe("parseMediaRefs", () => { + const sha = "a".repeat(64); + test("accepts the schema", () => { + expect(parseMediaRefs(JSON.stringify({ schema: 1, files: [{ path: "media/a.mp4", size: 3, sha256: sha }] }))).toEqual([{ path: "media/a.mp4", size: 3, sha256: sha }]); + }); + test.each([ + "not json", "null", "{}", JSON.stringify({ schema: 2, files: [] }), + JSON.stringify({ schema: 1, files: [{ path: "../x.mp4", size: 1, sha256: sha }] }), + JSON.stringify({ schema: 1, files: [{ path: "/abs.mp4", size: 1, sha256: sha }] }), + JSON.stringify({ schema: 1, files: [{ path: ".git/config", size: 1, sha256: sha }] }), + JSON.stringify({ schema: 1, files: [{ path: "a.mp4", size: -1, sha256: sha }] }), + JSON.stringify({ schema: 1, files: [{ path: "a.mp4", size: 1, sha256: "zz" }] }), + ])("rejects %p", (text) => { + expect(parseMediaRefs(text)).toBeNull(); + }); +});