diff --git a/apps/server/test/studio-import-e2e.test.ts b/apps/server/test/studio-import-e2e.test.ts new file mode 100644 index 0000000..9a4da26 --- /dev/null +++ b/apps/server/test/studio-import-e2e.test.ts @@ -0,0 +1,153 @@ +// End to end for v1.1 S: import -> make editable in the studio -> edit -> publish, through the production feature set, +// on SQLite and Postgres. The learner must see the change and keep progress and notes; media must still answer Range. +// Synthetic material only (ffmpeg test pattern videos). +import { describe, expect, test } from "bun:test"; +import { existsSync, mkdirSync, mkdtempSync, statSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { loadConfig } from "@stc/core"; +import { makeVideo } from "../../../packages/feature-import/test/fixtures"; +import { packageMigrations, productFeatures } from "../src/features"; +import { startServer } from "../src/main"; +import { PG, pgSchema } from "./pg"; + +const ORIGIN = "http://localhost:3000"; +const cookieFrom = (r: Response) => r.headers.getSetCookie().map((c) => c.split(";")[0]).filter((c) => !c.endsWith("=")).join("; "); +const CLIP1 = "01 Intro – Übersicht.mp4"; +const CLIP2 = "02 Next step.mp4"; + +async function run(databaseUrl: string) { + const dataDir = mkdtempSync(join(tmpdir(), "stc-e2e-si-")); + const inbox = join(dataDir, "import-inbox"); + const folder = join(inbox, "Synthetic Course"); + mkdirSync(folder, { recursive: true }); + await makeVideo(join(folder, CLIP1), 2); + await makeVideo(join(folder, CLIP2), 1); + const config = loadConfig({ + STC_PORT: "0", STC_DATABASE_URL: databaseUrl, STC_SECRET: "e2e-secret-0123456789abcdef0123456789", + STC_DATA_DIR: dataDir, STC_COURSE_DIRS: "", STC_PUBLIC_URL: ORIGIN, STC_IMPORT_ROOTS: inbox, + }); + const s = await startServer(config, productFeatures({ studioWatch: false }), { migrations: packageMigrations }); + const base = s.server.url.toString().replace(/\/$/, ""); + type Opts = { cookie?: string; body?: unknown; headers?: Record }; + const call = (method: string, path: string, opts: Opts = {}) => { + const headers: Record = { origin: ORIGIN, ...opts.headers }; + if (opts.cookie) headers.cookie = opts.cookie; + if (opts.body !== undefined) headers["content-type"] = "application/json"; + return fetch(base + path, { method, headers, body: opts.body === undefined ? undefined : JSON.stringify(opts.body) }); + }; + const json = async (method: string, path: string, opts: Opts = {}) => (await call(method, path, opts)).json() as Promise; + const signUp = async (email: string, name: string) => { + const r = await call("POST", "/api/auth/sign-up/email", { body: { email, name, password: "correct horse battery" } }); + expect(r.status).toBe(200); + return cookieFrom(r); + }; + const waitJob = async (path: string, cookie: string) => { + for (let i = 0; i < 600; i++) { + const j = await json("GET", path, { cookie }); + if (j.status === "done" || j.status === "failed") return j; + await Bun.sleep(50); + } + throw new Error(`timeout waiting for ${path}`); + }; + + try { + const owner = await signUp("olive@example.org", "Olive Owner"); + const lea = await signUp("lea@example.org", "Lea Learner"); + const leaId = (await json("GET", "/api/me", { cookie: lea })).user.id as string; + + // 1. import a folder of videos (restricted by default) + const imp = await call("POST", "/api/import/jobs", { cookie: owner, body: { kind: "video-folder", path: folder, options: { courseId: "synth", title: "Synthetic Course" } } }); + expect(imp.status).toBe(202); + const impId = ((await imp.json()) as { id: string }).id; + let done: any = null; + for (let i = 0; i < 600 && !done; i++) { + const j = ((await json("GET", "/api/import/jobs", { cookie: owner })) as any[]).find((x) => x.id === impId); + if (j && (j.status === "done" || j.status === "failed")) done = j; + else await Bun.sleep(50); + } + expect(done?.error ?? null).toBeNull(); + expect(done.status).toBe("done"); + expect(existsSync(join(dataDir, "courses/synth/stcourse.yaml"))).toBe(true); + + // 2. the owner grants the learner access; the learner makes progress and writes a note + expect((await call("PUT", "/api/admin/courses/synth/access", { cookie: owner, body: { acl: [{ principal_type: "user", principal_id: leaId }] } })).status).toBe(200); + const before = await json("GET", "/api/courses/synth", { cookie: lea }); + expect(before.modules[0].lessons.map((l: { id: string; title: string }) => [l.id, l.title])).toEqual([["l1", "Intro – Übersicht"], ["l2", "Next step"]]); + expect((await call("PUT", "/api/courses/synth/progress/l1", { cookie: lea, body: { state: "done" } })).status).toBe(200); + expect((await call("PUT", "/api/courses/synth/progress/l2", { cookie: lea, body: { state: "in-progress", positionSeconds: 1 } })).status).toBe(200); + expect((await call("POST", "/api/courses/synth/notes", { cookie: lea, body: { lessonId: "l2", anchorKind: "video", anchorValue: 1, body: "Remember this bit" } })).status).toBe(201); + const progressBefore = await json("GET", "/api/courses/synth/progress", { cookie: lea }); + const notesBefore = await json("GET", "/api/courses/synth/notes", { cookie: lea }); + expect(progressBefore.l1.state).toBe("done"); + expect(notesBefore.map((n: { body: string }) => n.body)).toEqual(["Remember this bit"]); + + // 3. only the owner may convert; the learner's attempt changes nothing + expect((await call("POST", "/api/studio/courses/synth/convert", { cookie: lea })).status).toBe(403); + const ro = await json("GET", "/api/studio/courses/synth", { cookie: owner }); + expect(ro).toMatchObject({ readOnly: true, imported: true, convertible: true }); + const conv = await call("POST", "/api/studio/courses/synth/convert", { cookie: owner }); + expect(conv.status).toBe(202); + const job = await waitJob(`/api/studio/jobs/${((await conv.json()) as { jobId: string }).jobId}`, owner); + expect(job.error).toBeNull(); + expect(job.result).toMatchObject({ courseId: "synth", media: 2 }); + expect(existsSync(join(dataDir, "courses/synth"))).toBe(false); + expect(existsSync(join(dataDir, "studio/repos/synth.git/HEAD"))).toBe(true); + const pubClip = join(dataDir, "worktrees/synth/published/media", CLIP1); + expect(statSync(pubClip).ino).toBe(statSync(join(dataDir, "media/synth/media", CLIP1)).ino); + + // 4. edit a lesson title in the studio and publish + const st = await json("GET", "/api/studio/courses/synth", { cookie: owner }); + expect(st.readOnly).toBe(false); + const m = structuredClone(st.manifestRaw); + expect(m.visibility).toBe("restricted"); + m.modules[0].lessons[0].title = "Welcome aboard"; + expect((await call("PUT", "/api/studio/courses/synth/files", { cookie: owner, body: { message: "Rename the first lesson", manifest: m, manifestBaseHash: st.manifestHash } })).status).toBe(200); + // drafts never reach learners + expect((await json("GET", "/api/courses/synth", { cookie: lea })).modules[0].lessons[0].title).toBe("Intro – Übersicht"); + const pub = await call("POST", "/api/studio/courses/synth/publish", { cookie: owner }); + expect(pub.status).toBe(200); + expect(await pub.json()).toMatchObject({ ok: true, published: true }); + + // 5. the learner sees the change and keeps progress and notes + const after = await json("GET", "/api/courses/synth", { cookie: lea }); + expect(after.modules[0].lessons.map((l: { title: string }) => l.title)).toEqual(["Welcome aboard", "Next step"]); + expect(await json("GET", "/api/courses/synth/progress", { cookie: lea })).toEqual(progressBefore); + expect(await json("GET", "/api/courses/synth/notes", { cookie: lea })).toEqual(notesBefore); + expect(after.progress.l1.state).toBe("done"); + + // 6. media still play: Range gives 206, an unsatisfiable range 416; the URL is unchanged + const lesson = await json("GET", "/api/courses/synth/lessons/l1", { cookie: lea }); + expect(lesson.lesson.title).toBe("Welcome aboard"); + expect(lesson.video.file).toBe(`media/${CLIP1}`); + const size = statSync(pubClip).size; + const part = await call("GET", lesson.video.url, { cookie: lea, headers: { range: "bytes=0-99" } }); + expect(part.status).toBe(206); + expect(part.headers.get("content-range")).toBe(`bytes 0-99/${size}`); + expect((await part.arrayBuffer()).byteLength).toBe(100); + expect((await call("GET", lesson.video.url, { cookie: lea, headers: { range: `bytes=${size + 10}-` } })).status).toBe(416); + const full = await call("GET", lesson.video.url, { cookie: lea }); + expect(full.status).toBe(200); + expect((await full.arrayBuffer()).byteLength).toBe(size); + + // 7. still restricted: anonymous users and other accounts see nothing + expect((await call("GET", "/api/courses/synth")).status).toBe(404); + expect((await call("GET", lesson.video.url)).status).toBe(404); + const other = await signUp("max@example.org", "Max Other"); + expect((await call("GET", "/api/courses/synth", { cookie: other })).status).toBe(404); + expect((await call("GET", "/api/media/synth/..%2F..%2Fmedia%2Fsynth%2Fstcourse.yaml", { cookie: lea })).status).toBe(404); + } finally { + await s.stop(); + } +} + +describe("studio import end to end", () => { + test("import, make editable, edit, publish [sqlite]", () => run("sqlite://:memory:"), 120_000); + const pg = PG; + if (pg) { + test("import, make editable, edit, publish [postgres]", async () => { + const db = await pgSchema(pg); + try { await run(db.url); } finally { await db.drop(); } + }, 120_000); + } +}); diff --git a/docs/contract-requests/studio.md b/docs/contract-requests/studio.md index 6fb10c8..f2511f4 100644 --- a/docs/contract-requests/studio.md +++ b/docs/contract-requests/studio.md @@ -22,7 +22,7 @@ Layout after the review (controller ruling on I4): Proposal: main.ts calls `createCourseRepository(studioCourseDirs(config))`, or `loadConfig` appends both dirs. -Imported courses (`source.kind` other than `authored`, with their root under `/courses`) are read-only in the studio for v1, with the message "Imported course: read-only in the studio for now". There is a `TODO(studio)` in `feature.ts` (`readOnlyMessage`) for converting one into a studio-managed repo. The studio refuses to create a course whose id already exists as `/courses/`. WP-B should likewise refuse an import id that `/studio/repos/.git` already uses. +Imported courses (`source.kind` other than `authored`, with their root under `/courses`) are read-only in the studio for v1, with the message "Imported course: read-only in the studio for now". There was a `TODO(studio)` in `feature.ts` (`readOnlyMessage`) for converting one into a studio-managed repo; v1.1 S resolved it with the owner-only "Make editable in the studio" action (see `docs/operations.md`, "Imported courses in the studio"). The studio refuses to create a course whose id already exists as `/courses/`. WP-B should likewise refuse an import id that `/studio/repos/.git` already uses. ## 3. Live preview: swap in the learner `LessonView` diff --git a/docs/operations.md b/docs/operations.md index 673de68..fc88288 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -22,7 +22,7 @@ Mounts (all course mounts are read-only; STCourses never changes your originals) | Host | Container | What | |---|---|---| -| volume `stc-data` | `/data` | database (SQLite), imported courses (`/data/courses`), studio repos (`/data/studio/repos`) and their published worktrees (`/data/worktrees`), caches (`/data/cache`), AI drafts for non-studio courses (`/data/drafts`) | +| volume `stc-data` | `/data` | database (SQLite), imported courses (`/data/courses`), studio repos (`/data/studio/repos`) and their published worktrees (`/data/worktrees`), media of imports made editable in the studio (`/data/media`), caches (`/data/cache`), AI drafts for non-studio courses (`/data/drafts`) | | `STC_COURSES_HOST` (default `./courses`) | `/courses` | course repos, one folder with a `stcourse.yaml` per course | | `STC_IMPORT_HOST` (default `./import-inbox`) | `/import` | files the owner may import (Packt zips, video folders, PDFs, code zips) | @@ -78,6 +78,81 @@ Without Docker: `bun install`, then `STC_SECRET=… NODE_ENV=production bun apps - Course media (`/api/media/…`) is sent with `X-Content-Type-Options: nosniff`, and SVG, HTML and JavaScript files with `Content-Security-Policy: sandbox`, so author content cannot run script on the app's origin. +## Imported courses in the studio + +Imported courses (`/courses/`) are read-only in the studio until the owner chooses **Make editable in the +studio** on the course's studio page. That runs a background job (`studio.convert-import`) which: + +1. commits the manifest, lessons, transcripts, small images, small PDFs and code into a new studio repo, + `/studio/repos/.git`, checked out as `/worktrees//published`. The manifest is committed + unchanged: the course keeps its id, its visibility (restricted after an import) and its `source.kind`, so only the + owner can later make it public or signed-in. ACL entries, progress and notes are keyed by course and lesson id + and are kept; +2. keeps **large media out of git**: every video or audio file (`mp4 m4v webm mov mkv avi mp3 m4a wav ogg oga opus + flac`) and any other file over 8 MiB. Each is listed with its path, size and SHA-256 in `.stcourses/media.json` + in the repo. They are kept out of `git status` by the repo's `info/exclude` (shared by every worktree), not by + the course's `.gitignore`, and the studio never counts a listed file as a change. So an author or a remote + rewriting `.gitignore` cannot block publishing or Sync. Git LFS is not used: it is not in the image and would + need an LFS server; +3. **moves the import folder to `/media/`** (decided over keeping it in place: a manifest left in + `/courses/` would be found as a second course with the same id, and the importer's "add to course" and + transcription jobs would keep changing a folder learners no longer see). `/media` is never scanned for + courses and nothing writes to it afterwards. File permissions are left alone, because an import made in `link` + mode shares its files with your originals; +4. puts every referenced file into the published worktree, and later into each author's draft worktree, as a + **hard link** to `/media//`. The manifest paths are unchanged, so `/api/media//…` serves the + same URLs with Range support, and no extra disk space is used. If `/worktrees` and `/media` are on + different filesystems, each worktree gets a **full copy** instead: the published one plus one per author who + opens a draft. Keep the data directory on one filesystem. + +In the studio, referenced media cannot be replaced or deleted (save, upload and conflict resolution refuse them with +409); upload a new file under another name instead. A studio write never goes through a hard link: a file with more +than one link is replaced, not overwritten. Publish and Sync refuse a commit that tracks a file at a referenced media +path, since it would silently replace the media. + +**Importer jobs and conversion.** Both use the same per-course lock. Conversion is refused (409, "an import job for +this course is still running") while an importer job for that course is queued or running. An importer job that +starts after the conversion fails with "this course is now edited in the studio" and writes nothing. + +**Crash safety.** The import folder is first renamed to `/courses/.converting-` (dot-folders are never +scanned), before the published worktree is checked out, so the course is never loaded twice. Until that checkout is +done, a failure undoes everything. After it, the conversion is committed and only the move remains: + +- on one filesystem, one `rename` to `/media/`; +- across filesystems, a copy into `/media/.staging-.`, `fsync`, a size and SHA-256 check against + `media.json`, a `rename` into place, and then the source is renamed to `/courses/.converted-.` and + deleted. + +At server start and before every conversion, STCourses recovers each course deterministically: + +- staging folders are deleted; +- a `.converting-` folder without a finished repo is renamed back (the next attempt starts over); +- a `.converting-` folder with a finished repo has its move finished; +- `.converted-` folders are deleted once `/media/` is complete. + +A job left running by a restart is requeued, and re-running a finished conversion reports success. + +Things to know: + +- Back up `/media` together with `/studio`: the repo alone does not contain the videos. +- **Sync** pushes only the git repo. A clone of the remote has the manifest, lessons and `.stcourses/media.json`, but + not the media files themselves; copy `/media/` alongside if you move the course to another server. +- If a worktree loses a media file (for example after a manual `git clean -x`), the next publish links it back into + the published worktree, and a new draft worktree gets every file. A file missing from `/media/` shows + as a course diagnostic ("lesson file not found"), never a crash. +- After conversion the importer can no longer add decks, code or transcripts to that course (it only edits + `/courses`). Add them in the studio. +- `/media` is fixed under the data directory (there is no separate setting). +- **Discovery skips dot-folders.** Every course directory (`STC_COURSE_DIRS`, `/courses`, `/worktrees`) + is scanned to depth 2 for `stcourse.yaml`, and folders whose name starts with `.` are never entered. Do not give a + course folder a name that starts with a dot. The conversion's `.converting-` and `.converted-.` folders + rely on this. +- **A failed conversion.** If the media move fails after the commit (disk full, or a copy that fails its checksum), + learners are switched to the studio course right away; its media are hard links into the hidden import folder. + The course's studio page shows **Conversion incomplete** with a **Retry** button for the owner. A retry (or the + next server start) finishes the move. A failure before the commit is rolled back, and learners keep the import + unchanged. + ## Memory budget The deployment must stay under 2 GB idle. Measured with this compose file (`--profile postgres`), the image built diff --git a/docs/superpowers/plans/2026-10-30-v11-studio-import.md b/docs/superpowers/plans/2026-10-30-v11-studio-import.md new file mode 100644 index 0000000..37b4123 --- /dev/null +++ b/docs/superpowers/plans/2026-10-30-v11-studio-import.md @@ -0,0 +1,69 @@ +# STCourses v1.1 — S: Edit imported courses in the studio + +> **For agentic workers:** executed inline (superpowers:executing-plans) by the S package owner, TDD, one commit per task. Steps use checkbox (`- [ ]`) syntax. + +**Goal:** an owner-only action, "Make editable in the studio", turns an imported course (`/courses/`) into a studio course (`/studio/repos/.git` plus `/worktrees//published`) with the same id, so authors can edit and publish it and learners keep their progress and notes. Resolves `TODO(studio): convert imported course to a studio-managed repo` (ruling R9). + +**Requirements:** `docs/superpowers/plans/2026-10-30-v1-1.md` ("Global Constraints", package S); spec sections 5, 8, 9.1, 12, 13; `CLAUDE.md` project rules. + +## Decisions + +1. **Large media stay out of git, referenced by path, size and checksum (spec 5).** No Git LFS (not installed in the image, and it would need a remote). A file is "large media" when its extension is video/audio (`mp4 m4v webm mov mkv avi mp3 m4a wav ogg oga opus flac`) or it is over 8 MiB. Everything else (manifest, Markdown, small images, small PDFs, code) is committed. + - The repo gets `.stcourses/media.json` (`{ schema: 1, files: [{ path, size, sha256 }] }`) and a root `.gitignore` with one anchored, escaped line per referenced path (an existing root `.gitignore` is kept above our block). + - Every studio worktree (published and each author's draft) gets the referenced files materialised at their manifest paths as **hard links** to the media store (copy with `--reflink=auto` across filesystems). Manifest paths do not change; core's media route (Range 206/416, 404 outside the root) serves them unchanged. No symlinks: the loader's `resolveInside` and the studio's write rules both refuse paths that leave the course root. +2. **The import folder is moved to `/media/`** (a `rename`, same filesystem) rather than kept in place. Reasons: a manifest left in `/courses/` would be discovered as a second course with the same id (and would win: imported dirs are scanned before worktrees); the importer's "add to course" and transcription jobs would keep editing a folder learners no longer see. `/media` is never scanned. Nothing writes there after the move; permissions are not changed, because the importer's `link` mode shares inodes with the owner's originals. +3. **Conversion runs as a job** (`studio.convert-import`), because hashing gigabytes of video can exceed Bun's 10 s idle timeout. The route checks everything it can first and answers `202 { jobId }`; the UI polls `GET /api/studio/jobs/:jobId`. The job is restart-safe: a conversion interrupted before the move is rolled back and redone; one interrupted after the repo exists only finishes the move. +4. **Same id, same manifest.** The manifest is committed unchanged: `visibility` (restricted by default for imports) and `source.kind` stay, so the v1 owner-only rule at publish still applies (only the owner can make it public or signed-in, or change `source.kind`). ACL rows, progress and notes are keyed by course and lesson ids, which do not change. +5. **Referenced media are read-only in the studio.** Saves, uploads, deletes and conflict resolutions that touch a referenced path or `.stcourses/media.json` get 409 with a message to upload under a new name. As defence in depth, any studio write to a file with more than one hard link unlinks it first, so a write can never reach a shared inode. +6. **Hostile input:** symlinks and special files in the import folder are skipped (never followed), `.git` folders are skipped, a large file whose name contains a newline (inexpressible in `.gitignore`) refuses the conversion, and the published worktree must be clean after checkout or the conversion rolls back. Materialisation checks each reference with `checkRelPath`, requires a regular file whose real path is inside `/media/`, and writes through `safeWritePath` (refuses symlinks in the destination path). + +## Global Constraints +- Everything in `CLAUDE.md` "Project rules" and the v1.1 "Global Constraints" applies. No DB change is needed (no migration). +- v1 safeguards stay: symlink-refusing writes, the remote allowlist, the owner-only visibility rule at publish, core's CSRF Origin check (applies to the new POST route automatically), repos at `/studio/repos/.git`. +- Synthetic fixtures only. `bun test`, `bun run typecheck` and the Postgres run (`STC_TEST_PG_URL`) stay green. + +## File structure +- Create `packages/feature-studio/src/server/media-refs.ts`: classification, `.gitignore` escaping, hashing, reference file read/write. +- Create `packages/feature-studio/src/server/convert.ts`: `convertImported` (stage, commit, worktree, materialise, move). +- Modify `packages/feature-studio/src/server/store.ts`: `mediaDir` option, `materializeMedia`, draft materialisation, write guards, `listFiles` includes referenced media, `convertImported` wiring. +- Modify `packages/feature-studio/src/server/feature.ts`: `POST /api/studio/courses/:courseId/convert`, `GET /api/studio/jobs/:jobId`, job handler, `convertible` flag; remove the TODO. +- Modify `packages/feature-studio/src/ui/*`: convert panel in the read-only view, imported tag on the studio home. +- Create tests: `packages/feature-studio/test/media-refs.test.ts`, `convert.test.ts`, `convert-routes.test.ts`, `convert-ui.test.tsx`; `apps/server/test/studio-import-e2e.test.ts`. +- Modify `docs/operations.md` (media decision, layout). + +### Task 1: Media references +- [ ] Tests: `isLargeMedia` by extension and size; `gitignoreLine` escapes `\ * ? [ ] # !` and trailing spaces, and `git check-ignore` matches exactly the listed paths for names with spaces, brackets, `#`, `!`, non-ASCII and trailing spaces (and does not match siblings); `sha256File` matches `Bun.CryptoHasher`; `parseMediaRefs` rejects bad shapes. +- [ ] Implement `media-refs.ts`. Run tests, typecheck. Commit. + +### Task 2: Conversion in the store +- [ ] Tests (`convert.test.ts`, synthetic import folder built in the test: manifest, lesson Markdown, a fake `.mp4` with space and non-ASCII name, a > 8 MiB PDF, a nested `.gitignore` that ignores a lesson file, a symlink pointing outside, a `.git` dir): + - the bare repo, published worktree and media store exist; the import folder is gone; `isManaged`; + - git tracks the manifest, lessons (including the nested-ignored one) and small files, never the video or big PDF; the symlink and `.git` contents are not tracked; + - `.stcourses/media.json` lists path, size, sha256; the published worktree is clean; the video in the worktree is a hard link (same inode) to `/media//…`; + - a draft worktree gets the media too; `listFiles` includes them; + - saving, uploading or deleting a referenced path or the reference file is refused (409); a write to a hard-linked file never changes the other link; + - refusals: not under the imports dir, `source.kind: authored`, manifest id mismatch, `root:` set, already a studio course, media store already present; each leaves the import folder untouched and no repo behind; + - resume: a leftover partial repo is rebuilt; a managed course whose import folder still exists only finishes the move; + - publish after an edit keeps the media in the published worktree. +- [ ] Implement `convert.ts` and the store changes. Run studio tests, typecheck. Commit. + +### Task 3: Routes and job +- [ ] Tests (`convert-routes.test.ts`, real job queue from `@stc/core`): + - owner `POST …/convert` gives 202 and a job id; polling reaches `done`; the course is then studio-managed, served from the published worktree, still restricted, with ACL rows unchanged; + - an author or learner gets 403; a foreign Origin gets 403 (CSRF); an overlay or studio course gets 409; an unknown course 404; a second request while one is queued gets 409; + - `GET /api/studio/courses/:id` for an imported course says `convertible: true` for the owner only; + - after conversion a non-owner author can edit and publish a lesson title, but cannot publish `visibility: public` (403); the owner can. +- [ ] Implement. Run tests, typecheck. Commit. + +### Task 4: Studio UI +- [ ] Test (`convert-ui.test.tsx`, server render): the read-only panel shows "Make editable in the studio" and the explanation only when `convertible`; progress and error states render. +- [ ] Implement `ConvertPanel` (button, job polling, then reload into the workbench) and the imported tag on the home list. Existing tokens and classes only; no new colours, fonts or motion. Commit. + +### Task 5: End-to-end +- [ ] `apps/server/test/studio-import-e2e.test.ts`, SQLite and Postgres: production feature set; owner signs up; a synthetic video folder is imported through `/api/import/jobs`; a learner is granted access by ACL, marks a lesson done and writes a note; owner converts; edits a lesson title in the studio; publishes. The learner sees the new title, the same progress and notes, media answer `Range` with 206 and a bad range with 416; anonymous users still get 404. +- [ ] Commit. + +### Task 6: Docs, smoke run, report +- [ ] `docs/operations.md`: the media decision and the new `/media` folder; backups; that media are not pushed to studio remotes. +- [ ] Dev server on port 3492 with `STC_DATA_DIR=/tmp/stc-v11-studio`: import, convert, edit, publish by curl. +- [ ] Full `bun test`, typecheck, Postgres run. Commit. Write the report. diff --git a/docs/superpowers/specs/2026-10-30-stcourses-design.md b/docs/superpowers/specs/2026-10-30-stcourses-design.md index bf00889..0462bc3 100644 --- a/docs/superpowers/specs/2026-10-30-stcourses-design.md +++ b/docs/superpowers/specs/2026-10-30-stcourses-design.md @@ -181,6 +181,7 @@ Phase 0 (section 10) pins down the exact schema. These principles are fixed: - The two reference courses must load by adding only a manifest and front matter, without moving or rewriting their existing files. - Their existing tooling (test scripts, `tools/*.mjs`) must keep working. - **Derived data lives in a cache directory that can be rebuilt from the files:** search index, slide images and text, transcripts, video posters and AI embeddings. +- **Discovery** (v1.1 note): the platform finds `stcourse.yaml` at depth 0–2 under each course directory and skips folders whose name starts with `.` (and `node_modules`). A course folder must therefore not be a dot-folder. The studio relies on this to hide an import folder while it is converted (`.converting-`, `.converted-.`). ## 6. Accounts, roles and visibility diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 80df762..888bba5 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -3,3 +3,4 @@ export { createJobQueue } from "./jobs"; export { serveMedia, contentType, contentDisposition, mediaSecurityHeaders, MEDIA_SANDBOX_CSP } from "./media"; export { createApp, canView, crossSiteRefusal, type Routes } from "./app"; export { composeServer, type ComposeOptions, type ComposedServer } from "./compose"; +export { withCourseLock } from "./locks"; diff --git a/packages/core/src/locks.ts b/packages/core/src/locks.ts new file mode 100644 index 0000000..1a1d720 --- /dev/null +++ b/packages/core/src/locks.ts @@ -0,0 +1,14 @@ +const locks = new Map>(); + +/** + * Serialises work on one course across features in this process: the studio (saves, publish, conversion of an import) + * and the importer's jobs that add to an existing course. A failed holder never blocks the next one. + */ +export function withCourseLock(courseId: string, fn: () => Promise): Promise { + const prev = locks.get(courseId) ?? Promise.resolve(); + const next = prev.then(fn, fn); + const tail = next.catch(() => {}); + locks.set(courseId, tail); + tail.then(() => { if (locks.get(courseId) === tail) locks.delete(courseId); }); + return next; +} diff --git a/packages/course-format/src/load.ts b/packages/course-format/src/load.ts index 579d6ee..b3171f9 100644 --- a/packages/course-format/src/load.ts +++ b/packages/course-format/src/load.ts @@ -39,7 +39,7 @@ export function resolveInside(root: string, rel: string): string | null { } } -/** Finds stcourse.yaml at depth 0-2 under each dir. */ +/** Finds stcourse.yaml at depth 0-2 under each dir. Dot-folders are skipped (hidden, or retired by the studio). */ export async function discoverManifests(courseDirs: string[]): Promise { const found: string[] = []; const seen = new Set(); @@ -60,7 +60,7 @@ export async function discoverManifests(courseDirs: string[]): Promise } entries.sort((a, b) => a.name.localeCompare(b.name)); for (const e of entries) { - if (e.isDirectory() && !SKIP_DIRS.has(e.name)) await walk(join(dir, e.name), depth + 1); + if (e.isDirectory() && !SKIP_DIRS.has(e.name) && !e.name.startsWith(".")) await walk(join(dir, e.name), depth + 1); } } for (const d of courseDirs) await walk(resolve(d), 0); diff --git a/packages/course-format/test/load.test.ts b/packages/course-format/test/load.test.ts index 2a8c0d6..a1528a2 100644 --- a/packages/course-format/test/load.test.ts +++ b/packages/course-format/test/load.test.ts @@ -45,6 +45,13 @@ describe("discovery", () => { const found = (await discoverManifests([d])).map((p) => p.slice(d.length + 1)); expect(found.sort()).toEqual(["a/b/stcourse.yaml", "a/stcourse.yaml", "stcourse.yaml"]); }); + + test("skips dot-folders (hidden or retired course folders, e.g. an import being converted)", async () => { + const d = await tmp(); + for (const p of ["a/stcourse.yaml", ".converting-a/stcourse.yaml", "x/.hidden/stcourse.yaml"]) await Bun.write(join(d, p), "x"); + const found = (await discoverManifests([d])).map((p) => p.slice(d.length + 1)); + expect(found).toEqual(["a/stcourse.yaml"]); + }); }); describe("overlays and errors", () => { diff --git a/packages/feature-import/src/context.ts b/packages/feature-import/src/context.ts index d0eff37..307bd90 100644 --- a/packages/feature-import/src/context.ts +++ b/packages/feature-import/src/context.ts @@ -1,6 +1,7 @@ import { mkdir, readdir, rename, stat } from "node:fs/promises"; import { join, relative, resolve } from "node:path"; import type { ServerContext } from "@stc/contracts"; +import { withCourseLock } from "@stc/core"; import { readRawManifest, writeManifest, type RawManifest } from "./util/manifest"; import { slugify, uniqueId } from "./util/text"; import { extractFrame, type Tools } from "./media"; @@ -65,9 +66,21 @@ export async function publishCourse(d: Deps, stage: string, id: string): Promise export interface ManagedCourse { id: string; dir: string; manifest: RawManifest } +export const STUDIO_OWNED = "this course is now edited in the studio; add files there instead"; + +/** + * Runs importer work that changes an existing course under the course lock shared with the studio (core + * `withCourseLock`), after checking, inside the lock, that the course is still an importer folder. A conversion to a + * studio course therefore either waits for the job, or the job fails cleanly with STUDIO_OWNED and writes nothing. + */ +export function withManagedCourse(d: Deps, courseId: string | undefined, fn: (c: ManagedCourse) => Promise): Promise { + return withCourseLock(courseId ?? "", async () => fn(await managedCourse(d, courseId))); +} + /** An existing course the importer may add to. Only folders directly under /courses qualify: overlays and authored repos are never edited. */ export async function managedCourse(d: Deps, courseId: string | undefined): Promise { if (!courseId) throw new Error("options.courseId is required: choose the course to add to"); + if (isPlainId(courseId) && (await exists(studioRepo(d, courseId)))) throw new Error(STUDIO_OWNED); const dir = join(coursesDir(d), courseId); const rel = relative(coursesDir(d), dir); if (!isPlainId(courseId) || rel.startsWith("..") || !(await exists(join(dir, "stcourse.yaml")))) { diff --git a/packages/feature-import/src/jobs/code-repo.ts b/packages/feature-import/src/jobs/code-repo.ts index 29aff79..3566222 100644 --- a/packages/feature-import/src/jobs/code-repo.ts +++ b/packages/feature-import/src/jobs/code-repo.ts @@ -1,7 +1,7 @@ import { cp, lstat, mkdir, readdir } from "node:fs/promises"; import { basename, extname, join } from "node:path"; import type { ImportResult, JobContext } from "@stc/contracts"; -import { finish, managedCourse, saveManifest, stageDir, type Deps } from "../context"; +import { finish, saveManifest, stageDir, withManagedCourse, type Deps, type ManagedCourse } from "../context"; import { attachDeck } from "../deck"; import { putFile } from "../media"; import { run } from "../util/exec"; @@ -48,9 +48,13 @@ async function findPdfs(dir: string, depth = 0, out: string[] = []): Promise { - const { path, options } = parsePayload(payload, "code-repo"); + const { options } = parsePayload(payload, "code-repo"); + return withManagedCourse(d, optStr(options, "courseId"), (course) => importCodeRepoInto(d, course, payload, jc)); +} + +async function importCodeRepoInto(d: Deps, course: ManagedCourse, payload: unknown, jc: JobContext): Promise { + const { path } = parsePayload(payload, "code-repo"); const warnings: string[] = []; - const course = await managedCourse(d, optStr(options, "courseId")); if (!GIT_URL.test(path)) await assertAllowedPath(d, path); const stage = await stageDir(d, "repo"); diff --git a/packages/feature-import/src/jobs/pdf-deck.ts b/packages/feature-import/src/jobs/pdf-deck.ts index 855815a..0b96464 100644 --- a/packages/feature-import/src/jobs/pdf-deck.ts +++ b/packages/feature-import/src/jobs/pdf-deck.ts @@ -1,7 +1,7 @@ import { stat } from "node:fs/promises"; import { join } from "node:path"; import type { ImportResult, JobContext } from "@stc/contracts"; -import { finish, managedCourse, saveManifest, type Deps } from "../context"; +import { finish, saveManifest, withManagedCourse, type Deps } from "../context"; import { assertAllowedPath } from "../roots"; import { attachDeck } from "../deck"; import { optStr, parsePayload } from "./common"; @@ -11,7 +11,7 @@ export async function importPdfDeck(d: Deps, payload: unknown, jc: JobContext): const warnings: string[] = []; await assertAllowedPath(d, path); if (!(await stat(path).catch(() => null))?.isFile() || !/\.pdf$/i.test(path)) throw new Error(`not a PDF file: ${path}`); - const course = await managedCourse(d, optStr(options, "courseId")); + return withManagedCourse(d, optStr(options, "courseId"), async (course) => { const attachTo = optStr(options, "attachTo"); if (!attachTo) throw new Error("options.attachTo is required: the lesson or module id the deck belongs to"); await jc.reportProgress(0.05, "Copying the deck"); @@ -25,4 +25,5 @@ export async function importPdfDeck(d: Deps, payload: unknown, jc: JobContext): await finish(d, course.id, warnings); await jc.reportProgress(1, r.pages ? `Deck attached (${r.pages} slides)` : "Deck attached"); return { courseId: course.id, manifestPath: join(course.dir, "stcourse.yaml"), warnings }; + }); } diff --git a/packages/feature-import/src/jobs/transcribe.ts b/packages/feature-import/src/jobs/transcribe.ts index dfbafb4..38cccaa 100644 --- a/packages/feature-import/src/jobs/transcribe.ts +++ b/packages/feature-import/src/jobs/transcribe.ts @@ -1,7 +1,7 @@ import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; import type { JobContext } from "@stc/contracts"; -import { finish, managedCourse, saveManifest, stageDir, transcriptCacheDir, type Deps } from "../context"; +import { finish, saveManifest, stageDir, transcriptCacheDir, withManagedCourse, type Deps, type ManagedCourse } from "../context"; import { run } from "../util/exec"; import { optStr, withStage } from "./common"; @@ -36,7 +36,10 @@ export async function transcribeCourse(d: Deps, payload: unknown, jc: JobContext const url = d.transcribeUrl; const courseId = optStr(p, "courseId") ?? ""; if (!url) return { courseId, skipped: true, reason: "STC_TRANSCRIBE_URL is not set", transcribed: 0, warnings: [] }; - const course = await managedCourse(d, courseId); + return withManagedCourse(d, courseId, (course) => transcribeInto(d, course, url, p, jc)); +} + +async function transcribeInto(d: Deps, course: ManagedCourse, url: string, p: Record, jc: JobContext): Promise { const only = Array.isArray(p.lessonIds) ? new Set(p.lessonIds.map(String)) : null; const lessons = (course.manifest.modules as any[]).flatMap((m) => m.lessons as any[]).filter((l) => l.type === "video" && !l.transcript && (!only || only.has(l.id))); const warnings: string[] = []; diff --git a/packages/feature-import/src/routes.ts b/packages/feature-import/src/routes.ts index 01384eb..0e0cae4 100644 --- a/packages/feature-import/src/routes.ts +++ b/packages/feature-import/src/routes.ts @@ -3,7 +3,7 @@ import { join, relative, resolve } from "node:path"; import type { Feature, RouteHandler } from "@stc/contracts"; import { serveMedia } from "@stc/core"; import { assertAllowedPath, importRoots, OUTSIDE } from "./roots"; -import { coverCacheDir, coursesDir, exists, isPlainId, managedCourse, saveManifest, type Deps } from "./context"; +import { coverCacheDir, coursesDir, exists, isPlainId, saveManifest, STUDIO_OWNED, withManagedCourse, type Deps } from "./context"; export const KINDS = ["packt-video", "code-repo", "pdf-deck", "video-folder"] as const; const OPTION_KEYS = ["title", "courseId", "attachTo", "media", "transcribe"] as const; @@ -90,13 +90,18 @@ export function importRoutes(deps: () => Deps): Feature["routes"] { if (!isPlainId(rc.params.courseId!)) return json({ error: "Invalid course id" }, 400); const src = join(coverCacheDir(d, rc.params.courseId!), name); if (!(await exists(src))) return json({ error: "No such candidate" }, 404); - let course; - try { course = await managedCourse(d, rc.params.courseId); } catch (e) { return json({ error: (e as Error).message }, 404); } - await mkdir(join(course.dir, "media"), { recursive: true }); - await copyFile(src, join(course.dir, "media", "cover.jpg")); - course.manifest.cover = "media/cover.jpg"; - await saveManifest(course.dir, course.manifest); - await d.ctx.courses.reload(course.id); + try { + await withManagedCourse(d, rc.params.courseId, async (course) => { + await mkdir(join(course.dir, "media"), { recursive: true }); + await copyFile(src, join(course.dir, "media", "cover.jpg")); + course.manifest.cover = "media/cover.jpg"; + await saveManifest(course.dir, course.manifest); + await d.ctx.courses.reload(course.id); + }); + } catch (e) { + const msg = (e as Error).message; + return json({ error: msg }, msg === STUDIO_OWNED ? 409 : 404); + } return json({ cover: "media/cover.jpg" }); }), }, diff --git a/packages/feature-import/test/studio-race.test.ts b/packages/feature-import/test/studio-race.test.ts new file mode 100644 index 0000000..4c7e6f7 --- /dev/null +++ b/packages/feature-import/test/studio-race.test.ts @@ -0,0 +1,56 @@ +// v1.1 S review fix: importer jobs and "Make editable in the studio" share the course lock. Either order is safe: +// a job that started first finishes and its files are converted with the course; a job that comes second fails +// cleanly and never recreates /courses/. +import { expect, test } from "bun:test"; +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { $ } from "bun"; +import { loadAll } from "@stc/course-format"; +import { createStudioStore } from "@stc/feature-studio"; +import { STUDIO_OWNED } from "../src/context"; +import { importCodeRepo } from "../src/jobs/code-repo"; +import { importPacktVideo } from "../src/jobs/packt-video"; +import { transcribeCourse } from "../src/jobs/transcribe"; +import { makeEnv } from "./harness"; +import { makePacktZip, makeRepoZip, tmp } from "./fixtures"; + +const owner = { name: "Olive Owner", email: "own@example.org" }; + +async function imported(env: Record = {}) { + const e = await makeEnv(env); + const r = await importPacktVideo(e.deps, { path: await makePacktZip(await tmp()), options: { title: "Race Test" } }, e.jc); + const store = createStudioStore({ dataDir: e.dataDir }); + const count = async () => (await loadAll([join(e.dataDir, "courses"), join(e.dataDir, "worktrees")])).filter((c) => c.id === r.courseId).length; + return { e, id: r.courseId, store, count }; +} + +test("an importer job that started first finishes; the conversion waits and commits its files", async () => { + const { e, id, store, count } = await imported(); + const zip = await makeRepoZip(await tmp()); + const job = importCodeRepo(e.deps, { kind: "code-repo", path: zip, options: { courseId: id } }, e.jc); + const conv = store.convertImported(id, owner); + const [res] = await Promise.all([job, conv]); + expect(res.courseId).toBe(id); + const pub = store.paths(id).published; + const files = (await $`git -c core.quotepath=false ls-files`.cwd(pub).quiet()).stdout.toString(); + expect(files).toContain("code/S02 - Images (Sample)/app/main.py"); + expect(await Bun.file(join(pub, "stcourse.yaml")).text()).toContain("code/S02 - Images (Sample)"); + expect(existsSync(join(e.dataDir, "courses", id))).toBe(false); + expect(await count()).toBe(1); +}); + +test("an importer job that comes after the conversion fails cleanly and writes nothing", async () => { + const { e, id, store, count } = await imported({ STC_TRANSCRIBE_URL: "http://127.0.0.1:9/never" }); + const zip = await makeRepoZip(await tmp()); + const conv = store.convertImported(id, owner); + const job = importCodeRepo(e.deps, { kind: "code-repo", path: zip, options: { courseId: id } }, e.jc); + const tr = transcribeCourse(e.deps, { courseId: id }, e.jc); + const [c, j, t] = await Promise.allSettled([conv, job, tr]); + expect(c.status).toBe("fulfilled"); + expect(j.status === "rejected" && (j.reason as Error).message).toBe(STUDIO_OWNED); + expect(t.status === "rejected" && (t.reason as Error).message).toBe(STUDIO_OWNED); + expect(existsSync(join(e.dataDir, "courses", id))).toBe(false); + expect(await count()).toBe(1); + // and later ones too + await expect(importCodeRepo(e.deps, { kind: "code-repo", path: zip, options: { courseId: id } }, e.jc)).rejects.toThrow(STUDIO_OWNED); +}); diff --git a/packages/feature-studio/src/index.ts b/packages/feature-studio/src/index.ts index 6ced9bb..4889acf 100644 --- a/packages/feature-studio/src/index.ts +++ b/packages/feature-studio/src/index.ts @@ -1,5 +1,5 @@ // Server entry of @stc/feature-studio. The browser imports "@stc/feature-studio/web" instead (no server code). -export { studioFeature, OUTSIDE_MESSAGE, IMPORTED_MESSAGE, uploadMaxBytes, UPLOAD_EXTENSIONS, type StudioFeature, type StudioFeatureOptions } from "./server/feature"; +export { studioFeature, OUTSIDE_MESSAGE, IMPORTED_MESSAGE, CONVERT_JOB, uploadMaxBytes, UPLOAD_EXTENSIONS, type StudioFeature, type StudioFeatureOptions } from "./server/feature"; export { createDraftWriter, type DraftWriter, type DraftWriterOptions } from "./server/draft-writer"; export { createStudioStore, contentHash, isAllowedRemote, ownerRequiredFor, type StudioStore, type Identity, type DraftStatus, type PublishResult, type SyncResult } from "./server/store"; export { createPublishedWatcher, type PublishedWatcher } from "./server/watch"; diff --git a/packages/feature-studio/src/server/convert.ts b/packages/feature-studio/src/server/convert.ts new file mode 100644 index 0000000..8e07f0b --- /dev/null +++ b/packages/feature-studio/src/server/convert.ts @@ -0,0 +1,77 @@ +import { constants } from "node:fs"; +import { copyFile, link, mkdir, open, readdir, stat } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { $ } from "bun"; +import { gitignoreLine, isLargeMedia } from "./media-refs"; +import { checkRelPath } from "./relpath"; + +export interface ScannedImport { + /** Committed into git. */ + small: string[]; + /** Kept out of git, referenced by path, size and checksum. */ + large: { path: string; size: number }[]; + /** Skipped and never followed: symlinks, sockets, `.git` folders. */ + skipped: string[]; +} + +/** Walks an import folder with lstat semantics: symlinks and special files are skipped, never followed. */ +export async function scanImport(root: string): Promise { + const out: ScannedImport = { small: [], large: [], skipped: [] }; + const walk = async (rel: string) => { + const entries = await readdir(join(root, rel), { withFileTypes: true }); + entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + for (const e of entries) { + const r = rel ? `${rel}/${e.name}` : e.name; + if (e.name.toLowerCase() === ".git") { out.skipped.push(r); continue; } + if (e.isDirectory()) await walk(r); + else if (e.isFile()) { + const size = (await stat(join(root, r))).size; + if (isLargeMedia(r, size)) out.large.push({ path: r, size }); + else out.small.push(r); + } else out.skipped.push(r); + } + }; + await walk(""); + return out; +} + +/** Problems that make a large file impossible to reference: a name git ignore files or the studio cannot express. */ +export function unreferencable(large: { path: string }[]): string[] { + const bad: string[] = []; + for (const f of large) { + try { gitignoreLine(f.path); if (checkRelPath(f.path) !== f.path) throw new Error("not normal"); } catch { bad.push(f.path); } + } + return bad; +} + +/** Test-only marker for a simulated crash: cleanup code must not run for it (a real crash would not run it either). */ +export class SimulatedCrash extends Error { + constructor(public step: string) { super(`simulated crash at ${step}`); } +} + +/** A hard link (no extra space), else a reflink or plain copy when linking is impossible (another filesystem, no + * permission, too many links). Other errors (the destination exists, the source vanished) are real failures. */ +export async function linkOrCopy(src: string, dest: string): Promise { + await mkdir(dirname(dest), { recursive: true }); + try { + await link(src, dest); + return; + } catch (e) { + if (!["EXDEV", "EPERM", "EMLINK", "ENOTSUP", "EOPNOTSUPP"].includes((e as NodeJS.ErrnoException).code ?? "")) throw e; + } + const r = await $`cp --reflink=auto -- ${src} ${dest}`.quiet().nothrow(); + if (r.exitCode !== 0) await copyFile(src, dest, constants.COPYFILE_EXCL); +} + +/** fsyncs every file and directory under `root` (or only `root` itself with `shallow`), so a rename after it is durable. */ +export async function fsyncTree(root: string, o: { shallow?: boolean } = {}): Promise { + const sync = async (p: string) => { const fh = await open(p, "r"); try { await fh.sync(); } finally { await fh.close(); } }; + if (!o.shallow) { + for (const e of await readdir(root, { withFileTypes: true })) { + const p = join(root, e.name); + if (e.isDirectory()) await fsyncTree(p); + else if (e.isFile()) await sync(p); + } + } + await sync(root); +} diff --git a/packages/feature-studio/src/server/feature.ts b/packages/feature-studio/src/server/feature.ts index 9d1d4ca..5f819b6 100644 --- a/packages/feature-studio/src/server/feature.ts +++ b/packages/feature-studio/src/server/feature.ts @@ -1,16 +1,17 @@ import { existsSync, realpathSync } from "node:fs"; import { basename, extname, resolve, sep } from "node:path"; -import type { AuthUser, Feature, RequestContext, RouteHandler, ServerContext } from "@stc/contracts"; +import type { AuthUser, Feature, JobHandler, RequestContext, RouteHandler, ServerContext } from "@stc/contracts"; import { canView, serveMedia } from "@stc/core"; import { ManifestSchema, parseManifest, resolveInside, type LoadedCourse } from "@stc/course-format"; import { aclRepo } from "@stc/db"; import { createDraftWriter, type DraftWriter } from "./draft-writer"; import { storeDirs, studioWorktreesDir } from "./paths"; -import { checkRelPath, createStudioStore, MANIFEST, ownerRequiredFor, StudioError, type FileWrite, type StudioStore } from "./store"; +import { checkRelPath, createStudioStore, MANIFEST, ownerRequiredFor, StudioError, type FileWrite, type StudioStore, type StudioStoreOptions } from "./store"; import { createPublishedWatcher, type PublishedWatcher } from "./watch"; export const OUTSIDE_MESSAGE = "This course is managed outside STCourses"; -export const IMPORTED_MESSAGE = "Imported course: read-only in the studio for now"; +export const IMPORTED_MESSAGE = "Imported course: not editable in the studio until the owner makes it editable"; +export const CONVERT_JOB = "studio.convert-import"; /** Upload cap in bytes from AppConfig.uploadMaxMb (STC_UPLOAD_MAX_MB, default 100; review I2). */ export function uploadMaxBytes(mb: number | undefined): number { @@ -44,9 +45,12 @@ async function readCapped(req: Request, max: number): Promise; store: StudioStore; watcher: PublishedWatcher; draftWriter: DraftWriter; + /** Resolves once interrupted conversions are recovered (every studio route waits for it). */ + ready: Promise; stop(): void; } @@ -54,6 +58,8 @@ export interface StudioFeatureOptions { /** Poll the published worktrees for external commits (default true). */ watch?: boolean; watchIntervalMs?: number; + /** Tests only: conversion failure injection (see StudioStoreOptions.testHooks). */ + storeTestHooks?: StudioStoreOptions["testHooks"]; } const json = (body: unknown, status = 200) => Response.json(body, { status }); @@ -72,7 +78,7 @@ function manifestToYaml(m: unknown): string { } export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = {}): StudioFeature { - const store = createStudioStore(storeDirs(ctx.config)); + const store = createStudioStore({ ...storeDirs(ctx.config), testHooks: opts.storeTestHooks }); const watcher = createPublishedWatcher(store, ctx.courses, { intervalMs: opts.watchIntervalMs }); const acl = aclRepo(ctx.db); const wtDir = resolve(ctx.config.studioWorktreesDir ?? studioWorktreesDir(ctx.config.dataDir)); @@ -99,24 +105,59 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = { const readOnlyMessage = (c: LoadedCourse): string => { const kind = c.manifest?.source?.kind; const underImports = c.root === importedRoot || c.root.startsWith(importedRoot + sep); - if (kind && kind !== "authored" && underImports) { - // TODO(studio): convert imported course to a studio-managed repo (git init into /studio/repos/.git - // plus a published worktree), so authors can edit it and the owner-only visibility rule applies on publish. - return IMPORTED_MESSAGE; - } + if (kind && kind !== "authored" && underImports) return IMPORTED_MESSAGE; return OUTSIDE_MESSAGE; }; + const isOwner = (u: AuthUser) => u.roles.includes("owner"); + /** v1.1 S: only the owner may turn an imported course into a studio course ("Make editable in the studio"). */ + const convertible = (u: AuthUser, courseId: string) => isOwner(u) && store.isConvertibleImport(courseId); + + /** Importer jobs (queued or running) that would write into this course's import folder. */ + const pendingImports = async (courseId: string) => + (await ctx.jobs.list()).filter((j) => { + if (!j.kind.startsWith("import.") || (j.status !== "queued" && j.status !== "running")) return false; + const p = (j.payload ?? {}) as { courseId?: unknown; options?: { courseId?: unknown } }; + return p.courseId === courseId || p.options?.courseId === courseId; + }); + /** 409 text naming the blocking importer jobs (id and kind), so the owner can find them on the import page. */ + const importRunning = (jobs: { id: string; kind: string }[]) => + `an import job for this course is still running (${jobs.map((j) => `${j.kind} ${j.id}`).join(", ")}); make it editable once it has finished`; + + /** The conversion job: payload `{ courseId, actor }`, checked again by the store under the course lock. */ + const convertJob: JobHandler = async (payload, jc) => { + await ready; + const p = (payload ?? {}) as { courseId?: unknown; actor?: { id?: unknown; name?: unknown; email?: unknown } }; + const courseId = typeof p.courseId === "string" ? p.courseId : ""; + const blocking = await pendingImports(courseId); + if (blocking.length) throw new StudioError(importRunning(blocking), 409); + const actor = { name: String(p.actor?.name ?? "STCourses"), email: String(p.actor?.email ?? "studio@stcourses.invalid") }; + try { + const r = await store.convertImported(courseId, actor, { onProgress: (f, m) => jc.reportProgress(f, m) }); + return { courseId, ...r }; + } finally { + // Success or failure, learners must get whatever is on disk now: the intact import (failure before the commit, + // rolled back) or the published worktree (after it), never a course whose folder has moved away. + await ctx.courses.reload(courseId).catch((e) => console.error("[studio] reload after conversion", e)); + if (store.isManaged(courseId)) await store.publishedHead(courseId).then((h) => watcher.note(courseId, h), () => {}); + } + }; const afterPublishedChange = async (courseId: string) => { await ctx.courses.reload(courseId); watcher.note(courseId, await store.publishedHead(courseId)); }; + // Crash recovery for conversions interrupted by a stop (v1.1 S); learners never see a course twice meanwhile. + const ready: Promise = store.recoverConversions() + .then(async (ids) => { if (ids.length) await ctx.courses.reload(); }) + .catch((e) => console.error("[studio] recovery", e)); + /** Wraps a handler: signed-in author/owner (or per-course author), error mapping. */ const guard = (h: (req: Request, rc: RequestContext, user: AuthUser) => Promise): RouteHandler => async (req, rc) => { const user = rc.user; if (!user) return fail(401, "sign in to use the studio"); try { + await ready; if (!hasRole(user) && (await authoredCourses(user)).length === 0) return fail(403, "the studio is for authors"); return await h(req, rc, user); } catch (e) { @@ -154,6 +195,8 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = { } return { readOnly: false, + conversionIncomplete: store.conversionIncomplete(courseId), + canRetryConversion: isOwner(user), course: { id: courseId, title: (manifestRaw as { title?: string } | null)?.title ?? courseId, managed: true }, status, manifestText: m?.content ?? "", @@ -188,7 +231,11 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = { if (all) { for (const c of ctx.courses.list()) { if (isStudioCourse(c) || managed.includes(c.id)) continue; - courses.push({ id: c.id, managed: false, title: c.manifest?.title ?? c.id, pitch: c.manifest?.pitch ?? null, accent: c.manifest?.accent ?? null, publishedStatus: c.status, message: readOnlyMessage(c) }); + const message = readOnlyMessage(c); + courses.push({ + id: c.id, managed: false, title: c.manifest?.title ?? c.id, pitch: c.manifest?.pitch ?? null, accent: c.manifest?.accent ?? null, publishedStatus: c.status, message, + imported: message === IMPORTED_MESSAGE, convertible: convertible(user, c.id), + }); } } return json({ courses, canCreate: all }); @@ -212,12 +259,42 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = { if (!store.isManaged(courseId)) { const c = ctx.courses.get(courseId); if (!c) return fail(404, "no such course"); - return json({ readOnly: true, message: readOnlyMessage(c), course: { id: c.id, title: c.manifest?.title ?? c.id, managed: false, root: c.root }, diagnostics: c.diagnostics }); + const message = readOnlyMessage(c); + return json({ readOnly: true, message, imported: message === IMPORTED_MESSAGE, convertible: convertible(user, c.id), course: { id: c.id, title: c.manifest?.title ?? c.id, managed: false, root: c.root }, diagnostics: c.diagnostics }); } return json(await courseState(courseId, user)); }), }, + "/api/studio/courses/:courseId/convert": { + POST: guard(async (_req, rc, user) => { + const courseId = rc.params.courseId ?? ""; + if (!isOwner(user)) return fail(403, "only the owner can make an imported course editable in the studio"); + // An incomplete conversion (failed after its commit) is finished by a new job: it recovers first. + const incomplete = store.conversionIncomplete(courseId); + if (store.isManaged(courseId) && !incomplete) return fail(409, `course "${courseId}" is already a studio course`); + if (!incomplete && !store.isConvertibleImport(courseId)) { + const c = ctx.courses.get(courseId); + return c ? fail(409, `${readOnlyMessage(c) === IMPORTED_MESSAGE ? "this imported course" : OUTSIDE_MESSAGE}: only imported courses in ${importedRoot} can be made editable`) : fail(404, "no such imported course"); + } + const pending = (await ctx.jobs.list(CONVERT_JOB)).find((j) => (j.payload as { courseId?: string } | null)?.courseId === courseId && (j.status === "queued" || j.status === "running")); + if (pending) return fail(409, "this course is already being made editable"); + const blocking = await pendingImports(courseId); + if (blocking.length) return fail(409, importRunning(blocking)); + const jobId = await ctx.jobs.enqueue(CONVERT_JOB, { courseId, actor: { id: user.id, name: who(user).name, email: who(user).email } }); + return json({ jobId }, 202); + }), + }, + + "/api/studio/jobs/:jobId": { + GET: guard(async (_req, rc, user) => { + if (!isOwner(user)) return fail(403, "only the owner can follow studio jobs"); + const j = await ctx.jobs.get(rc.params.jobId ?? ""); + if (!j || !j.kind.startsWith("studio.")) return fail(404, "no such job"); + return json({ id: j.id, kind: j.kind, status: j.status, progress: j.progress, message: j.message, error: j.error, result: j.result }); + }), + }, + "/api/studio/courses/:courseId/file": { GET: course(async (_req, rc, user, courseId) => { const path = rc.url.searchParams.get("path") ?? ""; @@ -364,5 +441,5 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = { watcher.start(); } - return { name: "studio", routes, multipartRoutes: ["/api/studio/courses/:courseId/upload"], store, watcher, draftWriter, stop: () => watcher.stop() }; + return { name: "studio", routes, jobs: { [CONVERT_JOB]: convertJob }, multipartRoutes: ["/api/studio/courses/:courseId/upload"], store, watcher, draftWriter, ready, stop: () => watcher.stop() }; } diff --git a/packages/feature-studio/src/server/git.ts b/packages/feature-studio/src/server/git.ts index 68a2895..074e051 100644 --- a/packages/feature-studio/src/server/git.ts +++ b/packages/feature-studio/src/server/git.ts @@ -55,15 +55,3 @@ export async function gitOk(cwd: string, args: string[], who?: Identity): Promis export function splitZ(out: string): string[] { return out.split("\0").filter(Boolean); } - -const locks = new Map>(); - -/** Serialises git work per key (the course's bare repo path) across every store instance in the process. */ -export function withLock(key: string, fn: () => Promise): Promise { - const prev = locks.get(key) ?? Promise.resolve(); - const next = prev.then(fn, fn); - const tail = next.catch(() => {}); - locks.set(key, tail); - tail.then(() => { if (locks.get(key) === tail) locks.delete(key); }); - return next; -} diff --git a/packages/feature-studio/src/server/media-refs.ts b/packages/feature-studio/src/server/media-refs.ts new file mode 100644 index 0000000..81c048f --- /dev/null +++ b/packages/feature-studio/src/server/media-refs.ts @@ -0,0 +1,61 @@ +import { extname } from "node:path"; +import { checkRelPath } from "./relpath"; + +/** + * Large media of a converted import stay out of git (v1.1 S, docs/operations.md "Imported courses in the studio"). + * The repo commits `MEDIA_REFS` (path, size and checksum per file, spec 5); the repo's `info/exclude` (not the + * author-editable `.gitignore`) gets one line per path, and the studio never counts them as changes. Every studio worktree gets the files as hard links to `/media//`. + */ +export const MEDIA_REFS = ".stcourses/media.json"; +export const LARGE_FILE_BYTES = 8 * 1024 * 1024; +const MEDIA_EXT = new Set(["mp4", "m4v", "webm", "mov", "mkv", "avi", "mp3", "m4a", "wav", "ogg", "oga", "opus", "flac"]); + +export interface MediaRef { path: string; size: number; sha256: string } + +/** Video and audio always; anything else only when it is over `LARGE_FILE_BYTES`. */ +export function isLargeMedia(rel: string, size: number): boolean { + return MEDIA_EXT.has(extname(rel).slice(1).toLowerCase()) || size > LARGE_FILE_BYTES; +} + +/** One anchored `.gitignore` pattern matching exactly `rel`. Throws for names a `.gitignore` line cannot hold. */ +export function gitignoreLine(rel: string): string { + if (/[\n\r]/.test(rel)) throw new Error(`a file name with a line break cannot be kept out of git: ${JSON.stringify(rel)}`); + let s = rel.replace(/[\\*?[\]]/g, (c) => `\\${c}`); + const trail = /( +)$/.exec(s)?.[1]?.length ?? 0; + if (trail) s = s.slice(0, -trail) + "\\ ".repeat(trail); + return `/${s}`; +} + +export const MEDIA_EXCLUDE_HEADER = "# Imported media kept outside git: see .stcourses/media.json (STCourses)"; + +/** Ignore-file content: `existing` (if any) first, then the header and one line per referenced file. */ +export function mediaGitignore(paths: string[], existing: string | null): string { + const own = existing ? (existing.endsWith("\n") ? existing : existing + "\n") + "\n" : ""; + return `${own}${MEDIA_EXCLUDE_HEADER}\n${paths.map(gitignoreLine).join("\n")}\n`; +} + +export async function sha256File(abs: string): Promise { + const h = new Bun.CryptoHasher("sha256"); + for await (const chunk of Bun.file(abs).stream()) h.update(chunk); + return h.digest("hex"); +} + +export function stringifyMediaRefs(files: MediaRef[]): string { + return JSON.stringify({ schema: 1, files }, null, 2) + "\n"; +} + +/** The reference list, or null when the text is not a valid one (every path must pass `checkRelPath`). */ +export function parseMediaRefs(text: string): MediaRef[] | null { + let raw: unknown; + try { raw = JSON.parse(text); } catch { return null; } + const r = raw as { schema?: unknown; files?: unknown } | null; + if (!r || r.schema !== 1 || !Array.isArray(r.files)) return null; + const out: MediaRef[] = []; + for (const f of r.files as Partial[]) { + if (!f || typeof f.path !== "string" || typeof f.size !== "number" || !Number.isInteger(f.size) || f.size < 0) return null; + if (typeof f.sha256 !== "string" || !/^[0-9a-f]{64}$/.test(f.sha256)) return null; + try { if (checkRelPath(f.path) !== f.path) return null; } catch { return null; } + out.push({ path: f.path, size: f.size, sha256: f.sha256 }); + } + return out; +} diff --git a/packages/feature-studio/src/server/relpath.ts b/packages/feature-studio/src/server/relpath.ts new file mode 100644 index 0000000..c6a914b --- /dev/null +++ b/packages/feature-studio/src/server/relpath.ts @@ -0,0 +1,12 @@ +import { isAbsolute, normalize, sep } from "node:path"; + +/** Normalised relative path, or throws. Rejects absolute paths, `..`, `.git` and NUL. */ +export function checkRelPath(p: string): string { + if (typeof p !== "string" || p === "" || p.includes("\0") || isAbsolute(p) || p.includes("\\")) throw new Error(`invalid path: ${JSON.stringify(p)}`); + const n = normalize(p); + const parts = n.split(sep); + if (n === "." || parts[0] === ".." || parts.some((x) => x === ".." || x.toLowerCase() === ".git") || n.endsWith(sep)) { + throw new Error(`invalid path: ${JSON.stringify(p)}`); + } + return n; +} diff --git a/packages/feature-studio/src/server/store.ts b/packages/feature-studio/src/server/store.ts index 638822a..799ca93 100644 --- a/packages/feature-studio/src/server/store.ts +++ b/packages/feature-studio/src/server/store.ts @@ -1,8 +1,14 @@ -import { constants, existsSync, lstatSync, mkdirSync, readdirSync, realpathSync, rmSync, statSync } from "node:fs"; -import { mkdir, open, rm, unlink } from "node:fs/promises"; -import { dirname, isAbsolute, join, normalize, sep } from "node:path"; -import { loadCourse, type Diagnostic } from "@stc/course-format"; -import { git, gitOk, splitZ, withLock, type Identity } from "./git"; +import { constants, existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs"; +import { cp, copyFile, mkdir, open, readFile, rename as fsRename, rm, unlink, writeFile } from "node:fs/promises"; +import { dirname, join, sep } from "node:path"; +import { loadCourse, parseManifest, type Diagnostic } from "@stc/course-format"; +import { withCourseLock } from "@stc/core"; +import { fsyncTree, linkOrCopy, scanImport, SimulatedCrash, unreferencable } from "./convert"; +import { git, gitOk, splitZ, type Identity } from "./git"; +import { gitignoreLine, MEDIA_EXCLUDE_HEADER, MEDIA_REFS, parseMediaRefs, sha256File, stringifyMediaRefs, type MediaRef } from "./media-refs"; +import { checkRelPath } from "./relpath"; + +export { checkRelPath } from "./relpath"; export type { Identity } from "./git"; @@ -19,17 +25,6 @@ export function safeAuthor(authorId: string): string { return s === "" ? "_" : s; } -/** Normalised relative path, or throws. Rejects absolute paths, `..`, `.git` and NUL. */ -export function checkRelPath(p: string): string { - if (typeof p !== "string" || p === "" || p.includes("\0") || isAbsolute(p) || p.includes("\\")) throw new Error(`invalid path: ${JSON.stringify(p)}`); - const n = normalize(p); - const parts = n.split(sep); - if (n === "." || parts[0] === ".." || parts.some((x) => x === ".." || x.toLowerCase() === ".git") || n.endsWith(sep)) { - throw new Error(`invalid path: ${JSON.stringify(p)}`); - } - return n; -} - export interface FileWrite { path: string; content?: string | Uint8Array; @@ -76,6 +71,20 @@ export type SyncResult = | { ok: false; reason: "conflict"; conflicts: string[] } | { ok: false; reason: "forbidden"; message: string }; +export interface ConvertResult { + commit: string; + /** Files kept out of git and referenced from MEDIA_REFS. */ + media: number; + /** Files committed. */ + files: number; + /** Paths skipped in the import folder (symlinks, special files, .git folders). */ + skipped: string[]; + /** True when an interrupted conversion was only finished (the repo already existed). */ + resumed: boolean; +} + +export type ConvertProgress = (fraction: number, message: string) => void | Promise; + export class StudioError extends Error { constructor(message: string, public status = 400) { super(message); } } @@ -129,9 +138,20 @@ export interface StudioStoreOptions { importedDir?: string; /** AppConfig.studioWorktreesDir (default `/worktrees`). */ worktreesDir?: string; + /** + * Media stores of converted imports (default `/media`): `/` is the former import folder. + * Not in AppConfig: production always uses the default; tests may point it elsewhere. + */ + mediaDir?: string; publishedBranch?: string; /** Tests only: allow local-path remotes (the HTTP feature never sets this). */ allowLocalRemotes?: boolean; + /** + * Tests only, read on every use (a test may change them between calls): simulate a crash (`crashAt`) or an ordinary + * failure (`failAt`) at a named conversion step, a cross-filesystem media move (`forceExdev`), or a copy whose + * bytes do not match MEDIA_REFS (`corruptStaging`). + */ + testHooks?: { crashAt?: string; failAt?: string; forceExdev?: boolean; corruptStaging?: boolean }; } export function createStudioStore(opts: StudioStoreOptions) { @@ -141,6 +161,7 @@ export function createStudioStore(opts: StudioStoreOptions) { const reposDir = opts.reposDir ?? join(dataDir, "studio", "repos"); const importedDir = opts.importedDir ?? join(dataDir, "courses"); const worktreesDir = opts.worktreesDir ?? join(dataDir, "worktrees"); + const mediaDir = opts.mediaDir ?? join(dataDir, "media"); const remoteGuard = opts.allowLocalRemotes ? ["-c", "protocol.ext.allow=never", "-c", "protocol.fd.allow=never"] : REMOTE_GUARD; const paths = (courseId: string) => { @@ -152,6 +173,8 @@ export function createStudioStore(opts: StudioStoreOptions) { published: join(root, "published"), draftsDir: join(root, "drafts"), draft: (authorId: string) => join(root, "drafts", safeAuthor(authorId)), + media: join(mediaDir, courseId), + imported: join(importedDir, courseId), }; }; const branchOf = (authorId: string) => `draft/${safeAuthor(authorId)}`; @@ -165,14 +188,30 @@ export function createStudioStore(opts: StudioStoreOptions) { if (!isManaged(courseId)) throw new StudioError(`course ${courseId} is not managed by the studio`, 404); return paths(courseId); }; - const lock = (courseId: string, fn: () => Promise) => withLock(paths(courseId).bare, fn); + /** The process-wide course lock shared with the importer's jobs (core `withCourseLock`). */ + const lock = (courseId: string, fn: () => Promise) => { paths(courseId); return withCourseLock(courseId, fn); }; + const convertingDir = (courseId: string) => join(importedDir, `.converting-${courseId}`); + const listDir = (dir: string): string[] => { try { return readdirSync(dir); } catch { return []; } }; + const crash = async (step: string) => { + if (opts.testHooks?.crashAt === step) throw new SimulatedCrash(step); + if (opts.testHooks?.failAt === step) throw new Error(`injected failure at ${step}`); + }; + const rename = async (from: string, to: string) => fsRename(from, to); const rev = async (cwd: string, ref: string) => (await gitOk(cwd, ["rev-parse", ref])).trim(); const count = async (cwd: string, range: string) => Number((await gitOk(cwd, ["rev-list", "--count", range])).trim()); const unmerged = async (cwd: string) => [...new Set(splitZ(await gitOk(cwd, ["diff", "--name-only", "-z", "--diff-filter=U"])))]; const isMerging = async (cwd: string) => (await git(cwd, ["rev-parse", "-q", "--verify", "MERGE_HEAD"])).code === 0; - const dirtyFiles = async (cwd: string) => - splitZ(await gitOk(cwd, ["status", "--porcelain", "-z", "--untracked-files=all"])).map((l) => l.slice(3)).filter(Boolean); + /** + * Uncommitted changes, except untracked referenced media: whatever a `.gitignore` (editable by authors and remotes) + * says, the files listed in MEDIA_REFS never make a worktree dirty. + */ + const dirtyFiles = async (cwd: string) => { + const media = new Set((await mediaRefsIn(cwd)).map((r) => r.path)); + return splitZ(await gitOk(cwd, ["status", "--porcelain", "-z", "--untracked-files=all"])) + .filter((l) => !(l.startsWith("?? ") && media.has(l.slice(3)))) + .map((l) => l.slice(3)).filter(Boolean); + }; const remoteUrl = async (bare: string) => { const r = await git(bare, ["remote", "get-url", "origin"]); return r.code === 0 ? r.stdout.trim() : null; @@ -241,9 +280,185 @@ export function createStudioStore(opts: StudioStoreOptions) { const branch = branchOf(authorId); const exists = (await git(p.bare, ["rev-parse", "-q", "--verify", `refs/heads/${branch}`])).code === 0; await gitOk(p.bare, exists ? ["worktree", "add", dir, branch] : ["worktree", "add", "-b", branch, dir, main]); + await materializeMedia(dir, p.media, p.bare); return dir; } + /** The draft's or worktree's media references (empty for courses that never were imports). */ + async function mediaRefsIn(dir: string): Promise { + const f = await readFileIn(dir, MEDIA_REFS); + return f ? parseMediaRefs(f.content) ?? [] : []; + } + + /** + * Puts every referenced file into `dir` as a hard link to `fromRoot/` (a copy across filesystems). + * Idempotent; existing destinations are left alone. A reference must name a regular file whose real path is inside + * `fromRoot`, and the destination path may not contain a symlink. Returns the references it could not satisfy. + */ + async function materializeMedia(dir: string, fromRoot: string, bare: string): Promise<{ linked: number; missing: string[] }> { + const refs = await mediaRefsIn(dir); + if (refs.length) await writeExclude(bare, refs.map((r) => r.path)); + const missing: string[] = []; + let linked = 0; + let realFrom: string; + try { realFrom = realpathSync(fromRoot); } catch { return { linked, missing: refs.map((r) => r.path) }; } + for (const r of refs) { + try { + const src = join(realFrom, r.path); + const st = lstatSync(src); + const real = realpathSync(src); + if (!st.isFile() || !real.startsWith(realFrom + sep)) throw new Error("not a regular file inside the media store"); + const dest = await safeWritePath(dir, r.path); + if (lexists(dest)) continue; + await linkOrCopy(real, dest); + linked++; + } catch { + missing.push(r.path); + } + } + if (missing.length) console.warn(`[studio] media missing for ${dir}: ${missing.join(", ")}`); + return { linked, missing }; + } + + /** + * Keeps referenced media out of `git status` through the repo's `info/exclude` (shared by every worktree, not + * reachable by authors or remotes, unlike `.gitignore`). Lines are only ever added. + */ + async function writeExclude(bare: string, paths: string[]): Promise { + const file = join(bare, "info", "exclude"); + const cur = await readFile(file, "utf8").catch(() => ""); + const lines = new Set(cur.split("\n")); + const add: string[] = []; + for (const x of paths) { + let line: string; + try { line = gitignoreLine(x); } catch { continue; } + if (!lines.has(line)) { add.push(line); lines.add(line); } + } + if (!add.length) return; + const head = cur.includes(MEDIA_EXCLUDE_HEADER) ? "" : `${MEDIA_EXCLUDE_HEADER}\n`; + await mkdir(dirname(file), { recursive: true }); + await writeFile(file, `${cur}${cur && !cur.endsWith("\n") ? "\n" : ""}${head}${add.join("\n")}\n`); + } + + /** Every reference present under `root` with its size (and, when asked, its sha256). */ + async function mediaComplete(root: string, refs: MediaRef[], hash: boolean): Promise { + for (const r of refs) { + try { + const st = lstatSync(join(root, r.path)); + if (!st.isFile() || st.size !== r.size) return false; + if (hash && (await sha256File(join(root, r.path))) !== r.sha256) return false; + } catch { + return false; + } + } + return true; + } + + /** Renames the old import folder aside (atomic, never discovered: dot-folder) and deletes it. */ + async function retireSource(courseId: string, from: string): Promise { + const gone = join(importedDir, `.converted-${courseId}.${Date.now()}`); + await rename(from, gone); + await crash("source-retired"); + await rm(gone, { recursive: true, force: true }); + } + + /** + * Makes `from` (the hidden import folder) the media store `/`. Same filesystem: one rename. + * Across filesystems: copy into a staging dir, fsync, verify against MEDIA_REFS, rename into place, then retire the + * source. An existing store that matches MEDIA_REFS means an earlier attempt got that far: only the source is retired. + */ + async function finishMove(courseId: string, from: string): Promise { + const p = paths(courseId); + const refs = await mediaRefsIn(p.published); + if (lexists(p.media)) { + if (!(await mediaComplete(p.media, refs, true))) throw new StudioError(`the media store ${p.media} exists but does not match ${MEDIA_REFS}; move it away and retry`, 409); + await retireSource(courseId, from); + return; + } + await crash("before-move"); + await mkdir(mediaDir, { recursive: true }); + try { + if (opts.testHooks?.forceExdev) throw Object.assign(new Error("EXDEV (test)"), { code: "EXDEV" }); + await rename(from, p.media); + return; + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== "EXDEV") throw e; + } + const staging = join(mediaDir, `.staging-${courseId}.${Bun.randomUUIDv7()}`); + try { + await cp(from, staging, { recursive: true, verbatimSymlinks: true, errorOnExist: true, force: false }); + if (opts.testHooks?.corruptStaging && refs[0]) await writeFile(join(staging, refs[0].path), "corrupted copy"); + await crash("staging-copied"); + await fsyncTree(staging); + if (!(await mediaComplete(staging, refs, true))) throw new Error(`the copied media in ${staging} do not match ${MEDIA_REFS}`); + await rename(staging, p.media); + await fsyncTree(mediaDir, { shallow: true }); + } catch (e) { + if (!(e instanceof SimulatedCrash)) await rm(staging, { recursive: true, force: true }); + throw e; + } + await crash("staging-placed"); + await retireSource(courseId, from); + } + + /** + * Deterministic recovery of one course's conversion, under its lock: + * - staging copies are deleted (a move is always redone from the source); + * - a hidden import folder without a finished repo is restored and the partial repo removed (the next attempt + * starts over); with a finished repo, the published worktree is reset to its commit and the move finished; + * - retired sources are deleted once the media store matches MEDIA_REFS. + */ + async function recoverUnlocked(courseId: string): Promise { + const p = paths(courseId); + for (const n of listDir(mediaDir)) if (n.startsWith(`.staging-${courseId}.`)) await rm(join(mediaDir, n), { recursive: true, force: true }); + const conv = convertingDir(courseId); + if (lexists(conv)) { + if (!isManaged(courseId)) { + rmSync(p.bare, { recursive: true, force: true }); + rmSync(p.root, { recursive: true, force: true }); + if (lexists(p.imported)) throw new StudioError(`both ${conv} and ${p.imported} exist; remove one of them by hand`, 409); + await rename(conv, p.imported); + } else { + await gitOk(p.published, ["reset", "-q", "--hard", "HEAD"]); + await finishMove(courseId, conv); + await materializeMedia(p.published, p.media, p.bare); + } + } + const retired = listDir(importedDir).filter((n) => n.startsWith(`.converted-${courseId}.`)); + if (retired.length) { + const refs = isManaged(courseId) ? await mediaRefsIn(p.published) : null; + if (refs && lexists(p.media) && (await mediaComplete(p.media, refs, false))) { + for (const n of retired) await rm(join(importedDir, n), { recursive: true, force: true }); + } else { + console.warn(`[studio] ${retired.join(", ")} kept: the media store of ${courseId} is not complete`); + } + } + } + + /** + * Referenced media paths that `treeish` (a draft branch, the remote's branch) tracks in git. Merging those would + * silently replace the media-store file with the commit's content, so publish and Sync refuse them. + */ + async function mediaClash(bare: string, treeish: string): Promise { + const refs = new Set(); + for (const t of [main, treeish]) { + const text = await showText(bare, `${t}:${MEDIA_REFS}`); + for (const r of (text && parseMediaRefs(text)) || []) refs.add(r.path); + } + if (!refs.size) return []; + const r = await git(bare, ["--literal-pathspecs", "ls-tree", "-r", "-z", "--name-only", treeish, "--", ...refs]); + return r.code === 0 ? splitZ(r.stdout) : []; + } + + /** Referenced media and the reference file are not editable in the studio (they are not in git). */ + async function refuseMediaWrites(dir: string, paths: string[]): Promise { + const locked = new Set([MEDIA_REFS, ...(await mediaRefsIn(dir)).map((r) => r.path)]); + const hit = paths.map((x) => checkRelPath(x)).filter((x) => locked.has(x)); + if (hit.length) { + throw new StudioError(`${hit.join(", ")}: imported media are kept outside git and cannot be changed in the studio; upload a file under a new name instead`, 409); + } + } + async function readFileIn(dir: string, rel: string): Promise<{ content: string; hash: string } | null> { let abs: string; try { abs = inside(dir, rel); } catch { return null; } @@ -268,6 +483,8 @@ export function createStudioStore(opts: StudioStoreOptions) { if (lexists(abs)) await unlink(abs); } else { const abs = await safeWritePath(dir, f.path); + // never write into an inode shared with another path (imported media, an owner's original): replace the file + if (lexists(abs) && lstatSync(abs).nlink > 1) await unlink(abs); // O_NOFOLLOW: a symlink swapped in after the check makes the open fail instead of writing through it const fh = await open(abs, constants.O_WRONLY | constants.O_CREAT | constants.O_TRUNC | constants.O_NOFOLLOW, 0o644); try { await fh.writeFile(f.content ?? ""); } finally { await fh.close(); } @@ -323,16 +540,182 @@ export function createStudioStore(opts: StudioStoreOptions) { }); }, + /** An importer folder `/` holding an imported (not authored) course the owner may convert. */ + isConvertibleImport(courseId: string): boolean { + if (!ID_RE.test(courseId) || isManaged(courseId)) return false; + const p = paths(courseId); + try { + if (!lstatSync(p.imported).isDirectory()) return false; + const m = parseLoose(readFileSync(join(p.imported, MANIFEST), "utf8")); + return !!m?.source?.kind && m.source.kind !== "authored"; + } catch { + return false; + } + }, + + /** + * A conversion that got past its commit but did not finish (the media move failed or was interrupted): the course + * is served from the published worktree, and `convertImported` (which recovers first) completes it. + */ + conversionIncomplete(courseId: string): boolean { + if (!ID_RE.test(courseId)) return false; + if (lexists(convertingDir(courseId))) return true; + return listDir(mediaDir).some((n) => n.startsWith(`.staging-${courseId}.`)) || listDir(importedDir).some((n) => n.startsWith(`.converted-${courseId}.`)); + }, + + /** Re-links referenced media into the published worktree (after an outside checkout removed them, say). */ + repairMedia: (courseId: string) => lock(courseId, async () => { + const p = requireManaged(courseId); + return materializeMedia(p.published, p.media, p.bare); + }), + + /** + * Finishes or undoes every interrupted conversion (server start; `convertImported` also does it for its course). + * Returns the ids it touched. Never throws: a course it cannot repair is logged and left for the owner. + */ + async recoverConversions(): Promise { + const ids = new Set(); + const scan = (dir: string, re: RegExp) => { for (const n of listDir(dir)) { const m = re.exec(n); if (m && ID_RE.test(m[1]!)) ids.add(m[1]!); } }; + scan(mediaDir, /^\.staging-([^.]+)\./); + scan(importedDir, /^\.converting-([^.]+)$/); + scan(importedDir, /^\.converted-([^.]+)\./); + const done: string[] = []; + for (const id of [...ids].sort()) { + try { await lock(id, () => recoverUnlocked(id)); done.push(id); } catch (e) { console.error(`[studio] could not recover the conversion of ${id}:`, e); } + } + return done; + }, + + /** + * Turns the imported course `/` into a studio course with the same id (v1.1 S, ruling R9): + * small files are committed as they are (manifest unchanged: visibility and source.kind stay), large media are + * referenced from MEDIA_REFS and hard-linked into each worktree, and the import folder ends up as `/`. + * Owner-only at the HTTP layer; holds the shared course lock, so importer jobs on the course wait or fail cleanly. + * + * Crash safety: the import folder is hidden (`/.converting-`, never discovered) before the + * published worktree appears, so the id is never loaded twice. Until the worktree is checked out, a failure undoes + * everything; after that the conversion is committed and `recoverUnlocked` (run first by every attempt and at + * server start) finishes the move. Across filesystems the media are copied into `/.staging-.`, + * fsynced, verified against MEDIA_REFS (size and sha256) and only then renamed into place. + */ + convertImported(courseId: string, who: Identity, opts: { onProgress?: ConvertProgress } = {}): Promise { + if (!ID_RE.test(courseId)) return Promise.reject(new StudioError(`invalid course id: ${courseId}`, 404)); + const progress = opts.onProgress ?? (() => {}); + return lock(courseId, async () => { + const p = paths(courseId); + await recoverUnlocked(courseId); + if (isManaged(courseId)) { + const refsFile = await readFileIn(p.published, MEDIA_REFS); + const media = refsFile ? (parseMediaRefs(refsFile.content) ?? []).length : 0; + const head = async () => rev(p.published, "HEAD"); + // already converted (a job re-run after a restart, say): report success + if (refsFile && lexists(p.media) && !lexists(p.imported)) return { commit: await head(), media, files: 0, skipped: [], resumed: true }; + // a conversion made before the import folder was hidden first: finish its move + if (refsFile && lexists(p.imported)) { + await finishMove(courseId, p.imported); + await materializeMedia(p.published, p.media, p.bare); + return { commit: await head(), media, files: 0, skipped: [], resumed: true }; + } + throw new StudioError(`course "${courseId}" is already a studio course`, 409); + } + const srcManifest = join(p.imported, MANIFEST); + if (!lexists(srcManifest)) throw new StudioError(`there is no imported course "${courseId}" in ${importedDir}`, 404); + const top = lstatSync(p.imported); + if (top.isSymbolicLink() || !top.isDirectory() || lstatSync(srcManifest).isSymbolicLink()) throw new StudioError("the import folder and its manifest must not be symlinks", 400); + if (lexists(p.media)) throw new StudioError(`the media store ${p.media} already exists; move it away and retry`, 409); + const parsed = parseManifest(await Bun.file(srcManifest).text()); + if (!parsed.ok) throw new StudioError(`the imported manifest is invalid: ${parsed.errors.join("; ")}`, 400); + const m = parsed.manifest; + if (m.id !== courseId) throw new StudioError(`the imported manifest has id "${m.id}", not "${courseId}"`, 400); + if (m.root) throw new StudioError("an imported course with root: cannot become a studio course", 400); + if (!m.source?.kind || m.source.kind === "authored") throw new StudioError("only imported courses can be made editable this way", 400); + + await progress(0.02, "Reading the import folder"); + const scan = await scanImport(p.imported); + const bad = unreferencable(scan.large); + if (bad.length) throw new StudioError(`these large files have names the studio cannot keep outside git; rename them first: ${bad.map((b) => JSON.stringify(b)).join(", ")}`, 400); + + // A previous attempt that never got as far as the published worktree: start again. + rmSync(p.bare, { recursive: true, force: true }); + rmSync(p.root, { recursive: true, force: true }); + + const total = scan.large.reduce((n, f) => n + f.size, 0) || 1; + let done = 0; + const refs: MediaRef[] = []; + for (const f of scan.large) { + await progress(0.05 + 0.75 * (done / total), `Checksumming ${f.path}`); + refs.push({ path: f.path, size: f.size, sha256: await sha256File(join(p.imported, f.path)) }); + done += f.size; + } + + const conv = convertingDir(courseId); + const work = join(dataDir, "tmp", `convert-${courseId}-${Bun.randomUUIDv7()}`); + const stage = join(work, "tree"); + const list = [...new Set([...scan.small, MEDIA_REFS])]; + let hidden = false; + try { + await progress(0.82, "Committing the course files"); + for (const f of scan.small) { + await mkdir(dirname(join(stage, f)), { recursive: true }); + await copyFile(join(p.imported, f), join(stage, f)); + } + await mkdir(join(stage, dirname(MEDIA_REFS)), { recursive: true }); + await writeFile(join(stage, MEDIA_REFS), stringifyMediaRefs(refs)); + await writeFile(join(work, "paths"), list.join("\0") + "\0"); + + mkdirSync(reposDir, { recursive: true }); + await gitOk(dataDir, ["init", "-q", "--bare", "-b", main, p.bare]); + await writeExclude(p.bare, refs.map((r) => r.path)); + const gd = ["--literal-pathspecs", "--git-dir", p.bare, "--work-tree", stage]; + // -f: a nested .gitignore from the course's own code must not drop course files + await gitOk(stage, [...gd, "add", "-f", `--pathspec-from-file=${join(work, "paths")}`, "--pathspec-file-nul"]); + await gitOk(stage, [...gd, "commit", "-q", "-m", `Make ${courseId} editable in the studio\n\nConverted from the import in ${p.imported}. ${refs.length} large media files stay outside git (${MEDIA_REFS}).`], who); + await crash("committed"); + + // Hide the import folder before the published worktree exists: the id is never discoverable twice. + await rename(p.imported, conv); + hidden = true; + await crash("hidden"); + mkdirSync(p.root, { recursive: true }); + await gitOk(p.bare, ["worktree", "add", p.published, main]); + await progress(0.9, "Linking media into the published course"); + const mat = await materializeMedia(p.published, conv, p.bare); + if (mat.missing.length) throw new Error(`could not link media: ${mat.missing.join(", ")}`); + const dirty = await dirtyFiles(p.published); + if (dirty.length) throw new Error(`the converted course is not clean after checkout: ${dirty.join(", ")}`); + await crash("worktree"); + } catch (e) { + if (!(e instanceof SimulatedCrash)) { + rmSync(p.bare, { recursive: true, force: true }); + rmSync(p.root, { recursive: true, force: true }); + if (hidden && !lexists(p.imported)) await rename(conv, p.imported); + } + throw e; + } finally { + await rm(work, { recursive: true, force: true }); + } + + // Committed: from here on a failure leaves a state that the next attempt (or a restart) finishes. + await progress(0.95, "Moving the import folder to the media store"); + await finishMove(courseId, conv); + await materializeMedia(p.published, p.media, p.bare); + return { commit: await rev(p.published, "HEAD"), media: refs.length, files: list.length, skipped: scan.skipped, resumed: false }; + }); + }, + ensureDraft: (courseId: string, authorId: string) => lock(courseId, () => ensureDraftUnlocked(courseId, authorId)), async listFiles(courseId: string, authorId: string): Promise { const dir = await api.ensureDraft(courseId, authorId); const tracked = splitZ(await gitOk(dir, ["ls-files", "-z", "--cached", "--others", "--exclude-standard"])); - return [...new Set(tracked)].filter((f) => existsSync(join(dir, f))).sort(); + const media = (await mediaRefsIn(dir)).map((r) => r.path); + return [...new Set([...tracked, ...media])].filter((f) => existsSync(join(dir, f))).sort(); }, async readFile(courseId: string, authorId: string, rel: string) { const dir = await api.ensureDraft(courseId, authorId); + // referenced media can be gigabytes and are not text: never read them whole (the raw route streams them) + if ((await mediaRefsIn(dir)).some((r) => r.path === rel)) return null; return readFileIn(dir, rel); }, @@ -348,6 +731,7 @@ export function createStudioStore(opts: StudioStoreOptions) { return lock(courseId, async () => { const dir = await ensureDraftUnlocked(courseId, who.id); if (await isMerging(dir)) throw new StudioError("the draft has a merge in progress: resolve or abort it first", 409); + await refuseMediaWrites(dir, files.map((f) => f.path)); const conflicts: FileConflict[] = []; for (const f of files) { if (f.baseHash === undefined) continue; @@ -419,6 +803,8 @@ export function createStudioStore(opts: StudioStoreOptions) { const diags = (await api.validateDraftUnlocked(courseId, dir)).filter((d) => d.level === "error"); if (diags.length) return { ok: false, reason: "invalid", diagnostics: diags }; if ((await count(p.bare, `${main}..${branch}`)) === 0) return { ok: true, commit: await rev(p.published, "HEAD"), published: false }; + const clash = await mediaClash(p.bare, branch); + if (clash.length) return { ok: false, reason: "invalid", diagnostics: clash.map((path) => ({ level: "error" as const, courseId, path, message: `${path} is imported media kept outside git; the draft must not commit a file there` })) }; const baseManifest = await showText(p.published, `HEAD:${MANIFEST}`); // Merge without committing, so the merged result is checked before it becomes the published version. const r = await git(p.published, ["merge", "--no-ff", "--no-commit", branch], who); @@ -440,6 +826,7 @@ export function createStudioStore(opts: StudioStoreOptions) { return { ok: false, reason: "invalid", diagnostics: mergedErrors }; } await gitOk(p.published, ["commit", "-q", "-m", `Publish ${branch}`], who); + await materializeMedia(p.published, p.media, p.bare); // Bring the draft level with what was just published, so "behind" stays meaningful. await git(dir, ["merge", "-q", "--ff-only", main]); return { ok: true, commit: await rev(p.published, "HEAD"), published: true }; @@ -501,6 +888,7 @@ export function createStudioStore(opts: StudioStoreOptions) { return lock(courseId, async () => { const dir = await ensureDraftUnlocked(courseId, who.id); if (!(await isMerging(dir))) throw new StudioError("no merge in progress", 409); + await refuseMediaWrites(dir, files.map((f) => f.path)); const m = files.find((f) => checkRelPath(f.path) === MANIFEST); if (m && !who.owner) { const needOwner = ownerRequiredFor(await showText(requireManaged(courseId).bare, `${main}:${MANIFEST}`), m.content); @@ -562,6 +950,8 @@ export function createStudioStore(opts: StudioStoreOptions) { const local = await rev(p.bare, main); const remote = await rev(p.bare, remoteRef); if (local !== remote) { + const clash = await mediaClash(p.bare, remoteRef); + if (clash.length) return { ok: false, reason: "forbidden", message: `the remote commits files over imported media kept outside git: ${clash.join(", ")}` }; const localBehind = (await git(p.bare, ["merge-base", "--is-ancestor", local, remote])).code === 0; const remoteBehind = (await git(p.bare, ["merge-base", "--is-ancestor", remote, local])).code === 0; const baseManifest = await showText(p.bare, `${main}:${MANIFEST}`); diff --git a/packages/feature-studio/src/ui/ConvertPanel.tsx b/packages/feature-studio/src/ui/ConvertPanel.tsx new file mode 100644 index 0000000..3288d1d --- /dev/null +++ b/packages/feature-studio/src/ui/ConvertPanel.tsx @@ -0,0 +1,102 @@ +import { useEffect, useRef, useState } from "react"; +import { Pill, ProgressBar } from "@stc/ui"; +import { api, coursePath } from "./api"; + +export type ConvertPhase = + | { kind: "idle" } + | { kind: "running"; progress: number; message: string | null } + | { kind: "failed"; error: string }; + +export interface ReadOnlyInfo { title: string; message: string; root: string; imported?: boolean; convertible?: boolean } + +/** The owner's "Make editable in the studio" panel for an imported course (v1.1 S). Presentational. */ +export function ConvertPanelView({ phase, onConvert }: { phase: ConvertPhase; onConvert: () => void }) { + const running = phase.kind === "running"; + return ( +
+

Make editable in the studio

+

+ The course becomes a studio course with the same id, so learners keep their progress and notes. The manifest, + lessons and small files go into a git repository you edit on a draft and publish. Videos and other large files + stay outside git, on this server, and keep playing at the same paths. Visibility does not change. +

+ {running ? ( +
+ + {phase.message ?? "Waiting to start…"} +
+ ) : null} + {phase.kind === "failed" ?

{phase.error}

: null} +
+ {phase.kind === "failed" ? "Try again" : "Make editable in the studio"} +
+
+ ); +} + +/** Starts the conversion job (`POST …/convert`) and polls it; calls `onDone` once it has finished. */ +function useConvertJob(courseId: string, onDone: () => void) { + const [phase, setPhase] = useState({ kind: "idle" }); + const live = useRef(true); + useEffect(() => () => { live.current = false; }, []); + + const start = async () => { + setPhase({ kind: "running", progress: 0, message: null }); + const { status, data } = await api("POST", coursePath(courseId, "/convert")); + if (status !== 202) return setPhase({ kind: "failed", error: data?.error ?? `Could not start (${status})` }); + for (;;) { + await new Promise((r) => setTimeout(r, 700)); + if (!live.current) return; + const j = await api("GET", `/api/studio/jobs/${encodeURIComponent(data.jobId)}`); + if (j.status !== 200) return setPhase({ kind: "failed", error: j.data?.error ?? `Could not follow the conversion (${j.status})` }); + if (j.data.status === "done") return onDone(); + if (j.data.status === "failed") return setPhase({ kind: "failed", error: j.data.error ?? "The conversion failed" }); + setPhase({ kind: "running", progress: j.data.progress ?? 0, message: j.data.message }); + } + }; + return { phase, start }; +} + +export function ConvertPanel({ courseId, onDone }: { courseId: string; onDone: () => void }) { + const { phase, start } = useConvertJob(courseId, onDone); + return ; +} + +/** A conversion that failed after its commit: the course is served, but its media move did not finish. */ +export function IncompleteConversionView({ phase, canRetry, onRetry }: { phase: ConvertPhase; canRetry: boolean; onRetry: () => void }) { + return ( +
+ Conversion incomplete. Learners see this course, but moving its imported media into place did not + finish. {canRetry ? "Retry to complete it." : "Ask the owner to retry it."} + {phase.kind === "running" ? ( +
+ + {phase.message ?? "Waiting to start…"} +
+ ) : null} + {phase.kind === "failed" ?

{phase.error}

: null} + {canRetry ?
Retry
: null} +
+ ); +} + +export function IncompleteConversion({ courseId, canRetry, onDone }: { courseId: string; canRetry: boolean; onDone: () => void }) { + const { phase, start } = useConvertJob(courseId, onDone); + return ; +} + +/** What the studio shows for a course it does not manage. */ +export function ReadOnlyView({ courseId, info, onConverted }: { courseId: string; info: ReadOnlyInfo; onConverted: () => void }) { + if (info.imported) { + return ( + <> +
+ Imported course.{" "} + {info.convertible ? "It is read-only until you make it editable in the studio." : "It is read-only here. Only the owner can make it editable in the studio."} +
+ {info.convertible ? : null} + + ); + } + return
{info.message}. It is read-only here; edit it in its own repository at {info.root}.
; +} diff --git a/packages/feature-studio/src/ui/CourseStudio.tsx b/packages/feature-studio/src/ui/CourseStudio.tsx index c5b1cbb..f7084e9 100644 --- a/packages/feature-studio/src/ui/CourseStudio.tsx +++ b/packages/feature-studio/src/ui/CourseStudio.tsx @@ -3,6 +3,7 @@ import { Link, useParams, useSearchParams } from "react-router-dom"; import { CourseCover, Pill } from "@stc/ui"; import { api, coursePath, uniqueId, type Diagnostic, type ManifestLesson } from "./api"; import { AccessMessage } from "./StudioHome"; +import { IncompleteConversion, ReadOnlyView } from "./ConvertPanel"; import { clone, useWorkspace, type Workspace } from "./workspace"; import { CourseSettings } from "./editors/CourseSettings"; import { LessonEditor } from "./editors/LessonEditor"; @@ -15,7 +16,7 @@ type Notice = { kind: "ok" | "warn" | "bad"; text: string; files?: string[]; dia export default function CourseStudio() { const { courseId = "" } = useParams(); - const { ws, readOnly, error } = useWorkspace(courseId); + const { ws, readOnly, error, reload } = useWorkspace(courseId); if (error) return

Studio

{error.status === 404 ?
No such course.
: }
; if (readOnly) { @@ -23,7 +24,7 @@ export default function CourseStudio() {

Studio

{readOnly.title}

-
{readOnly.message}. It is read-only here; edit it in its own repository at {readOnly.root}.
+
); } @@ -106,6 +107,8 @@ function Workbench({ ws }: { ws: Workspace }) { Publish + {ws.state.conversionIncomplete ? : null} + {notice ? (
{notice.text} diff --git a/packages/feature-studio/src/ui/StudioHome.tsx b/packages/feature-studio/src/ui/StudioHome.tsx index 0a68b8c..2c21710 100644 --- a/packages/feature-studio/src/ui/StudioHome.tsx +++ b/packages/feature-studio/src/ui/StudioHome.tsx @@ -6,7 +6,7 @@ import "./studio.css"; interface CourseRow { id: string; title: string; pitch: string | null; accent: string | null; managed: boolean; publishedStatus: "ok" | "error"; - ahead?: number; behind?: number; merging?: boolean; remote?: string | null; message?: string; + ahead?: number; behind?: number; merging?: boolean; remote?: string | null; message?: string; imported?: boolean; convertible?: boolean; } export function AccessMessage({ status, error }: { status: number; error?: string }) { @@ -99,7 +99,7 @@ export default function StudioHome() { {outside.length ? (

Managed outside STCourses

-

These courses come from repositories the platform does not own. Edit them in their own repository.

+

These courses come from repositories the platform does not own, or from imports. Edit them in their own repository; the owner can make an imported course editable here.

    {outside.map((c) => (
  • @@ -108,7 +108,7 @@ export default function StudioHome() { {c.title} {c.id} · {c.message}
- Read-only + {c.convertible ? Make editable : {c.imported ? "Imported" : "Read-only"}} ))} diff --git a/packages/feature-studio/src/ui/api.ts b/packages/feature-studio/src/ui/api.ts index 8efb89f..ec8b49d 100644 --- a/packages/feature-studio/src/ui/api.ts +++ b/packages/feature-studio/src/ui/api.ts @@ -42,10 +42,12 @@ export interface DraftStatus { export interface Diagnostic { level: "error" | "warning"; lessonId?: string; path?: string; message: string } export interface CourseState { - readOnly: false; course: { id: string; title: string }; status: DraftStatus; manifestText: string; manifestHash: string | null; + readOnly: false; course: { id: string; title: string }; status: DraftStatus; + /** A conversion from an import that did not finish its media move (v1.1 S); the owner may retry it. */ + conversionIncomplete?: boolean; canRetryConversion?: boolean; manifestText: string; manifestHash: string | null; manifestRaw: ManifestDoc | null; manifestErrors: string[]; diagnostics: Diagnostic[]; files: string[]; } -export interface ReadOnlyState { readOnly: true; message: string; course: { id: string; title: string; root: string }; diagnostics: Diagnostic[] } +export interface ReadOnlyState { readOnly: true; message: string; imported?: boolean; convertible?: boolean; course: { id: string; title: string; root: string }; diagnostics: Diagnostic[] } export interface FileWrite { path: string; content?: string; baseHash?: string | null; delete?: boolean } export interface FileConflict { path: string; current: string | null; currentHash: string | null } diff --git a/packages/feature-studio/src/ui/studio.css b/packages/feature-studio/src/ui/studio.css index c1ab02a..bb864b9 100644 --- a/packages/feature-studio/src/ui/studio.css +++ b/packages/feature-studio/src/ui/studio.css @@ -96,3 +96,8 @@ .studio-content { padding: 0 16px 48px; width: 100%; } .studio-bar { margin: 0 -16px 20px; padding: 10px 16px; } } + +/* "Make editable in the studio" (imported courses) */ +.studio-convert-progress { display: flex; align-items: center; gap: 12px; margin: 0 0 12px; max-width: 760px; } +.studio-convert-progress .stc-bar { flex: 1; } +a.studio-tag { text-decoration: none; } diff --git a/packages/feature-studio/src/ui/workspace.ts b/packages/feature-studio/src/ui/workspace.ts index ec80f89..cd7b8e8 100644 --- a/packages/feature-studio/src/ui/workspace.ts +++ b/packages/feature-studio/src/ui/workspace.ts @@ -1,4 +1,5 @@ import { useCallback, useEffect, useState } from "react"; +import type { ReadOnlyInfo } from "./ConvertPanel"; import { api, coursePath, rawUrl, type CourseState, type FileConflict, type FileWrite, type ManifestDoc } from "./api"; export type SaveOutcome = { ok: true } | { ok: false; conflicts?: FileConflict[]; error?: string }; @@ -51,7 +52,7 @@ export type OpenFile = ReturnType; export function useWorkspace(courseId: string) { const [state, setState] = useState(null); - const [readOnly, setReadOnly] = useState<{ message: string; title: string; root: string } | null>(null); + const [readOnly, setReadOnly] = useState(null); const [error, setError] = useState<{ status: number; error?: string } | null>(null); const [doc, setDocState] = useState(null); const [docDirty, setDocDirty] = useState(false); @@ -61,7 +62,8 @@ export function useWorkspace(courseId: string) { const { status, data } = await api("GET", coursePath(courseId)); if (status !== 200) { setError({ status, error: data?.error }); return; } setError(null); - if (data.readOnly) { setReadOnly({ message: data.message, title: data.course.title, root: data.course.root }); return; } + if (data.readOnly) { setReadOnly({ message: data.message, title: data.course.title, root: data.course.root, imported: !!data.imported, convertible: !!data.convertible }); return; } + setReadOnly(null); setState(data); setDocState(data.manifestRaw ? clone(data.manifestRaw) : null); setDocDirty(false); diff --git a/packages/feature-studio/test/app-helpers.ts b/packages/feature-studio/test/app-helpers.ts index ea53c32..32213ba 100644 --- a/packages/feature-studio/test/app-helpers.ts +++ b/packages/feature-studio/test/app-helpers.ts @@ -1,9 +1,9 @@ import { resolve } from "node:path"; -import type { AppConfig, AuthUser, ServerContext } from "@stc/contracts"; -import { createApp, resolveConfig } from "@stc/core"; +import type { AppConfig, AuthUser, JobQueue, ServerContext } from "@stc/contracts"; +import { createApp, createJobQueue, resolveConfig } from "@stc/core"; import { createCourseRepository } from "@stc/course-format"; import { connect, migrate } from "@stc/db"; -import { studioFeature } from "../src/server/feature"; +import { studioFeature, type StudioFeatureOptions } from "../src/server/feature"; import { studioCourseDirs } from "../src/server/paths"; import { tempDir } from "./helpers"; @@ -15,7 +15,7 @@ export const users: Record = { }; /** A server context like main.ts builds, with header-selected users (`x-user: ada`). */ -export async function setupApp(extraCourseDirs: string[] = [], dbUrl = "sqlite://:memory:", prepare?: (dataDir: string) => void, configOverrides: Partial = {}) { +export async function setupApp(extraCourseDirs: string[] = [], dbUrl = "sqlite://:memory:", prepare?: (dataDir: string) => void, configOverrides: Partial = {}, featureOpts: StudioFeatureOptions = {}) { const dataDir = tempDir(); prepare?.(dataDir); let dropSchema = async () => {}; @@ -34,10 +34,14 @@ export async function setupApp(extraCourseDirs: string[] = [], dbUrl = "sqlite:/ const config = resolveConfig({ port: 0, databaseUrl: "sqlite://:memory:", dataDir, courseDirs: extraCourseDirs, secret: "t", publicUrl: "http://x", ...configOverrides }); const courses = await createCourseRepository(studioCourseDirs(config)); const ctx = { - config, db, courses, jobs: null as never, + config, db, courses, jobs: null as unknown as JobQueue, auth: { userFromRequest: async (req: Request) => users[req.headers.get("x-user") ?? ""] ?? null }, } satisfies ServerContext; - const feature = studioFeature(ctx, { watch: false }); + const feature = studioFeature(ctx, { watch: false, ...featureOpts }); + // the real queue, as composeServer builds it from the features' jobs + const queue = createJobQueue(db, feature.jobs ?? {}, { pollMs: 20 }); + ctx.jobs = queue; + queue.start(); const app = createApp(ctx, [feature]); const call = async (as: string | null, method: string, path: string, body?: unknown) => { const headers: Record = { origin: "http://x" }; @@ -51,5 +55,14 @@ export async function setupApp(extraCourseDirs: string[] = [], dbUrl = "sqlite:/ try { json = JSON.parse(text); } catch { json = text; } return { status: r.status, json }; }; - return { ctx, db, dataDir, feature, app, call, stop: async () => { feature.stop(); await db.close(); await dropSchema(); } }; + /** Polls a job until it is done or failed. */ + const waitJob = async (id: string) => { + for (let i = 0; i < 500; i++) { + const j = await queue.get(id); + if (j && (j.status === "done" || j.status === "failed")) return j; + await Bun.sleep(20); + } + throw new Error(`job ${id} did not finish`); + }; + return { ctx, db, dataDir, feature, app, call, waitJob, stop: async () => { feature.stop(); await queue.stop(); await db.close(); await dropSchema(); } }; } diff --git a/packages/feature-studio/test/convert-routes.test.ts b/packages/feature-studio/test/convert-routes.test.ts new file mode 100644 index 0000000..29cdc23 --- /dev/null +++ b/packages/feature-studio/test/convert-routes.test.ts @@ -0,0 +1,231 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { studioFeature } from "../src/server/feature"; +import { SimulatedCrash } from "../src/server/convert"; +import { createStudioStore } from "../src/server/store"; +import { REPO, setupApp } from "./app-helpers"; +import { makeImportedCourse, VIDEO, videoBytes } from "./import-fixture"; + +let env: Awaited>; +afterEach(() => env?.stop()); + +const PG = process.env.STC_TEST_PG_URL; +const dbs: [string, string][] = [["sqlite", "sqlite://:memory:"], ...(PG ? [["postgres", PG] as [string, string]] : [])]; + +async function convert(as = "owner", id = "imp") { + const r = await env.call(as, "POST", `/api/studio/courses/${id}/convert`); + expect(r.status).toBe(202); + const job = await env.waitJob(r.json.jobId); + expect(job.error).toBeNull(); + expect(job.status).toBe("done"); + return job; +} + +describe.each(dbs)("convert routes [%s]", (_name, dbUrl) => { + test("the owner converts; the course is then served from the published worktree, still restricted, ACL kept", async () => { + env = await setupApp([], dbUrl, (d) => makeImportedCourse(d)); + const { call, ctx, db } = env; + await db.sql`INSERT INTO course_acl (course_id, principal_type, principal_id) VALUES ('imp', 'user', 'u-lea')`; + expect(ctx.courses.get("imp")?.root).toBe(join(env.dataDir, "courses/imp")); + + const ro = (await call("owner", "GET", "/api/studio/courses/imp")).json; + expect(ro).toMatchObject({ readOnly: true, convertible: true }); + expect((await call("ada", "GET", "/api/studio/courses/imp")).json).toMatchObject({ readOnly: true, convertible: false }); + expect((await call("owner", "GET", "/api/studio/courses")).json.courses).toContainEqual(expect.objectContaining({ id: "imp", managed: false, imported: true, convertible: true })); + + const job = await convert(); + expect(job.result).toMatchObject({ courseId: "imp", media: 3, resumed: false }); + const c = ctx.courses.get("imp")!; + expect(c.root).toBe(join(env.dataDir, "worktrees/imp/published")); + expect(c.status).toBe("ok"); + expect(c.manifest?.visibility).toBe("restricted"); + expect(ctx.courses.list().filter((x) => x.id === "imp")).toHaveLength(1); + expect(await db.sql`SELECT principal_id FROM course_acl WHERE course_id = 'imp'`).toEqual([{ principal_id: "u-lea" }]); + + // media through core's route, Range included + const r = await env.app.fetch(new Request(`http://x/api/media/imp/${VIDEO.split("/").map(encodeURIComponent).join("/")}`, { headers: { "x-user": "lea", range: "bytes=10-19" } })); + expect(r.status).toBe(206); + expect(new Uint8Array(await r.arrayBuffer())).toEqual(videoBytes(4096, 7).slice(10, 20)); + // anonymous users still see nothing + expect((await env.app.fetch(new Request(`http://x/api/media/imp/${encodeURI(VIDEO)}`))).status).toBe(404); + + // the studio now edits it + const st = (await call("owner", "GET", "/api/studio/courses/imp")).json; + expect(st.readOnly).toBe(false); + expect(st.files).toContain(VIDEO); + expect((await call("owner", "GET", "/api/studio/courses")).json.courses).toContainEqual(expect.objectContaining({ id: "imp", managed: true })); + }); + + test("an author edits and publishes a lesson title but cannot make the course public; the owner can", async () => { + env = await setupApp([], dbUrl, (d) => makeImportedCourse(d)); + const { call, ctx } = env; + await convert(); + let st = (await call("ada", "GET", "/api/studio/courses/imp")).json; + const m = structuredClone(st.manifestRaw); + m.modules[0].lessons[0].title = "Welcome aboard"; + expect((await call("ada", "PUT", "/api/studio/courses/imp/files", { message: "Rename", manifest: m, manifestBaseHash: st.manifestHash })).status).toBe(200); + expect((await call("ada", "POST", "/api/studio/courses/imp/publish")).json).toMatchObject({ ok: true, published: true }); + expect(ctx.courses.get("imp")?.lessonIndex.get("l1")?.lesson.title).toBe("Welcome aboard"); + + st = (await call("ada", "GET", "/api/studio/courses/imp")).json; + const pub = structuredClone(st.manifestRaw); + pub.visibility = "public"; + expect((await call("ada", "PUT", "/api/studio/courses/imp/files", { message: "Go public", manifest: pub, manifestBaseHash: st.manifestHash })).status).toBe(403); + st = (await call("owner", "GET", "/api/studio/courses/imp")).json; + expect((await call("owner", "PUT", "/api/studio/courses/imp/files", { message: "Go public", manifest: { ...st.manifestRaw, visibility: "public" }, manifestBaseHash: st.manifestHash })).status).toBe(200); + expect((await call("owner", "POST", "/api/studio/courses/imp/publish")).status).toBe(200); + expect(ctx.courses.get("imp")?.manifest?.visibility).toBe("public"); + }); +}); + +describe.each(dbs)("conversion and importer jobs [%s]", (_name, dbUrl) => { + test("409 while an import job for the course is queued; the job itself re-checks", async () => { + env = await setupApp([], dbUrl, (d) => makeImportedCourse(d)); + // no importer feature in this app: its jobs stay queued + await env.ctx.jobs.enqueue("import.code-repo", { kind: "code-repo", path: "/x", options: { courseId: "imp" } }); + const blocker = await env.ctx.jobs.list("import.code-repo"); + const r = await env.call("owner", "POST", "/api/studio/courses/imp/convert"); + expect(r.status).toBe(409); + expect(r.json.error).toContain("an import job for this course is still running"); + expect(r.json.error).toContain(`import.code-repo ${blocker[0]!.id}`); + // a convert job that was queued anyway (before the import was) fails cleanly and changes nothing + const id = await env.ctx.jobs.enqueue("studio.convert-import", { courseId: "imp", actor: { id: "u-own", name: "O", email: "o@x" } }); + const j = await env.waitJob(id); + expect(j.status).toBe("failed"); + expect(j.error).toContain("an import job for this course is still running"); + expect(existsSync(join(env.dataDir, "courses/imp/stcourse.yaml"))).toBe(true); + expect(existsSync(join(env.dataDir, "studio/repos/imp.git"))).toBe(false); + // an import job for another course does not block + await env.db.sql`DELETE FROM jobs WHERE kind = 'import.code-repo'`; + await env.ctx.jobs.enqueue("import.transcribe", { courseId: "other" }); + expect((await env.call("owner", "POST", "/api/studio/courses/imp/convert")).status).toBe(202); + }); + + test("server start recovers a conversion that crashed mid-move; learners see the course once", async () => { + env = await setupApp([], dbUrl, (d) => makeImportedCourse(d)); + const crashing = createStudioStore({ dataDir: env.dataDir, testHooks: { crashAt: "staging-copied", forceExdev: true } }); + await expect(crashing.convertImported("imp", { name: "O", email: "o@x" })).rejects.toBeInstanceOf(SimulatedCrash); + await env.ctx.courses.reload(); + expect(env.ctx.courses.list().filter((c) => c.id === "imp").length).toBeLessThanOrEqual(1); + // a restart: a new studio feature recovers before serving + const restarted = studioFeature(env.ctx, { watch: false }); + await restarted.ready; + restarted.stop(); + const loaded = env.ctx.courses.list().filter((c) => c.id === "imp"); + expect(loaded).toHaveLength(1); + expect(loaded[0]!.root).toBe(join(env.dataDir, "worktrees/imp/published")); + expect(existsSync(join(env.dataDir, "courses/imp"))).toBe(false); + const r = await env.app.fetch(new Request(`http://x/api/media/imp/${VIDEO.split("/").map(encodeURIComponent).join("/")}`, { headers: { "x-user": "owner", range: "bytes=0-9" } })); + expect(r.status).toBe(206); + // the requeued job (or the owner) retrying reports success + expect((await env.call("owner", "POST", "/api/studio/courses/imp/convert")).status).toBe(409); + expect(await env.feature.store.convertImported("imp", { name: "O", email: "o@x" })).toMatchObject({ resumed: true }); + }); +}); + +describe.each(dbs)("a conversion that fails without a restart [%s]", (_name, dbUrl) => { + const media = (as: string) => env.app.fetch(new Request(`http://x/api/media/imp/${VIDEO.split("/").map(encodeURIComponent).join("/")}`, { headers: { "x-user": as, range: "bytes=10-19" } })); + const learnerView = async (where: "import" | "studio") => { + const all = env.ctx.courses.list().filter((c) => c.id === "imp"); + expect(all).toHaveLength(1); + expect(all[0]!.status).toBe("ok"); + expect(all[0]!.root).toBe(join(env.dataDir, where === "import" ? "courses/imp" : "worktrees/imp/published")); + const r = await media("lea"); + expect(r.status).toBe(206); + expect(new Uint8Array(await r.arrayBuffer())).toEqual(videoBytes(4096, 7).slice(10, 20)); + }; + + const cases: [string, { forceExdev?: boolean; failAt?: string; corruptStaging?: boolean }, "import" | "studio", RegExp][] = [ + ["the cross-filesystem copy fails (disk full)", { forceExdev: true, failAt: "staging-copied" }, "studio", /injected failure at staging-copied/], + ["the copied media fail their checksum", { forceExdev: true, corruptStaging: true }, "studio", /do not match/], + ["the checkout fails before the commit point", { failAt: "worktree" }, "import", /injected failure at worktree/], + ]; + test.each(cases)("%s: learners keep a working course and the owner's retry completes", async (_c, hooks, after, error) => { + env = await setupApp([], dbUrl, (d) => makeImportedCourse(d), {}, { storeTestHooks: hooks }); + await env.db.sql`INSERT INTO course_acl (course_id, principal_type, principal_id) VALUES ('imp', 'user', 'u-lea')`; + await learnerView("import"); + + const r = await env.call("owner", "POST", "/api/studio/courses/imp/convert"); + expect(r.status).toBe(202); + const j = await env.waitJob(r.json.jobId); + expect(j.status).toBe("failed"); + expect(j.error).toMatch(error); + // no restart: learners already get the intact import or the finished studio course + await learnerView(after); + + if (after === "studio") { + const st = (await env.call("owner", "GET", "/api/studio/courses/imp")).json; + expect(st).toMatchObject({ readOnly: false, conversionIncomplete: true, canRetryConversion: true }); + expect((await env.call("ada", "GET", "/api/studio/courses/imp")).json).toMatchObject({ conversionIncomplete: true, canRetryConversion: false }); + } else { + expect((await env.call("owner", "GET", "/api/studio/courses/imp")).json).toMatchObject({ readOnly: true, convertible: true }); + } + + // the cause goes away (space freed, disk fixed); the owner retries through the same route + delete hooks.failAt; + delete hooks.corruptStaging; + expect((await env.call("ada", "POST", "/api/studio/courses/imp/convert")).status).toBe(403); + const again = await env.call("owner", "POST", "/api/studio/courses/imp/convert"); + expect(again.status).toBe(202); + const j2 = await env.waitJob(again.json.jobId); + expect(j2.error).toBeNull(); + expect(j2.status).toBe("done"); + await learnerView("studio"); + expect((await env.call("owner", "GET", "/api/studio/courses/imp")).json.conversionIncomplete).toBe(false); + expect(readdirSync(join(env.dataDir, "courses"))).toEqual([]); + expect(readdirSync(join(env.dataDir, "media"))).toEqual(["imp"]); + expect((await env.call("owner", "POST", "/api/studio/courses/imp/convert")).status).toBe(409); + }); +}); + +describe("convert route refusals", () => { + test("only the owner; cross-site requests are refused; nothing changes", async () => { + env = await setupApp([], undefined, (d) => makeImportedCourse(d)); + for (const who of ["ada", "lea"]) expect((await env.call(who, "POST", "/api/studio/courses/imp/convert")).status).toBe(403); + expect((await env.call(null, "POST", "/api/studio/courses/imp/convert")).status).toBe(401); + const forged = await env.app.fetch(new Request("http://x/api/studio/courses/imp/convert", { method: "POST", headers: { "x-user": "owner", origin: "http://evil.example" } })); + expect(forged.status).toBe(403); + const noOrigin = await env.app.fetch(new Request("http://x/api/studio/courses/imp/convert", { method: "POST", headers: { "x-user": "owner" } })); + expect(noOrigin.status).toBe(403); + expect(existsSync(join(env.dataDir, "courses/imp/stcourse.yaml"))).toBe(true); + expect(existsSync(join(env.dataDir, "studio/repos/imp.git"))).toBe(false); + expect(await env.ctx.jobs.list("studio.convert-import")).toEqual([]); + }); + + test("overlay, studio and unknown courses; a second request while one is pending", async () => { + env = await setupApp([resolve(REPO, "fixtures/courses")], undefined, (d) => makeImportedCourse(d)); + expect((await env.call("owner", "POST", "/api/studio/courses/sample/convert")).status).toBe(409); + expect((await env.call("owner", "POST", "/api/studio/courses/nope/convert")).status).toBe(404); + await env.call("owner", "POST", "/api/studio/courses", { id: "c1", title: "C" }); + expect((await env.call("owner", "POST", "/api/studio/courses/c1/convert")).status).toBe(409); + + await env.ctx.jobs.enqueue("studio.convert-import", { courseId: "imp", actor: { id: "u-own", name: "O", email: "o@x" } }); + // whether the first job is still queued, running or already done, a second request is refused + const r = await env.call("owner", "POST", "/api/studio/courses/imp/convert"); + expect(r.status).toBe(409); + }); + + test("job status is owner-only and limited to studio jobs", async () => { + env = await setupApp([], undefined, (d) => makeImportedCourse(d)); + const { jobId } = (await env.call("owner", "POST", "/api/studio/courses/imp/convert")).json; + expect((await env.call("ada", "GET", `/api/studio/jobs/${jobId}`)).status).toBe(403); + await env.waitJob(jobId); + const j = (await env.call("owner", "GET", `/api/studio/jobs/${jobId}`)).json; + expect(j).toMatchObject({ id: jobId, status: "done", progress: 1, error: null }); + const other = await env.ctx.jobs.enqueue("import.transcribe", { courseId: "x" }); + expect((await env.call("owner", "GET", `/api/studio/jobs/${other}`)).status).toBe(404); + }); + + test("a failed conversion reports the error and leaves the import as it was", async () => { + env = await setupApp([], undefined, (d) => makeImportedCourse(d)); + const manifest = join(env.dataDir, "courses/imp/stcourse.yaml"); + // the route only checks that this is an imported course; the job validates the manifest fully + await Bun.write(manifest, readFileSync(manifest, "utf8").replace("id: imp", "id: other")); + const { jobId } = (await env.call("owner", "POST", "/api/studio/courses/imp/convert")).json; + const j = await env.waitJob(jobId); + expect(j.status).toBe("failed"); + expect(j.error).toContain('"other"'); + expect(existsSync(join(env.dataDir, "studio/repos/imp.git"))).toBe(false); + }); +}); diff --git a/packages/feature-studio/test/convert-ui.test.tsx b/packages/feature-studio/test/convert-ui.test.tsx new file mode 100644 index 0000000..c8359a8 --- /dev/null +++ b/packages/feature-studio/test/convert-ui.test.tsx @@ -0,0 +1,51 @@ +import { expect, test } from "bun:test"; +import { renderToString } from "react-dom/server"; +import { ConvertPanelView, IncompleteConversionView, ReadOnlyView } from "../src/ui/ConvertPanel"; + +const noop = () => {}; + +test("the owner sees the action and what it does", () => { + const html = renderToString(); + expect(html).toContain("Make editable in the studio"); + expect(html).toContain("same id"); + expect(html).toContain("progress and notes"); + expect(html).toContain("outside git"); + expect(html).toContain(" { + const running = renderToString(); + expect(running).toContain('role="progressbar"'); + expect(running).toContain('aria-valuenow="40"'); + expect(running).toContain("Checksumming media/a.mp4"); + expect(running).toContain("disabled"); + const failed = renderToString(); + expect(failed).toContain('role="alert"'); + expect(failed).toContain("the imported manifest is invalid"); + expect(failed).toContain("Try again"); +}); + +test("read-only view: imported for the owner offers the action; for others explains who can", () => { + const owner = renderToString(); + expect(owner).toContain("Make editable in the studio"); + expect(owner).not.toContain("/d/courses/imp"); + const author = renderToString(); + expect(author).not.toContain("); + expect(outside).toContain("/repos/o"); + expect(outside).not.toContain(" { + const owner = renderToString(); + expect(owner).toContain("Conversion incomplete"); + expect(owner).toContain(">Retry"); + const author = renderToString(); + expect(author).toContain("Conversion incomplete"); + expect(author).not.toContain("); + expect(failed).toContain('role="alert"'); + expect(failed).toContain("disk full"); +}); diff --git a/packages/feature-studio/test/convert.test.ts b/packages/feature-studio/test/convert.test.ts new file mode 100644 index 0000000..a22408a --- /dev/null +++ b/packages/feature-studio/test/convert.test.ts @@ -0,0 +1,299 @@ +import { describe, expect, test } from "bun:test"; +import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { loadAll, loadCourse } from "@stc/course-format"; +import { createStudioStore } from "../src/server/store"; +import { SimulatedCrash } from "../src/server/convert"; +import { gitignoreLine, MEDIA_REFS, parseMediaRefs } from "../src/server/media-refs"; +import { BIG_PDF, makeImportedCourse, VIDEO, VIDEO2, videoBytes } from "./import-fixture"; +import { ada, extGit, newStore, tempDir } from "./helpers"; + +const ino = (p: string) => statSync(p).ino; +const tracked = async (dir: string) => (await extGit(dir, "-c", "core.quotepath=false", "ls-files")).split("\n").filter(Boolean).sort(); + +describe("convertImported", () => { + test("builds the repo and published worktree; large media stay out of git as hard links", async () => { + const { dataDir, store } = await newStore(); + const src = makeImportedCourse(dataDir, { hostile: true }); + const srcVideoIno = ino(join(src, VIDEO)); + const r = await store.convertImported("imp", ada); + expect(r.media).toBe(3); + expect(store.isManaged("imp")).toBe(true); + expect(existsSync(join(dataDir, "studio/repos/imp.git/HEAD"))).toBe(true); + expect(existsSync(src)).toBe(false); + const mediaRoot = join(dataDir, "media/imp"); + expect(existsSync(join(mediaRoot, "stcourse.yaml"))).toBe(true); + + const pub = store.paths("imp").published; + expect(await tracked(pub)).toEqual([ + MEDIA_REFS, "code/S01 - Basics/.gitignore", "code/S01 - Basics/README.md", "media/cover.jpg", "stcourse.yaml", "transcripts/l1.md", + ]); + // the symlink was skipped, never followed; .git contents never copied + expect(existsSync(join(pub, "code/leak.txt"))).toBe(false); + expect(existsSync(join(pub, "code/.git"))).toBe(false); + expect(await extGit(pub, "status", "--porcelain", "--untracked-files=all")).toBe(""); + // the exclusions live in the repo's info/exclude, which authors and remotes cannot change + const exclude = readFileSync(join(dataDir, "studio/repos/imp.git/info/exclude"), "utf8"); + for (const p of [VIDEO, VIDEO2, BIG_PDF]) expect(exclude).toContain(gitignoreLine(p)); + + const refs = parseMediaRefs(readFileSync(join(pub, MEDIA_REFS), "utf8"))!; + expect(refs.map((x) => x.path).sort()).toEqual([BIG_PDF, VIDEO2, VIDEO].sort()); + const v = refs.find((x) => x.path === VIDEO)!; + expect(v.size).toBe(4096); + expect(v.sha256).toBe(new Bun.CryptoHasher("sha256").update(videoBytes(4096, 7)).digest("hex")); + for (const p of [VIDEO, VIDEO2, BIG_PDF]) expect(ino(join(pub, p))).toBe(ino(join(mediaRoot, p))); + expect(ino(join(mediaRoot, VIDEO))).toBe(srcVideoIno); + + // the course loads from the published worktree exactly as before + const c = await loadCourse(join(pub, "stcourse.yaml")); + expect(c.status).toBe("ok"); + expect(c.manifest?.visibility).toBe("restricted"); + expect(c.manifest?.source?.kind).toBe("video-folder"); + // the commit is attributed to the converting owner + expect(await extGit(pub, "log", "-1", "--format=%an <%ae>")).toBe(`${ada.name} <${ada.email}>`); + }); + + test("a draft gets the media; listFiles shows them; the media and reference file are read-only", async () => { + const { dataDir, store } = await newStore(); + makeImportedCourse(dataDir); + await store.convertImported("imp", ada); + const draft = await store.ensureDraft("imp", ada.id); + expect(ino(join(draft, VIDEO))).toBe(ino(join(dataDir, "media/imp", VIDEO))); + const files = await store.listFiles("imp", ada.id); + expect(files).toContain(VIDEO); + expect(files).toContain(BIG_PDF); + for (const path of [VIDEO, MEDIA_REFS]) { + await expect(store.saveFiles("imp", ada, [{ path, content: "x" }], "x")).rejects.toMatchObject({ status: 409 }); + await expect(store.saveFiles("imp", ada, [{ path, delete: true }], "x")).rejects.toMatchObject({ status: 409 }); + } + expect(readFileSync(join(dataDir, "media/imp", VIDEO))).toEqual(Buffer.from(videoBytes(4096, 7))); + }); + + test("a studio write to a hard-linked file breaks the link instead of writing through it", async () => { + const { dataDir, store } = await newStore(); + makeImportedCourse(dataDir); + await store.convertImported("imp", ada); + const draft = await store.ensureDraft("imp", ada.id); + // a planted hard link at an ordinary (non-referenced) path + const outside = join(dataDir, "media/imp/transcripts/l1.md"); + rmSync(join(draft, "transcripts/l1.md")); + const { linkSync } = await import("node:fs"); + linkSync(outside, join(draft, "transcripts/l1.md")); + expect((await store.saveFiles("imp", ada, [{ path: "transcripts/l1.md", content: "# changed\n" }], "edit")).ok).toBe(true); + expect(readFileSync(outside, "utf8")).toBe("# Intro\n\n[0:00] Hello.\n"); + expect(readFileSync(join(draft, "transcripts/l1.md"), "utf8")).toBe("# changed\n"); + }); + + test("edit and publish: the published worktree keeps its media and stays clean", async () => { + const { dataDir, store } = await newStore(); + makeImportedCourse(dataDir); + await store.convertImported("imp", ada); + const m = (await store.readFile("imp", ada.id, "stcourse.yaml"))!; + const next = m.content.replace("title: Intro", "title: Welcome aboard"); + expect((await store.saveFiles("imp", ada, [{ path: "stcourse.yaml", content: next, baseHash: m.hash }], "Rename lesson")).ok).toBe(true); + const r = await store.publish("imp", ada); + expect(r).toMatchObject({ ok: true, published: true }); + const pub = store.paths("imp").published; + expect(readFileSync(join(pub, "stcourse.yaml"), "utf8")).toContain("title: Welcome aboard"); + expect(ino(join(pub, VIDEO))).toBe(ino(join(dataDir, "media/imp", VIDEO))); + expect(await extGit(pub, "status", "--porcelain", "--untracked-files=all")).toBe(""); + }); + + test("a non-owner cannot publish the converted course as public; the owner can", async () => { + const { dataDir, store } = await newStore(); + makeImportedCourse(dataDir); + await store.convertImported("imp", ada); + const m = (await store.readFile("imp", ada.id, "stcourse.yaml"))!; + await store.saveFiles("imp", ada, [{ path: "stcourse.yaml", content: m.content.replace("visibility: restricted", "visibility: public"), baseHash: m.hash }], "go public"); + expect(await store.publish("imp", ada)).toMatchObject({ ok: false, reason: "forbidden" }); + expect(await store.publish("imp", { ...ada, owner: true })).toMatchObject({ ok: true, published: true }); + }); + + describe("refusals leave the import untouched and no repo behind", () => { + const cases: [string, (dataDir: string) => void, number][] = [ + ["no such import", () => {}, 404], + ["an authored course", (d) => makeImportedCourse(d, { kind: "authored" }), 400], + ["a manifest id mismatch", (d) => makeImportedCourse(d, { manifestId: "other" }), 400], + ["a root: field", (d) => makeImportedCourse(d, { root: "." }), 400], + ["a media store already there", (d) => { makeImportedCourse(d); mkdirSync(join(d, "media/imp"), { recursive: true }); }, 409], + ]; + test.each(cases)("%s", async (_name, prepare, status) => { + const { dataDir, store } = await newStore(); + prepare(dataDir); + const before = existsSync(join(dataDir, "courses/imp/stcourse.yaml")) ? readFileSync(join(dataDir, "courses/imp/stcourse.yaml"), "utf8") : null; + await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status }); + expect(existsSync(join(dataDir, "studio/repos/imp.git"))).toBe(false); + expect(existsSync(join(dataDir, "worktrees/imp"))).toBe(false); + if (before !== null) expect(readFileSync(join(dataDir, "courses/imp/stcourse.yaml"), "utf8")).toBe(before); + }); + + test("a studio course with the same id", async () => { + const { store } = await newStore(); + await store.createCourse({ id: "imp", title: "Studio" }, ada); + await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status: 409 }); + expect(store.isManaged("imp")).toBe(true); + }); + + test("a large file whose name has a line break", async () => { + const { dataDir, store } = await newStore(); + const src = makeImportedCourse(dataDir); + writeFileSync(join(src, "media/bad\nname.mp4"), "x"); + await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status: 400 }); + expect(existsSync(join(dataDir, "studio/repos/imp.git"))).toBe(false); + expect(existsSync(join(src, "stcourse.yaml"))).toBe(true); + }); + }); + + describe("restart safety", () => { + test("a leftover partial repo (no published worktree) is rebuilt", async () => { + const { dataDir, store } = await newStore(); + makeImportedCourse(dataDir); + mkdirSync(join(dataDir, "studio/repos/imp.git"), { recursive: true }); + writeFileSync(join(dataDir, "studio/repos/imp.git/junk"), "x"); + await store.convertImported("imp", ada); + expect(store.isManaged("imp")).toBe(true); + expect(existsSync(join(dataDir, "studio/repos/imp.git/junk"))).toBe(false); + }); + + test("a managed course whose import folder is still there only finishes the move", async () => { + const { dataDir, store } = await newStore(); + makeImportedCourse(dataDir); + await store.convertImported("imp", ada); + const head = await store.publishedHead("imp"); + // simulate a crash between the checkout and the move + const { renameSync } = await import("node:fs"); + renameSync(join(dataDir, "media/imp"), join(dataDir, "courses/imp")); + const r = await store.convertImported("imp", ada); + expect(r.resumed).toBe(true); + expect(await store.publishedHead("imp")).toBe(head); + expect(existsSync(join(dataDir, "courses/imp"))).toBe(false); + expect(existsSync(join(dataDir, "media/imp", VIDEO))).toBe(true); + }); + }); +}); + +describe("an author's or a remote's .gitignore never blocks the course", () => { + const editManifest = async (store: Awaited>["store"], title: string) => { + const m = (await store.readFile("imp", ada.id, "stcourse.yaml"))!; + const text = m.content.replace(/title: (Intro|Welcome \d+)\n/, `title: ${title}\n`); + expect((await store.saveFiles("imp", ada, [{ path: "stcourse.yaml", content: text, baseHash: m.hash }], title)).ok).toBe(true); + }; + + test("un-ignoring the media in .gitignore: publish keeps working, again and again", async () => { + const { dataDir, store } = await newStore(); + makeImportedCourse(dataDir); + await store.convertImported("imp", ada); + expect((await store.saveFiles("imp", ada, [{ path: ".gitignore", content: "!*.mp4\n!*.pdf\n!/media/**\n" }], "tidy .gitignore")).ok).toBe(true); + expect(await store.publish("imp", ada)).toMatchObject({ ok: true, published: true }); + const pub = store.paths("imp").published; + // git itself now sees the media as untracked; the studio does not count them + expect(await extGit(pub, "-c", "core.quotepath=false", "status", "--porcelain")).toContain("media/"); + expect((await store.status("imp", ada.id)).publishedDirty).toEqual([]); + await editManifest(store, "Welcome 2"); + expect(await store.publish("imp", ada)).toMatchObject({ ok: true, published: true }); + await editManifest(store, "Welcome 3"); + expect(await store.publish("imp", ada)).toMatchObject({ ok: true, published: true }); + expect(readFileSync(join(pub, "stcourse.yaml"), "utf8")).toContain("title: Welcome 3"); + }); + + test("a remote commit that rewrites .gitignore: Sync and publish keep working", async () => { + const { dataDir, store } = await newStore({ allowLocalRemotes: true }); + makeImportedCourse(dataDir); + await store.convertImported("imp", ada); + const remote = join(tempDir(), "remote.git"); + await extGit(tempDir(), "init", "-q", "--bare", "-b", "main", remote); + await store.setRemote("imp", remote); + expect(await store.sync("imp", ada)).toMatchObject({ ok: true, pushed: true }); + const other = join(tempDir(), "other"); + await extGit(tempDir(), "clone", "-q", remote, other); + writeFileSync(join(other, ".gitignore"), "!*\n"); + await extGit(other, "add", ".gitignore"); + await extGit(other, "commit", "-qm", "unignore everything"); + await extGit(other, "push", "-q", "origin", "main"); + expect(await store.sync("imp", ada)).toMatchObject({ ok: true, pulled: true }); + expect(await store.updateDraft("imp", ada)).toMatchObject({ ok: true }); + await editManifest(store, "Welcome 2"); + expect(await store.publish("imp", ada)).toMatchObject({ ok: true, published: true }); + expect(await store.sync("imp", ada)).toMatchObject({ ok: true, pushed: true }); + }); + + test("a remote commit that puts a file over referenced media is refused at Sync; the media stay", async () => { + const { dataDir, store } = await newStore({ allowLocalRemotes: true }); + makeImportedCourse(dataDir); + await store.convertImported("imp", ada); + const remote = join(tempDir(), "remote.git"); + await extGit(tempDir(), "init", "-q", "--bare", "-b", "main", remote); + await store.setRemote("imp", remote); + await store.sync("imp", ada); + const other = join(tempDir(), "other"); + await extGit(tempDir(), "clone", "-q", remote, other); + mkdirSync(join(other, "media"), { recursive: true }); + writeFileSync(join(other, VIDEO), "not the video"); + await extGit(other, "add", "-f", VIDEO); + await extGit(other, "commit", "-qm", "replace media"); + await extGit(other, "push", "-q", "origin", "main"); + expect(await store.sync("imp", ada)).toMatchObject({ ok: false, reason: "forbidden" }); + expect(ino(join(store.paths("imp").published, VIDEO))).toBe(ino(join(dataDir, "media/imp", VIDEO))); + }); + + test("referenced media are not read whole into the editor", async () => { + const { dataDir, store } = await newStore(); + makeImportedCourse(dataDir); + await store.convertImported("imp", ada); + expect(await store.readFile("imp", ada.id, VIDEO)).toBeNull(); + expect(await store.readFile("imp", ada.id, "transcripts/l1.md")).not.toBeNull(); + }); +}); + +describe("crash safety: a retry after any crash point succeeds and the id is never loaded twice", () => { + const loaded = async (dataDir: string) => (await loadAll([join(dataDir, "courses"), join(dataDir, "worktrees")])).filter((c) => c.id === "imp"); + const sameFs = ["committed", "hidden", "worktree", "before-move"]; + const crossFs = [...sameFs, "staging-copied", "staging-placed", "source-retired"]; + const cases: [string, boolean][] = [...sameFs.map((s) => [s, false] as [string, boolean]), ...crossFs.map((s) => [s, true] as [string, boolean])]; + + test.each(cases)("crash at %s (cross-filesystem: %p)", async (step, exdev) => { + const dataDir = tempDir(); + makeImportedCourse(dataDir); + const crashing = createStudioStore({ dataDir, testHooks: { crashAt: step, forceExdev: exdev } }); + await expect(crashing.convertImported("imp", ada)).rejects.toBeInstanceOf(SimulatedCrash); + expect((await loaded(dataDir)).length).toBeLessThanOrEqual(1); + + // the process restarts: recovery first, then the owner (or the requeued job) retries + const fresh = createStudioStore({ dataDir, testHooks: { forceExdev: exdev } }); + await fresh.recoverConversions(); + expect((await loaded(dataDir)).length).toBeLessThanOrEqual(1); + await fresh.convertImported("imp", ada); + // and once more: a re-run of a finished conversion reports success + expect(await fresh.convertImported("imp", ada)).toMatchObject({ resumed: true }); + + expect(fresh.isManaged("imp")).toBe(true); + const all = await loaded(dataDir); + expect(all).toHaveLength(1); + expect(all[0]!.status).toBe("ok"); + expect(all[0]!.root).toBe(fresh.paths("imp").published); + expect(readdirSync(join(dataDir, "courses"))).toEqual([]); + expect(readdirSync(join(dataDir, "media"))).toEqual(["imp"]); + const pub = fresh.paths("imp").published; + for (const p of [VIDEO, VIDEO2, BIG_PDF]) { + expect(statSync(join(pub, p)).size).toBe(statSync(join(dataDir, "media/imp", p)).size); + if (!exdev) expect(ino(join(pub, p))).toBe(ino(join(dataDir, "media/imp", p))); + } + expect(readFileSync(join(dataDir, "media/imp", VIDEO))).toEqual(Buffer.from(videoBytes(4096, 7))); + expect(await extGit(pub, "status", "--porcelain", "--untracked-files=all")).toBe(""); + }); + + test("a failed cross-filesystem copy leaves no staging dir and a retry finishes", async () => { + const dataDir = tempDir(); + makeImportedCourse(dataDir); + const store = createStudioStore({ dataDir, testHooks: { forceExdev: true } }); + // the media store's parent is a file: the copy fails in-process + writeFileSync(join(dataDir, "media"), "not a dir"); + await expect(store.convertImported("imp", ada)).rejects.toThrow(); + expect((await loaded(dataDir)).length).toBeLessThanOrEqual(1); + rmSync(join(dataDir, "media")); + await store.convertImported("imp", ada); + expect(readdirSync(join(dataDir, "media"))).toEqual(["imp"]); + expect(readdirSync(join(dataDir, "courses"))).toEqual([]); + expect(await loaded(dataDir)).toHaveLength(1); + }); +}); diff --git a/packages/feature-studio/test/import-fixture.ts b/packages/feature-studio/test/import-fixture.ts new file mode 100644 index 0000000..007c77f --- /dev/null +++ b/packages/feature-studio/test/import-fixture.ts @@ -0,0 +1,86 @@ +// A synthetic imported course, laid out as the importer leaves it in /courses/. Nothing here is real material. +import { mkdirSync, symlinkSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { LARGE_FILE_BYTES } from "../src/server/media-refs"; +import { tempDir } from "./helpers"; + +export const VIDEO = "media/01 Intro – Übersicht.mp4"; +export const VIDEO2 = "media/[2] Next step.mp4"; +export const BIG_PDF = "decks/big deck.pdf"; +/** Fake MP4 bytes: the server only streams them, nothing decodes them. */ +export const videoBytes = (n = 4096, seed = 7) => new Uint8Array(n).map((_, i) => (i * seed) % 256); + +export interface ImportFixtureOpts { + id?: string; + kind?: string; + manifestId?: string; + root?: string; + visibility?: string; + /** Extra hostile bits: a symlink to outside, a .git dir, a nested .gitignore that hides a lesson. */ + hostile?: boolean; +} + +function put(dir: string, rel: string, content: string | Uint8Array) { + mkdirSync(dirname(join(dir, rel)), { recursive: true }); + writeFileSync(join(dir, rel), content); +} + +export function manifestYaml(o: ImportFixtureOpts = {}): string { + const id = o.manifestId ?? o.id ?? "imp"; + return [ + "schema: 1", + `id: ${id}`, + "title: Imported Course", + `visibility: ${o.visibility ?? "restricted"}`, + ...(o.root ? [`root: ${o.root}`] : []), + "cover: media/cover.jpg", + "source:", + ` kind: ${o.kind ?? "video-folder"}`, + " origin: /import/synthetic", + "decks:", + " - id: d1", + ` file: ${BIG_PDF}`, + " attachTo: l1", + "modules:", + " - id: m1", + " title: Videos", + " lessons:", + " - id: l1", + " title: Intro", + " type: video", + ` file: "${VIDEO}"`, + " transcript: transcripts/l1.md", + " - id: l2", + " title: Next step", + " type: video", + ` file: "${VIDEO2}"`, + " - id: notes", + " title: Reading", + " type: text", + " file: code/S01 - Basics/README.md", + "", + ].join("\n"); +} + +/** Writes the course into `/courses/` and returns that folder. */ +export function makeImportedCourse(dataDir: string, o: ImportFixtureOpts = {}): string { + const id = o.id ?? "imp"; + const dir = join(dataDir, "courses", id); + mkdirSync(dir, { recursive: true }); + put(dir, "stcourse.yaml", manifestYaml(o)); + put(dir, VIDEO, videoBytes(4096, 7)); + put(dir, VIDEO2, videoBytes(2048, 11)); + put(dir, "media/cover.jpg", new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0, 0x10, 0x4a, 0x46, 0x49, 0x46, 0, 1, 0xff, 0xd9])); + put(dir, BIG_PDF, new Uint8Array(LARGE_FILE_BYTES + 100).fill(0x25)); + put(dir, "transcripts/l1.md", "# Intro\n\n[0:00] Hello.\n"); + put(dir, "code/S01 - Basics/README.md", "# Basics\n\nRead me.\n"); + if (o.hostile) { + // the companion code repo's own .gitignore hides Markdown: the lesson must still be committed + put(dir, "code/S01 - Basics/.gitignore", "*.md\n"); + put(dir, "code/.git/config", "[core]\n"); + const outside = join(tempDir(), "secret.txt"); + writeFileSync(outside, "outside\n"); + symlinkSync(outside, join(dir, "code/leak.txt")); + } + return dir; +} diff --git a/packages/feature-studio/test/media-refs.test.ts b/packages/feature-studio/test/media-refs.test.ts new file mode 100644 index 0000000..03d7e3a --- /dev/null +++ b/packages/feature-studio/test/media-refs.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, test } from "bun:test"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { $ } from "bun"; +import { gitignoreLine, isLargeMedia, LARGE_FILE_BYTES, mediaGitignore, parseMediaRefs, sha256File } from "../src/server/media-refs"; +import { tempDir } from "./helpers"; + +describe("isLargeMedia", () => { + test("video and audio by extension, whatever the size", () => { + for (const p of ["a.mp4", "x/B.MOV", "c.webm", "d.mkv", "e.mp3", "f.m4a", "g.wav", "h.flac", "i.avi", "j.m4v"]) expect(isLargeMedia(p, 10)).toBe(true); + }); + test("other files only above the size limit", () => { + expect(isLargeMedia("deck.pdf", 1000)).toBe(false); + expect(isLargeMedia("deck.pdf", LARGE_FILE_BYTES + 1)).toBe(true); + expect(isLargeMedia("lesson.md", LARGE_FILE_BYTES)).toBe(false); + }); +}); + +describe("gitignoreLine", () => { + test("anchors and escapes", () => { + expect(gitignoreLine("media/a.mp4")).toBe("/media/a.mp4"); + expect(gitignoreLine("media/[1] *?.mp4")).toBe("/media/\\[1\\] \\*\\?.mp4"); + expect(gitignoreLine("a\\b.mp4")).toBe("/a\\\\b.mp4"); + expect(gitignoreLine("trail.mp4 ")).toBe("/trail.mp4\\ \\ "); + }); + test("refuses names a .gitignore cannot express", () => { + expect(() => gitignoreLine("a\nb.mp4")).toThrow(); + expect(() => gitignoreLine("a\rb.mp4")).toThrow(); + }); + + test("git ignores exactly the listed paths", async () => { + const dir = tempDir(); + await $`git init -q ${dir}`.quiet(); + const listed = ["media/01 Intro – Übersicht.mp4", "media/[draft] *star?.mp4", "#hash.mp4", "!bang.mp4", "media/trailing.mp4 ", "deep/a b/c.mkv"]; + const siblings = ["media/01 Intro – Übersicht.mp4.txt", "media/d.mp4", "hash.mp4", "bang.mp4", "media/trailing.mp4", "deep/a b/c.mkv2", "sub/#hash.mp4"]; + for (const p of [...listed, ...siblings]) { + mkdirSync(join(dir, p, ".."), { recursive: true }); + writeFileSync(join(dir, p), "x"); + } + writeFileSync(join(dir, ".gitignore"), mediaGitignore(listed, null)); + const input = Buffer.from([...listed, ...siblings].join("\0") + "\0"); + const r = await $`git -c core.quotepath=false check-ignore -z --no-index --stdin < ${input}`.cwd(dir).quiet().nothrow(); + const ignored = r.stdout.toString().split("\0").filter(Boolean).sort(); + expect(ignored).toEqual([...listed].sort()); + }); + + test("an existing .gitignore is kept above the media block", () => { + const out = mediaGitignore(["a.mp4"], "node_modules/\n"); + expect(out.startsWith("node_modules/\n")).toBe(true); + expect(out).toContain("/a.mp4\n"); + expect(mediaGitignore(["a.mp4"], "dist")).toStartWith("dist\n"); + }); +}); + +test("sha256File streams the file", async () => { + const dir = tempDir(); + const bytes = new Uint8Array(3 * 1024 * 1024 + 7).map((_, i) => i % 251); + writeFileSync(join(dir, "f.bin"), bytes); + expect(await sha256File(join(dir, "f.bin"))).toBe(new Bun.CryptoHasher("sha256").update(bytes).digest("hex")); +}); + +describe("parseMediaRefs", () => { + const sha = "a".repeat(64); + test("accepts the schema", () => { + expect(parseMediaRefs(JSON.stringify({ schema: 1, files: [{ path: "media/a.mp4", size: 3, sha256: sha }] }))).toEqual([{ path: "media/a.mp4", size: 3, sha256: sha }]); + }); + test.each([ + "not json", "null", "{}", JSON.stringify({ schema: 2, files: [] }), + JSON.stringify({ schema: 1, files: [{ path: "../x.mp4", size: 1, sha256: sha }] }), + JSON.stringify({ schema: 1, files: [{ path: "/abs.mp4", size: 1, sha256: sha }] }), + JSON.stringify({ schema: 1, files: [{ path: ".git/config", size: 1, sha256: sha }] }), + JSON.stringify({ schema: 1, files: [{ path: "a.mp4", size: -1, sha256: sha }] }), + JSON.stringify({ schema: 1, files: [{ path: "a.mp4", size: 1, sha256: "zz" }] }), + ])("rejects %p", (text) => { + expect(parseMediaRefs(text)).toBeNull(); + }); +}); diff --git a/packages/feature-studio/test/security.test.ts b/packages/feature-studio/test/security.test.ts index e9b3690..0b5ec29 100644 --- a/packages/feature-studio/test/security.test.ts +++ b/packages/feature-studio/test/security.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, test } from "bun:test"; import { existsSync, mkdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs"; import { join } from "node:path"; +import { IMPORTED_MESSAGE } from "../src/server/feature"; import { isAllowedRemote, ownerRequiredFor } from "../src/server/store"; import { ada, extGit, newStore, tempDir } from "./helpers"; import { setupApp } from "./app-helpers"; @@ -229,11 +230,11 @@ describe("I4: layout and imported courses", () => { }); expect(env.ctx.courses.get("imp")?.status).toBe("ok"); const list = (await env.call("ada", "GET", "/api/studio/courses")).json; - expect(list.courses).toContainEqual(expect.objectContaining({ id: "imp", managed: false, message: "Imported course: read-only in the studio for now" })); - expect((await env.call("ada", "GET", "/api/studio/courses/imp")).json.message).toBe("Imported course: read-only in the studio for now"); + expect(list.courses).toContainEqual(expect.objectContaining({ id: "imp", managed: false, message: IMPORTED_MESSAGE })); + expect((await env.call("ada", "GET", "/api/studio/courses/imp")).json.message).toBe(IMPORTED_MESSAGE); const w = await env.call("ada", "PUT", "/api/studio/courses/imp/files", { message: "x", files: [{ path: "a.md", content: "x" }] }); expect(w.status).toBe(403); - expect(w.json.error).toBe("Imported course: read-only in the studio for now"); + expect(w.json.error).toBe(IMPORTED_MESSAGE); expect((await env.call("ada", "POST", "/api/studio/courses", { id: "imp", title: "Dup" })).status).toBe(409); // studio repos are not in the importer's folder expect((await env.call("ada", "POST", "/api/studio/courses", { id: "c1", title: "C" })).status).toBe(201);