merge: v1.1 edit imported courses in the studio

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Lucas Winther
2026-10-30 21:01:40 +01:00
co-authored by Claude Opus 5.5
35 changed files with 1962 additions and 83 deletions
+153
View File
@@ -0,0 +1,153 @@
// End to end for v1.1 S: import -> make editable in the studio -> edit -> publish, through the production feature set,
// on SQLite and Postgres. The learner must see the change and keep progress and notes; media must still answer Range.
// Synthetic material only (ffmpeg test pattern videos).
import { describe, expect, test } from "bun:test";
import { existsSync, mkdirSync, mkdtempSync, statSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { loadConfig } from "@stc/core";
import { makeVideo } from "../../../packages/feature-import/test/fixtures";
import { packageMigrations, productFeatures } from "../src/features";
import { startServer } from "../src/main";
import { PG, pgSchema } from "./pg";
const ORIGIN = "http://localhost:3000";
const cookieFrom = (r: Response) => r.headers.getSetCookie().map((c) => c.split(";")[0]).filter((c) => !c.endsWith("=")).join("; ");
const CLIP1 = "01 Intro – Übersicht.mp4";
const CLIP2 = "02 Next step.mp4";
async function run(databaseUrl: string) {
const dataDir = mkdtempSync(join(tmpdir(), "stc-e2e-si-"));
const inbox = join(dataDir, "import-inbox");
const folder = join(inbox, "Synthetic Course");
mkdirSync(folder, { recursive: true });
await makeVideo(join(folder, CLIP1), 2);
await makeVideo(join(folder, CLIP2), 1);
const config = loadConfig({
STC_PORT: "0", STC_DATABASE_URL: databaseUrl, STC_SECRET: "e2e-secret-0123456789abcdef0123456789",
STC_DATA_DIR: dataDir, STC_COURSE_DIRS: "", STC_PUBLIC_URL: ORIGIN, STC_IMPORT_ROOTS: inbox,
});
const s = await startServer(config, productFeatures({ studioWatch: false }), { migrations: packageMigrations });
const base = s.server.url.toString().replace(/\/$/, "");
type Opts = { cookie?: string; body?: unknown; headers?: Record<string, string> };
const call = (method: string, path: string, opts: Opts = {}) => {
const headers: Record<string, string> = { origin: ORIGIN, ...opts.headers };
if (opts.cookie) headers.cookie = opts.cookie;
if (opts.body !== undefined) headers["content-type"] = "application/json";
return fetch(base + path, { method, headers, body: opts.body === undefined ? undefined : JSON.stringify(opts.body) });
};
const json = async (method: string, path: string, opts: Opts = {}) => (await call(method, path, opts)).json() as Promise<any>;
const signUp = async (email: string, name: string) => {
const r = await call("POST", "/api/auth/sign-up/email", { body: { email, name, password: "correct horse battery" } });
expect(r.status).toBe(200);
return cookieFrom(r);
};
const waitJob = async (path: string, cookie: string) => {
for (let i = 0; i < 600; i++) {
const j = await json("GET", path, { cookie });
if (j.status === "done" || j.status === "failed") return j;
await Bun.sleep(50);
}
throw new Error(`timeout waiting for ${path}`);
};
try {
const owner = await signUp("[email protected]", "Olive Owner");
const lea = await signUp("[email protected]", "Lea Learner");
const leaId = (await json("GET", "/api/me", { cookie: lea })).user.id as string;
// 1. import a folder of videos (restricted by default)
const imp = await call("POST", "/api/import/jobs", { cookie: owner, body: { kind: "video-folder", path: folder, options: { courseId: "synth", title: "Synthetic Course" } } });
expect(imp.status).toBe(202);
const impId = ((await imp.json()) as { id: string }).id;
let done: any = null;
for (let i = 0; i < 600 && !done; i++) {
const j = ((await json("GET", "/api/import/jobs", { cookie: owner })) as any[]).find((x) => x.id === impId);
if (j && (j.status === "done" || j.status === "failed")) done = j;
else await Bun.sleep(50);
}
expect(done?.error ?? null).toBeNull();
expect(done.status).toBe("done");
expect(existsSync(join(dataDir, "courses/synth/stcourse.yaml"))).toBe(true);
// 2. the owner grants the learner access; the learner makes progress and writes a note
expect((await call("PUT", "/api/admin/courses/synth/access", { cookie: owner, body: { acl: [{ principal_type: "user", principal_id: leaId }] } })).status).toBe(200);
const before = await json("GET", "/api/courses/synth", { cookie: lea });
expect(before.modules[0].lessons.map((l: { id: string; title: string }) => [l.id, l.title])).toEqual([["l1", "Intro – Übersicht"], ["l2", "Next step"]]);
expect((await call("PUT", "/api/courses/synth/progress/l1", { cookie: lea, body: { state: "done" } })).status).toBe(200);
expect((await call("PUT", "/api/courses/synth/progress/l2", { cookie: lea, body: { state: "in-progress", positionSeconds: 1 } })).status).toBe(200);
expect((await call("POST", "/api/courses/synth/notes", { cookie: lea, body: { lessonId: "l2", anchorKind: "video", anchorValue: 1, body: "Remember this bit" } })).status).toBe(201);
const progressBefore = await json("GET", "/api/courses/synth/progress", { cookie: lea });
const notesBefore = await json("GET", "/api/courses/synth/notes", { cookie: lea });
expect(progressBefore.l1.state).toBe("done");
expect(notesBefore.map((n: { body: string }) => n.body)).toEqual(["Remember this bit"]);
// 3. only the owner may convert; the learner's attempt changes nothing
expect((await call("POST", "/api/studio/courses/synth/convert", { cookie: lea })).status).toBe(403);
const ro = await json("GET", "/api/studio/courses/synth", { cookie: owner });
expect(ro).toMatchObject({ readOnly: true, imported: true, convertible: true });
const conv = await call("POST", "/api/studio/courses/synth/convert", { cookie: owner });
expect(conv.status).toBe(202);
const job = await waitJob(`/api/studio/jobs/${((await conv.json()) as { jobId: string }).jobId}`, owner);
expect(job.error).toBeNull();
expect(job.result).toMatchObject({ courseId: "synth", media: 2 });
expect(existsSync(join(dataDir, "courses/synth"))).toBe(false);
expect(existsSync(join(dataDir, "studio/repos/synth.git/HEAD"))).toBe(true);
const pubClip = join(dataDir, "worktrees/synth/published/media", CLIP1);
expect(statSync(pubClip).ino).toBe(statSync(join(dataDir, "media/synth/media", CLIP1)).ino);
// 4. edit a lesson title in the studio and publish
const st = await json("GET", "/api/studio/courses/synth", { cookie: owner });
expect(st.readOnly).toBe(false);
const m = structuredClone(st.manifestRaw);
expect(m.visibility).toBe("restricted");
m.modules[0].lessons[0].title = "Welcome aboard";
expect((await call("PUT", "/api/studio/courses/synth/files", { cookie: owner, body: { message: "Rename the first lesson", manifest: m, manifestBaseHash: st.manifestHash } })).status).toBe(200);
// drafts never reach learners
expect((await json("GET", "/api/courses/synth", { cookie: lea })).modules[0].lessons[0].title).toBe("Intro – Übersicht");
const pub = await call("POST", "/api/studio/courses/synth/publish", { cookie: owner });
expect(pub.status).toBe(200);
expect(await pub.json()).toMatchObject({ ok: true, published: true });
// 5. the learner sees the change and keeps progress and notes
const after = await json("GET", "/api/courses/synth", { cookie: lea });
expect(after.modules[0].lessons.map((l: { title: string }) => l.title)).toEqual(["Welcome aboard", "Next step"]);
expect(await json("GET", "/api/courses/synth/progress", { cookie: lea })).toEqual(progressBefore);
expect(await json("GET", "/api/courses/synth/notes", { cookie: lea })).toEqual(notesBefore);
expect(after.progress.l1.state).toBe("done");
// 6. media still play: Range gives 206, an unsatisfiable range 416; the URL is unchanged
const lesson = await json("GET", "/api/courses/synth/lessons/l1", { cookie: lea });
expect(lesson.lesson.title).toBe("Welcome aboard");
expect(lesson.video.file).toBe(`media/${CLIP1}`);
const size = statSync(pubClip).size;
const part = await call("GET", lesson.video.url, { cookie: lea, headers: { range: "bytes=0-99" } });
expect(part.status).toBe(206);
expect(part.headers.get("content-range")).toBe(`bytes 0-99/${size}`);
expect((await part.arrayBuffer()).byteLength).toBe(100);
expect((await call("GET", lesson.video.url, { cookie: lea, headers: { range: `bytes=${size + 10}-` } })).status).toBe(416);
const full = await call("GET", lesson.video.url, { cookie: lea });
expect(full.status).toBe(200);
expect((await full.arrayBuffer()).byteLength).toBe(size);
// 7. still restricted: anonymous users and other accounts see nothing
expect((await call("GET", "/api/courses/synth")).status).toBe(404);
expect((await call("GET", lesson.video.url)).status).toBe(404);
const other = await signUp("[email protected]", "Max Other");
expect((await call("GET", "/api/courses/synth", { cookie: other })).status).toBe(404);
expect((await call("GET", "/api/media/synth/..%2F..%2Fmedia%2Fsynth%2Fstcourse.yaml", { cookie: lea })).status).toBe(404);
} finally {
await s.stop();
}
}
describe("studio import end to end", () => {
test("import, make editable, edit, publish [sqlite]", () => run("sqlite://:memory:"), 120_000);
const pg = PG;
if (pg) {
test("import, make editable, edit, publish [postgres]", async () => {
const db = await pgSchema(pg);
try { await run(db.url); } finally { await db.drop(); }
}, 120_000);
}
});
+1 -1
View File
@@ -22,7 +22,7 @@ Layout after the review (controller ruling on I4):
Proposal: main.ts calls `createCourseRepository(studioCourseDirs(config))`, or `loadConfig` appends both dirs.
Imported courses (`source.kind` other than `authored`, with their root under `<dataDir>/courses`) are read-only in the studio for v1, with the message "Imported course: read-only in the studio for now". There is a `TODO(studio)` in `feature.ts` (`readOnlyMessage`) for converting one into a studio-managed repo. The studio refuses to create a course whose id already exists as `<dataDir>/courses/<id>`. WP-B should likewise refuse an import id that `<dataDir>/studio/repos/<id>.git` already uses.
Imported courses (`source.kind` other than `authored`, with their root under `<dataDir>/courses`) are read-only in the studio for v1, with the message "Imported course: read-only in the studio for now". There was a `TODO(studio)` in `feature.ts` (`readOnlyMessage`) for converting one into a studio-managed repo; v1.1 S resolved it with the owner-only "Make editable in the studio" action (see `docs/operations.md`, "Imported courses in the studio"). The studio refuses to create a course whose id already exists as `<dataDir>/courses/<id>`. WP-B should likewise refuse an import id that `<dataDir>/studio/repos/<id>.git` already uses.
## 3. Live preview: swap in the learner `LessonView`
+76 -1
View File
@@ -22,7 +22,7 @@ Mounts (all course mounts are read-only; STCourses never changes your originals)
| Host | Container | What |
|---|---|---|
| volume `stc-data` | `/data` | database (SQLite), imported courses (`/data/courses`), studio repos (`/data/studio/repos`) and their published worktrees (`/data/worktrees`), caches (`/data/cache`), AI drafts for non-studio courses (`/data/drafts`) |
| volume `stc-data` | `/data` | database (SQLite), imported courses (`/data/courses`), studio repos (`/data/studio/repos`) and their published worktrees (`/data/worktrees`), media of imports made editable in the studio (`/data/media`), caches (`/data/cache`), AI drafts for non-studio courses (`/data/drafts`) |
| `STC_COURSES_HOST` (default `./courses`) | `/courses` | course repos, one folder with a `stcourse.yaml` per course |
| `STC_IMPORT_HOST` (default `./import-inbox`) | `/import` | files the owner may import (Packt zips, video folders, PDFs, code zips) |
@@ -78,6 +78,81 @@ Without Docker: `bun install`, then `STC_SECRET=… NODE_ENV=production bun apps
- Course media (`/api/media/…`) is sent with `X-Content-Type-Options: nosniff`, and SVG, HTML and JavaScript files
with `Content-Security-Policy: sandbox`, so author content cannot run script on the app's origin.
## Imported courses in the studio
Imported courses (`<data>/courses/<id>`) are read-only in the studio until the owner chooses **Make editable in the
studio** on the course's studio page. That runs a background job (`studio.convert-import`) which:
1. commits the manifest, lessons, transcripts, small images, small PDFs and code into a new studio repo,
`<data>/studio/repos/<id>.git`, checked out as `<data>/worktrees/<id>/published`. The manifest is committed
unchanged: the course keeps its id, its visibility (restricted after an import) and its `source.kind`, so only the
owner can later make it public or signed-in. ACL entries, progress and notes are keyed by course and lesson id
and are kept;
2. keeps **large media out of git**: every video or audio file (`mp4 m4v webm mov mkv avi mp3 m4a wav ogg oga opus
flac`) and any other file over 8 MiB. Each is listed with its path, size and SHA-256 in `.stcourses/media.json`
in the repo. They are kept out of `git status` by the repo's `info/exclude` (shared by every worktree), not by
the course's `.gitignore`, and the studio never counts a listed file as a change. So an author or a remote
rewriting `.gitignore` cannot block publishing or Sync. Git LFS is not used: it is not in the image and would
need an LFS server;
3. **moves the import folder to `<data>/media/<id>`** (decided over keeping it in place: a manifest left in
`<data>/courses/<id>` would be found as a second course with the same id, and the importer's "add to course" and
transcription jobs would keep changing a folder learners no longer see). `<data>/media` is never scanned for
courses and nothing writes to it afterwards. File permissions are left alone, because an import made in `link`
mode shares its files with your originals;
4. puts every referenced file into the published worktree, and later into each author's draft worktree, as a
**hard link** to `<data>/media/<id>/<path>`. The manifest paths are unchanged, so `/api/media/<id>/…` serves the
same URLs with Range support, and no extra disk space is used. If `<data>/worktrees` and `<data>/media` are on
different filesystems, each worktree gets a **full copy** instead: the published one plus one per author who
opens a draft. Keep the data directory on one filesystem.
In the studio, referenced media cannot be replaced or deleted (save, upload and conflict resolution refuse them with
409); upload a new file under another name instead. A studio write never goes through a hard link: a file with more
than one link is replaced, not overwritten. Publish and Sync refuse a commit that tracks a file at a referenced media
path, since it would silently replace the media.
**Importer jobs and conversion.** Both use the same per-course lock. Conversion is refused (409, "an import job for
this course is still running") while an importer job for that course is queued or running. An importer job that
starts after the conversion fails with "this course is now edited in the studio" and writes nothing.
**Crash safety.** The import folder is first renamed to `<data>/courses/.converting-<id>` (dot-folders are never
scanned), before the published worktree is checked out, so the course is never loaded twice. Until that checkout is
done, a failure undoes everything. After it, the conversion is committed and only the move remains:
- on one filesystem, one `rename` to `<data>/media/<id>`;
- across filesystems, a copy into `<data>/media/.staging-<id>.<n>`, `fsync`, a size and SHA-256 check against
`media.json`, a `rename` into place, and then the source is renamed to `<data>/courses/.converted-<id>.<n>` and
deleted.
At server start and before every conversion, STCourses recovers each course deterministically:
- staging folders are deleted;
- a `.converting-` folder without a finished repo is renamed back (the next attempt starts over);
- a `.converting-` folder with a finished repo has its move finished;
- `.converted-` folders are deleted once `<data>/media/<id>` is complete.
A job left running by a restart is requeued, and re-running a finished conversion reports success.
Things to know:
- Back up `<data>/media` together with `<data>/studio`: the repo alone does not contain the videos.
- **Sync** pushes only the git repo. A clone of the remote has the manifest, lessons and `.stcourses/media.json`, but
not the media files themselves; copy `<data>/media/<id>` alongside if you move the course to another server.
- If a worktree loses a media file (for example after a manual `git clean -x`), the next publish links it back into
the published worktree, and a new draft worktree gets every file. A file missing from `<data>/media/<id>` shows
as a course diagnostic ("lesson file not found"), never a crash.
- After conversion the importer can no longer add decks, code or transcripts to that course (it only edits
`<data>/courses`). Add them in the studio.
- `<data>/media` is fixed under the data directory (there is no separate setting).
- **Discovery skips dot-folders.** Every course directory (`STC_COURSE_DIRS`, `<data>/courses`, `<data>/worktrees`)
is scanned to depth 2 for `stcourse.yaml`, and folders whose name starts with `.` are never entered. Do not give a
course folder a name that starts with a dot. The conversion's `.converting-<id>` and `.converted-<id>.<n>` folders
rely on this.
- **A failed conversion.** If the media move fails after the commit (disk full, or a copy that fails its checksum),
learners are switched to the studio course right away; its media are hard links into the hidden import folder.
The course's studio page shows **Conversion incomplete** with a **Retry** button for the owner. A retry (or the
next server start) finishes the move. A failure before the commit is rolled back, and learners keep the import
unchanged.
## Memory budget
The deployment must stay under 2 GB idle. Measured with this compose file (`--profile postgres`), the image built
@@ -0,0 +1,69 @@
# STCourses v1.1 — S: Edit imported courses in the studio
> **For agentic workers:** executed inline (superpowers:executing-plans) by the S package owner, TDD, one commit per task. Steps use checkbox (`- [ ]`) syntax.
**Goal:** an owner-only action, "Make editable in the studio", turns an imported course (`<dataDir>/courses/<id>`) into a studio course (`<dataDir>/studio/repos/<id>.git` plus `<dataDir>/worktrees/<id>/published`) with the same id, so authors can edit and publish it and learners keep their progress and notes. Resolves `TODO(studio): convert imported course to a studio-managed repo` (ruling R9).
**Requirements:** `docs/superpowers/plans/2026-10-30-v1-1.md` ("Global Constraints", package S); spec sections 5, 8, 9.1, 12, 13; `CLAUDE.md` project rules.
## Decisions
1. **Large media stay out of git, referenced by path, size and checksum (spec 5).** No Git LFS (not installed in the image, and it would need a remote). A file is "large media" when its extension is video/audio (`mp4 m4v webm mov mkv avi mp3 m4a wav ogg oga opus flac`) or it is over 8 MiB. Everything else (manifest, Markdown, small images, small PDFs, code) is committed.
- The repo gets `.stcourses/media.json` (`{ schema: 1, files: [{ path, size, sha256 }] }`) and a root `.gitignore` with one anchored, escaped line per referenced path (an existing root `.gitignore` is kept above our block).
- Every studio worktree (published and each author's draft) gets the referenced files materialised at their manifest paths as **hard links** to the media store (copy with `--reflink=auto` across filesystems). Manifest paths do not change; core's media route (Range 206/416, 404 outside the root) serves them unchanged. No symlinks: the loader's `resolveInside` and the studio's write rules both refuse paths that leave the course root.
2. **The import folder is moved to `<dataDir>/media/<id>`** (a `rename`, same filesystem) rather than kept in place. Reasons: a manifest left in `<dataDir>/courses/<id>` would be discovered as a second course with the same id (and would win: imported dirs are scanned before worktrees); the importer's "add to course" and transcription jobs would keep editing a folder learners no longer see. `<dataDir>/media` is never scanned. Nothing writes there after the move; permissions are not changed, because the importer's `link` mode shares inodes with the owner's originals.
3. **Conversion runs as a job** (`studio.convert-import`), because hashing gigabytes of video can exceed Bun's 10 s idle timeout. The route checks everything it can first and answers `202 { jobId }`; the UI polls `GET /api/studio/jobs/:jobId`. The job is restart-safe: a conversion interrupted before the move is rolled back and redone; one interrupted after the repo exists only finishes the move.
4. **Same id, same manifest.** The manifest is committed unchanged: `visibility` (restricted by default for imports) and `source.kind` stay, so the v1 owner-only rule at publish still applies (only the owner can make it public or signed-in, or change `source.kind`). ACL rows, progress and notes are keyed by course and lesson ids, which do not change.
5. **Referenced media are read-only in the studio.** Saves, uploads, deletes and conflict resolutions that touch a referenced path or `.stcourses/media.json` get 409 with a message to upload under a new name. As defence in depth, any studio write to a file with more than one hard link unlinks it first, so a write can never reach a shared inode.
6. **Hostile input:** symlinks and special files in the import folder are skipped (never followed), `.git` folders are skipped, a large file whose name contains a newline (inexpressible in `.gitignore`) refuses the conversion, and the published worktree must be clean after checkout or the conversion rolls back. Materialisation checks each reference with `checkRelPath`, requires a regular file whose real path is inside `<dataDir>/media/<id>`, and writes through `safeWritePath` (refuses symlinks in the destination path).
## Global Constraints
- Everything in `CLAUDE.md` "Project rules" and the v1.1 "Global Constraints" applies. No DB change is needed (no migration).
- v1 safeguards stay: symlink-refusing writes, the remote allowlist, the owner-only visibility rule at publish, core's CSRF Origin check (applies to the new POST route automatically), repos at `<dataDir>/studio/repos/<id>.git`.
- Synthetic fixtures only. `bun test`, `bun run typecheck` and the Postgres run (`STC_TEST_PG_URL`) stay green.
## File structure
- Create `packages/feature-studio/src/server/media-refs.ts`: classification, `.gitignore` escaping, hashing, reference file read/write.
- Create `packages/feature-studio/src/server/convert.ts`: `convertImported` (stage, commit, worktree, materialise, move).
- Modify `packages/feature-studio/src/server/store.ts`: `mediaDir` option, `materializeMedia`, draft materialisation, write guards, `listFiles` includes referenced media, `convertImported` wiring.
- Modify `packages/feature-studio/src/server/feature.ts`: `POST /api/studio/courses/:courseId/convert`, `GET /api/studio/jobs/:jobId`, job handler, `convertible` flag; remove the TODO.
- Modify `packages/feature-studio/src/ui/*`: convert panel in the read-only view, imported tag on the studio home.
- Create tests: `packages/feature-studio/test/media-refs.test.ts`, `convert.test.ts`, `convert-routes.test.ts`, `convert-ui.test.tsx`; `apps/server/test/studio-import-e2e.test.ts`.
- Modify `docs/operations.md` (media decision, layout).
### Task 1: Media references
- [ ] Tests: `isLargeMedia` by extension and size; `gitignoreLine` escapes `\ * ? [ ] # !` and trailing spaces, and `git check-ignore` matches exactly the listed paths for names with spaces, brackets, `#`, `!`, non-ASCII and trailing spaces (and does not match siblings); `sha256File` matches `Bun.CryptoHasher`; `parseMediaRefs` rejects bad shapes.
- [ ] Implement `media-refs.ts`. Run tests, typecheck. Commit.
### Task 2: Conversion in the store
- [ ] Tests (`convert.test.ts`, synthetic import folder built in the test: manifest, lesson Markdown, a fake `.mp4` with space and non-ASCII name, a > 8 MiB PDF, a nested `.gitignore` that ignores a lesson file, a symlink pointing outside, a `.git` dir):
- the bare repo, published worktree and media store exist; the import folder is gone; `isManaged`;
- git tracks the manifest, lessons (including the nested-ignored one) and small files, never the video or big PDF; the symlink and `.git` contents are not tracked;
- `.stcourses/media.json` lists path, size, sha256; the published worktree is clean; the video in the worktree is a hard link (same inode) to `<dataDir>/media/<id>/…`;
- a draft worktree gets the media too; `listFiles` includes them;
- saving, uploading or deleting a referenced path or the reference file is refused (409); a write to a hard-linked file never changes the other link;
- refusals: not under the imports dir, `source.kind: authored`, manifest id mismatch, `root:` set, already a studio course, media store already present; each leaves the import folder untouched and no repo behind;
- resume: a leftover partial repo is rebuilt; a managed course whose import folder still exists only finishes the move;
- publish after an edit keeps the media in the published worktree.
- [ ] Implement `convert.ts` and the store changes. Run studio tests, typecheck. Commit.
### Task 3: Routes and job
- [ ] Tests (`convert-routes.test.ts`, real job queue from `@stc/core`):
- owner `POST …/convert` gives 202 and a job id; polling reaches `done`; the course is then studio-managed, served from the published worktree, still restricted, with ACL rows unchanged;
- an author or learner gets 403; a foreign Origin gets 403 (CSRF); an overlay or studio course gets 409; an unknown course 404; a second request while one is queued gets 409;
- `GET /api/studio/courses/:id` for an imported course says `convertible: true` for the owner only;
- after conversion a non-owner author can edit and publish a lesson title, but cannot publish `visibility: public` (403); the owner can.
- [ ] Implement. Run tests, typecheck. Commit.
### Task 4: Studio UI
- [ ] Test (`convert-ui.test.tsx`, server render): the read-only panel shows "Make editable in the studio" and the explanation only when `convertible`; progress and error states render.
- [ ] Implement `ConvertPanel` (button, job polling, then reload into the workbench) and the imported tag on the home list. Existing tokens and classes only; no new colours, fonts or motion. Commit.
### Task 5: End-to-end
- [ ] `apps/server/test/studio-import-e2e.test.ts`, SQLite and Postgres: production feature set; owner signs up; a synthetic video folder is imported through `/api/import/jobs`; a learner is granted access by ACL, marks a lesson done and writes a note; owner converts; edits a lesson title in the studio; publishes. The learner sees the new title, the same progress and notes, media answer `Range` with 206 and a bad range with 416; anonymous users still get 404.
- [ ] Commit.
### Task 6: Docs, smoke run, report
- [ ] `docs/operations.md`: the media decision and the new `<dataDir>/media` folder; backups; that media are not pushed to studio remotes.
- [ ] Dev server on port 3492 with `STC_DATA_DIR=/tmp/stc-v11-studio`: import, convert, edit, publish by curl.
- [ ] Full `bun test`, typecheck, Postgres run. Commit. Write the report.
@@ -181,6 +181,7 @@ Phase 0 (section 10) pins down the exact schema. These principles are fixed:
- The two reference courses must load by adding only a manifest and front matter, without moving or rewriting their existing files.
- Their existing tooling (test scripts, `tools/*.mjs`) must keep working.
- **Derived data lives in a cache directory that can be rebuilt from the files:** search index, slide images and text, transcripts, video posters and AI embeddings.
- **Discovery** (v1.1 note): the platform finds `stcourse.yaml` at depth 0–2 under each course directory and skips folders whose name starts with `.` (and `node_modules`). A course folder must therefore not be a dot-folder. The studio relies on this to hide an import folder while it is converted (`.converting-<id>`, `.converted-<id>.<n>`).
## 6. Accounts, roles and visibility
+1
View File
@@ -3,3 +3,4 @@ export { createJobQueue } from "./jobs";
export { serveMedia, contentType, contentDisposition, mediaSecurityHeaders, MEDIA_SANDBOX_CSP } from "./media";
export { createApp, canView, crossSiteRefusal, type Routes } from "./app";
export { composeServer, type ComposeOptions, type ComposedServer } from "./compose";
export { withCourseLock } from "./locks";
+14
View File
@@ -0,0 +1,14 @@
const locks = new Map<string, Promise<unknown>>();
/**
* Serialises work on one course across features in this process: the studio (saves, publish, conversion of an import)
* and the importer's jobs that add to an existing course. A failed holder never blocks the next one.
*/
export function withCourseLock<T>(courseId: string, fn: () => Promise<T>): Promise<T> {
const prev = locks.get(courseId) ?? Promise.resolve();
const next = prev.then(fn, fn);
const tail = next.catch(() => {});
locks.set(courseId, tail);
tail.then(() => { if (locks.get(courseId) === tail) locks.delete(courseId); });
return next;
}
+2 -2
View File
@@ -39,7 +39,7 @@ export function resolveInside(root: string, rel: string): string | null {
}
}
/** Finds stcourse.yaml at depth 0-2 under each dir. */
/** Finds stcourse.yaml at depth 0-2 under each dir. Dot-folders are skipped (hidden, or retired by the studio). */
export async function discoverManifests(courseDirs: string[]): Promise<string[]> {
const found: string[] = [];
const seen = new Set<string>();
@@ -60,7 +60,7 @@ export async function discoverManifests(courseDirs: string[]): Promise<string[]>
}
entries.sort((a, b) => a.name.localeCompare(b.name));
for (const e of entries) {
if (e.isDirectory() && !SKIP_DIRS.has(e.name)) await walk(join(dir, e.name), depth + 1);
if (e.isDirectory() && !SKIP_DIRS.has(e.name) && !e.name.startsWith(".")) await walk(join(dir, e.name), depth + 1);
}
}
for (const d of courseDirs) await walk(resolve(d), 0);
+7
View File
@@ -45,6 +45,13 @@ describe("discovery", () => {
const found = (await discoverManifests([d])).map((p) => p.slice(d.length + 1));
expect(found.sort()).toEqual(["a/b/stcourse.yaml", "a/stcourse.yaml", "stcourse.yaml"]);
});
test("skips dot-folders (hidden or retired course folders, e.g. an import being converted)", async () => {
const d = await tmp();
for (const p of ["a/stcourse.yaml", ".converting-a/stcourse.yaml", "x/.hidden/stcourse.yaml"]) await Bun.write(join(d, p), "x");
const found = (await discoverManifests([d])).map((p) => p.slice(d.length + 1));
expect(found).toEqual(["a/stcourse.yaml"]);
});
});
describe("overlays and errors", () => {
+13
View File
@@ -1,6 +1,7 @@
import { mkdir, readdir, rename, stat } from "node:fs/promises";
import { join, relative, resolve } from "node:path";
import type { ServerContext } from "@stc/contracts";
import { withCourseLock } from "@stc/core";
import { readRawManifest, writeManifest, type RawManifest } from "./util/manifest";
import { slugify, uniqueId } from "./util/text";
import { extractFrame, type Tools } from "./media";
@@ -65,9 +66,21 @@ export async function publishCourse(d: Deps, stage: string, id: string): Promise
export interface ManagedCourse { id: string; dir: string; manifest: RawManifest }
export const STUDIO_OWNED = "this course is now edited in the studio; add files there instead";
/**
* Runs importer work that changes an existing course under the course lock shared with the studio (core
* `withCourseLock`), after checking, inside the lock, that the course is still an importer folder. A conversion to a
* studio course therefore either waits for the job, or the job fails cleanly with STUDIO_OWNED and writes nothing.
*/
export function withManagedCourse<T>(d: Deps, courseId: string | undefined, fn: (c: ManagedCourse) => Promise<T>): Promise<T> {
return withCourseLock(courseId ?? "", async () => fn(await managedCourse(d, courseId)));
}
/** An existing course the importer may add to. Only folders directly under <dataDir>/courses qualify: overlays and authored repos are never edited. */
export async function managedCourse(d: Deps, courseId: string | undefined): Promise<ManagedCourse> {
if (!courseId) throw new Error("options.courseId is required: choose the course to add to");
if (isPlainId(courseId) && (await exists(studioRepo(d, courseId)))) throw new Error(STUDIO_OWNED);
const dir = join(coursesDir(d), courseId);
const rel = relative(coursesDir(d), dir);
if (!isPlainId(courseId) || rel.startsWith("..") || !(await exists(join(dir, "stcourse.yaml")))) {
@@ -1,7 +1,7 @@
import { cp, lstat, mkdir, readdir } from "node:fs/promises";
import { basename, extname, join } from "node:path";
import type { ImportResult, JobContext } from "@stc/contracts";
import { finish, managedCourse, saveManifest, stageDir, type Deps } from "../context";
import { finish, saveManifest, stageDir, withManagedCourse, type Deps, type ManagedCourse } from "../context";
import { attachDeck } from "../deck";
import { putFile } from "../media";
import { run } from "../util/exec";
@@ -48,9 +48,13 @@ async function findPdfs(dir: string, depth = 0, out: string[] = []): Promise<str
}
export async function importCodeRepo(d: Deps, payload: unknown, jc: JobContext): Promise<ImportResult> {
const { path, options } = parsePayload(payload, "code-repo");
const { options } = parsePayload(payload, "code-repo");
return withManagedCourse(d, optStr(options, "courseId"), (course) => importCodeRepoInto(d, course, payload, jc));
}
async function importCodeRepoInto(d: Deps, course: ManagedCourse, payload: unknown, jc: JobContext): Promise<ImportResult> {
const { path } = parsePayload(payload, "code-repo");
const warnings: string[] = [];
const course = await managedCourse(d, optStr(options, "courseId"));
if (!GIT_URL.test(path)) await assertAllowedPath(d, path);
const stage = await stageDir(d, "repo");
+3 -2
View File
@@ -1,7 +1,7 @@
import { stat } from "node:fs/promises";
import { join } from "node:path";
import type { ImportResult, JobContext } from "@stc/contracts";
import { finish, managedCourse, saveManifest, type Deps } from "../context";
import { finish, saveManifest, withManagedCourse, type Deps } from "../context";
import { assertAllowedPath } from "../roots";
import { attachDeck } from "../deck";
import { optStr, parsePayload } from "./common";
@@ -11,7 +11,7 @@ export async function importPdfDeck(d: Deps, payload: unknown, jc: JobContext):
const warnings: string[] = [];
await assertAllowedPath(d, path);
if (!(await stat(path).catch(() => null))?.isFile() || !/\.pdf$/i.test(path)) throw new Error(`not a PDF file: ${path}`);
const course = await managedCourse(d, optStr(options, "courseId"));
return withManagedCourse(d, optStr(options, "courseId"), async (course) => {
const attachTo = optStr(options, "attachTo");
if (!attachTo) throw new Error("options.attachTo is required: the lesson or module id the deck belongs to");
await jc.reportProgress(0.05, "Copying the deck");
@@ -25,4 +25,5 @@ export async function importPdfDeck(d: Deps, payload: unknown, jc: JobContext):
await finish(d, course.id, warnings);
await jc.reportProgress(1, r.pages ? `Deck attached (${r.pages} slides)` : "Deck attached");
return { courseId: course.id, manifestPath: join(course.dir, "stcourse.yaml"), warnings };
});
}
@@ -1,7 +1,7 @@
import { mkdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
import type { JobContext } from "@stc/contracts";
import { finish, managedCourse, saveManifest, stageDir, transcriptCacheDir, type Deps } from "../context";
import { finish, saveManifest, stageDir, transcriptCacheDir, withManagedCourse, type Deps, type ManagedCourse } from "../context";
import { run } from "../util/exec";
import { optStr, withStage } from "./common";
@@ -36,7 +36,10 @@ export async function transcribeCourse(d: Deps, payload: unknown, jc: JobContext
const url = d.transcribeUrl;
const courseId = optStr(p, "courseId") ?? "";
if (!url) return { courseId, skipped: true, reason: "STC_TRANSCRIBE_URL is not set", transcribed: 0, warnings: [] };
const course = await managedCourse(d, courseId);
return withManagedCourse(d, courseId, (course) => transcribeInto(d, course, url, p, jc));
}
async function transcribeInto(d: Deps, course: ManagedCourse, url: string, p: Record<string, unknown>, jc: JobContext): Promise<TranscribeResult> {
const only = Array.isArray(p.lessonIds) ? new Set(p.lessonIds.map(String)) : null;
const lessons = (course.manifest.modules as any[]).flatMap((m) => m.lessons as any[]).filter((l) => l.type === "video" && !l.transcript && (!only || only.has(l.id)));
const warnings: string[] = [];
+13 -8
View File
@@ -3,7 +3,7 @@ import { join, relative, resolve } from "node:path";
import type { Feature, RouteHandler } from "@stc/contracts";
import { serveMedia } from "@stc/core";
import { assertAllowedPath, importRoots, OUTSIDE } from "./roots";
import { coverCacheDir, coursesDir, exists, isPlainId, managedCourse, saveManifest, type Deps } from "./context";
import { coverCacheDir, coursesDir, exists, isPlainId, saveManifest, STUDIO_OWNED, withManagedCourse, type Deps } from "./context";
export const KINDS = ["packt-video", "code-repo", "pdf-deck", "video-folder"] as const;
const OPTION_KEYS = ["title", "courseId", "attachTo", "media", "transcribe"] as const;
@@ -90,13 +90,18 @@ export function importRoutes(deps: () => Deps): Feature["routes"] {
if (!isPlainId(rc.params.courseId!)) return json({ error: "Invalid course id" }, 400);
const src = join(coverCacheDir(d, rc.params.courseId!), name);
if (!(await exists(src))) return json({ error: "No such candidate" }, 404);
let course;
try { course = await managedCourse(d, rc.params.courseId); } catch (e) { return json({ error: (e as Error).message }, 404); }
await mkdir(join(course.dir, "media"), { recursive: true });
await copyFile(src, join(course.dir, "media", "cover.jpg"));
course.manifest.cover = "media/cover.jpg";
await saveManifest(course.dir, course.manifest);
await d.ctx.courses.reload(course.id);
try {
await withManagedCourse(d, rc.params.courseId, async (course) => {
await mkdir(join(course.dir, "media"), { recursive: true });
await copyFile(src, join(course.dir, "media", "cover.jpg"));
course.manifest.cover = "media/cover.jpg";
await saveManifest(course.dir, course.manifest);
await d.ctx.courses.reload(course.id);
});
} catch (e) {
const msg = (e as Error).message;
return json({ error: msg }, msg === STUDIO_OWNED ? 409 : 404);
}
return json({ cover: "media/cover.jpg" });
}),
},
@@ -0,0 +1,56 @@
// v1.1 S review fix: importer jobs and "Make editable in the studio" share the course lock. Either order is safe:
// a job that started first finishes and its files are converted with the course; a job that comes second fails
// cleanly and never recreates <dataDir>/courses/<id>.
import { expect, test } from "bun:test";
import { existsSync } from "node:fs";
import { join } from "node:path";
import { $ } from "bun";
import { loadAll } from "@stc/course-format";
import { createStudioStore } from "@stc/feature-studio";
import { STUDIO_OWNED } from "../src/context";
import { importCodeRepo } from "../src/jobs/code-repo";
import { importPacktVideo } from "../src/jobs/packt-video";
import { transcribeCourse } from "../src/jobs/transcribe";
import { makeEnv } from "./harness";
import { makePacktZip, makeRepoZip, tmp } from "./fixtures";
const owner = { name: "Olive Owner", email: "[email protected]" };
async function imported(env: Record<string, string> = {}) {
const e = await makeEnv(env);
const r = await importPacktVideo(e.deps, { path: await makePacktZip(await tmp()), options: { title: "Race Test" } }, e.jc);
const store = createStudioStore({ dataDir: e.dataDir });
const count = async () => (await loadAll([join(e.dataDir, "courses"), join(e.dataDir, "worktrees")])).filter((c) => c.id === r.courseId).length;
return { e, id: r.courseId, store, count };
}
test("an importer job that started first finishes; the conversion waits and commits its files", async () => {
const { e, id, store, count } = await imported();
const zip = await makeRepoZip(await tmp());
const job = importCodeRepo(e.deps, { kind: "code-repo", path: zip, options: { courseId: id } }, e.jc);
const conv = store.convertImported(id, owner);
const [res] = await Promise.all([job, conv]);
expect(res.courseId).toBe(id);
const pub = store.paths(id).published;
const files = (await $`git -c core.quotepath=false ls-files`.cwd(pub).quiet()).stdout.toString();
expect(files).toContain("code/S02 - Images (Sample)/app/main.py");
expect(await Bun.file(join(pub, "stcourse.yaml")).text()).toContain("code/S02 - Images (Sample)");
expect(existsSync(join(e.dataDir, "courses", id))).toBe(false);
expect(await count()).toBe(1);
});
test("an importer job that comes after the conversion fails cleanly and writes nothing", async () => {
const { e, id, store, count } = await imported({ STC_TRANSCRIBE_URL: "http://127.0.0.1:9/never" });
const zip = await makeRepoZip(await tmp());
const conv = store.convertImported(id, owner);
const job = importCodeRepo(e.deps, { kind: "code-repo", path: zip, options: { courseId: id } }, e.jc);
const tr = transcribeCourse(e.deps, { courseId: id }, e.jc);
const [c, j, t] = await Promise.allSettled([conv, job, tr]);
expect(c.status).toBe("fulfilled");
expect(j.status === "rejected" && (j.reason as Error).message).toBe(STUDIO_OWNED);
expect(t.status === "rejected" && (t.reason as Error).message).toBe(STUDIO_OWNED);
expect(existsSync(join(e.dataDir, "courses", id))).toBe(false);
expect(await count()).toBe(1);
// and later ones too
await expect(importCodeRepo(e.deps, { kind: "code-repo", path: zip, options: { courseId: id } }, e.jc)).rejects.toThrow(STUDIO_OWNED);
});
+1 -1
View File
@@ -1,5 +1,5 @@
// Server entry of @stc/feature-studio. The browser imports "@stc/feature-studio/web" instead (no server code).
export { studioFeature, OUTSIDE_MESSAGE, IMPORTED_MESSAGE, uploadMaxBytes, UPLOAD_EXTENSIONS, type StudioFeature, type StudioFeatureOptions } from "./server/feature";
export { studioFeature, OUTSIDE_MESSAGE, IMPORTED_MESSAGE, CONVERT_JOB, uploadMaxBytes, UPLOAD_EXTENSIONS, type StudioFeature, type StudioFeatureOptions } from "./server/feature";
export { createDraftWriter, type DraftWriter, type DraftWriterOptions } from "./server/draft-writer";
export { createStudioStore, contentHash, isAllowedRemote, ownerRequiredFor, type StudioStore, type Identity, type DraftStatus, type PublishResult, type SyncResult } from "./server/store";
export { createPublishedWatcher, type PublishedWatcher } from "./server/watch";
@@ -0,0 +1,77 @@
import { constants } from "node:fs";
import { copyFile, link, mkdir, open, readdir, stat } from "node:fs/promises";
import { dirname, join } from "node:path";
import { $ } from "bun";
import { gitignoreLine, isLargeMedia } from "./media-refs";
import { checkRelPath } from "./relpath";
export interface ScannedImport {
/** Committed into git. */
small: string[];
/** Kept out of git, referenced by path, size and checksum. */
large: { path: string; size: number }[];
/** Skipped and never followed: symlinks, sockets, `.git` folders. */
skipped: string[];
}
/** Walks an import folder with lstat semantics: symlinks and special files are skipped, never followed. */
export async function scanImport(root: string): Promise<ScannedImport> {
const out: ScannedImport = { small: [], large: [], skipped: [] };
const walk = async (rel: string) => {
const entries = await readdir(join(root, rel), { withFileTypes: true });
entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
for (const e of entries) {
const r = rel ? `${rel}/${e.name}` : e.name;
if (e.name.toLowerCase() === ".git") { out.skipped.push(r); continue; }
if (e.isDirectory()) await walk(r);
else if (e.isFile()) {
const size = (await stat(join(root, r))).size;
if (isLargeMedia(r, size)) out.large.push({ path: r, size });
else out.small.push(r);
} else out.skipped.push(r);
}
};
await walk("");
return out;
}
/** Problems that make a large file impossible to reference: a name git ignore files or the studio cannot express. */
export function unreferencable(large: { path: string }[]): string[] {
const bad: string[] = [];
for (const f of large) {
try { gitignoreLine(f.path); if (checkRelPath(f.path) !== f.path) throw new Error("not normal"); } catch { bad.push(f.path); }
}
return bad;
}
/** Test-only marker for a simulated crash: cleanup code must not run for it (a real crash would not run it either). */
export class SimulatedCrash extends Error {
constructor(public step: string) { super(`simulated crash at ${step}`); }
}
/** A hard link (no extra space), else a reflink or plain copy when linking is impossible (another filesystem, no
* permission, too many links). Other errors (the destination exists, the source vanished) are real failures. */
export async function linkOrCopy(src: string, dest: string): Promise<void> {
await mkdir(dirname(dest), { recursive: true });
try {
await link(src, dest);
return;
} catch (e) {
if (!["EXDEV", "EPERM", "EMLINK", "ENOTSUP", "EOPNOTSUPP"].includes((e as NodeJS.ErrnoException).code ?? "")) throw e;
}
const r = await $`cp --reflink=auto -- ${src} ${dest}`.quiet().nothrow();
if (r.exitCode !== 0) await copyFile(src, dest, constants.COPYFILE_EXCL);
}
/** fsyncs every file and directory under `root` (or only `root` itself with `shallow`), so a rename after it is durable. */
export async function fsyncTree(root: string, o: { shallow?: boolean } = {}): Promise<void> {
const sync = async (p: string) => { const fh = await open(p, "r"); try { await fh.sync(); } finally { await fh.close(); } };
if (!o.shallow) {
for (const e of await readdir(root, { withFileTypes: true })) {
const p = join(root, e.name);
if (e.isDirectory()) await fsyncTree(p);
else if (e.isFile()) await sync(p);
}
}
await sync(root);
}
+89 -12
View File
@@ -1,16 +1,17 @@
import { existsSync, realpathSync } from "node:fs";
import { basename, extname, resolve, sep } from "node:path";
import type { AuthUser, Feature, RequestContext, RouteHandler, ServerContext } from "@stc/contracts";
import type { AuthUser, Feature, JobHandler, RequestContext, RouteHandler, ServerContext } from "@stc/contracts";
import { canView, serveMedia } from "@stc/core";
import { ManifestSchema, parseManifest, resolveInside, type LoadedCourse } from "@stc/course-format";
import { aclRepo } from "@stc/db";
import { createDraftWriter, type DraftWriter } from "./draft-writer";
import { storeDirs, studioWorktreesDir } from "./paths";
import { checkRelPath, createStudioStore, MANIFEST, ownerRequiredFor, StudioError, type FileWrite, type StudioStore } from "./store";
import { checkRelPath, createStudioStore, MANIFEST, ownerRequiredFor, StudioError, type FileWrite, type StudioStore, type StudioStoreOptions } from "./store";
import { createPublishedWatcher, type PublishedWatcher } from "./watch";
export const OUTSIDE_MESSAGE = "This course is managed outside STCourses";
export const IMPORTED_MESSAGE = "Imported course: read-only in the studio for now";
export const IMPORTED_MESSAGE = "Imported course: not editable in the studio until the owner makes it editable";
export const CONVERT_JOB = "studio.convert-import";
/** Upload cap in bytes from AppConfig.uploadMaxMb (STC_UPLOAD_MAX_MB, default 100; review I2). */
export function uploadMaxBytes(mb: number | undefined): number {
@@ -44,9 +45,12 @@ async function readCapped(req: Request, max: number): Promise<Uint8Array<ArrayBu
}
export interface StudioFeature extends Feature {
jobs: Record<string, JobHandler>;
store: StudioStore;
watcher: PublishedWatcher;
draftWriter: DraftWriter;
/** Resolves once interrupted conversions are recovered (every studio route waits for it). */
ready: Promise<void>;
stop(): void;
}
@@ -54,6 +58,8 @@ export interface StudioFeatureOptions {
/** Poll the published worktrees for external commits (default true). */
watch?: boolean;
watchIntervalMs?: number;
/** Tests only: conversion failure injection (see StudioStoreOptions.testHooks). */
storeTestHooks?: StudioStoreOptions["testHooks"];
}
const json = (body: unknown, status = 200) => Response.json(body, { status });
@@ -72,7 +78,7 @@ function manifestToYaml(m: unknown): string {
}
export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = {}): StudioFeature {
const store = createStudioStore(storeDirs(ctx.config));
const store = createStudioStore({ ...storeDirs(ctx.config), testHooks: opts.storeTestHooks });
const watcher = createPublishedWatcher(store, ctx.courses, { intervalMs: opts.watchIntervalMs });
const acl = aclRepo(ctx.db);
const wtDir = resolve(ctx.config.studioWorktreesDir ?? studioWorktreesDir(ctx.config.dataDir));
@@ -99,24 +105,59 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = {
const readOnlyMessage = (c: LoadedCourse): string => {
const kind = c.manifest?.source?.kind;
const underImports = c.root === importedRoot || c.root.startsWith(importedRoot + sep);
if (kind && kind !== "authored" && underImports) {
// TODO(studio): convert imported course to a studio-managed repo (git init into <dataDir>/studio/repos/<id>.git
// plus a published worktree), so authors can edit it and the owner-only visibility rule applies on publish.
return IMPORTED_MESSAGE;
}
if (kind && kind !== "authored" && underImports) return IMPORTED_MESSAGE;
return OUTSIDE_MESSAGE;
};
const isOwner = (u: AuthUser) => u.roles.includes("owner");
/** v1.1 S: only the owner may turn an imported course into a studio course ("Make editable in the studio"). */
const convertible = (u: AuthUser, courseId: string) => isOwner(u) && store.isConvertibleImport(courseId);
/** Importer jobs (queued or running) that would write into this course's import folder. */
const pendingImports = async (courseId: string) =>
(await ctx.jobs.list()).filter((j) => {
if (!j.kind.startsWith("import.") || (j.status !== "queued" && j.status !== "running")) return false;
const p = (j.payload ?? {}) as { courseId?: unknown; options?: { courseId?: unknown } };
return p.courseId === courseId || p.options?.courseId === courseId;
});
/** 409 text naming the blocking importer jobs (id and kind), so the owner can find them on the import page. */
const importRunning = (jobs: { id: string; kind: string }[]) =>
`an import job for this course is still running (${jobs.map((j) => `${j.kind} ${j.id}`).join(", ")}); make it editable once it has finished`;
/** The conversion job: payload `{ courseId, actor }`, checked again by the store under the course lock. */
const convertJob: JobHandler = async (payload, jc) => {
await ready;
const p = (payload ?? {}) as { courseId?: unknown; actor?: { id?: unknown; name?: unknown; email?: unknown } };
const courseId = typeof p.courseId === "string" ? p.courseId : "";
const blocking = await pendingImports(courseId);
if (blocking.length) throw new StudioError(importRunning(blocking), 409);
const actor = { name: String(p.actor?.name ?? "STCourses"), email: String(p.actor?.email ?? "[email protected]") };
try {
const r = await store.convertImported(courseId, actor, { onProgress: (f, m) => jc.reportProgress(f, m) });
return { courseId, ...r };
} finally {
// Success or failure, learners must get whatever is on disk now: the intact import (failure before the commit,
// rolled back) or the published worktree (after it), never a course whose folder has moved away.
await ctx.courses.reload(courseId).catch((e) => console.error("[studio] reload after conversion", e));
if (store.isManaged(courseId)) await store.publishedHead(courseId).then((h) => watcher.note(courseId, h), () => {});
}
};
const afterPublishedChange = async (courseId: string) => {
await ctx.courses.reload(courseId);
watcher.note(courseId, await store.publishedHead(courseId));
};
// Crash recovery for conversions interrupted by a stop (v1.1 S); learners never see a course twice meanwhile.
const ready: Promise<void> = store.recoverConversions()
.then(async (ids) => { if (ids.length) await ctx.courses.reload(); })
.catch((e) => console.error("[studio] recovery", e));
/** Wraps a handler: signed-in author/owner (or per-course author), error mapping. */
const guard = (h: (req: Request, rc: RequestContext, user: AuthUser) => Promise<Response>): RouteHandler => async (req, rc) => {
const user = rc.user;
if (!user) return fail(401, "sign in to use the studio");
try {
await ready;
if (!hasRole(user) && (await authoredCourses(user)).length === 0) return fail(403, "the studio is for authors");
return await h(req, rc, user);
} catch (e) {
@@ -154,6 +195,8 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = {
}
return {
readOnly: false,
conversionIncomplete: store.conversionIncomplete(courseId),
canRetryConversion: isOwner(user),
course: { id: courseId, title: (manifestRaw as { title?: string } | null)?.title ?? courseId, managed: true },
status,
manifestText: m?.content ?? "",
@@ -188,7 +231,11 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = {
if (all) {
for (const c of ctx.courses.list()) {
if (isStudioCourse(c) || managed.includes(c.id)) continue;
courses.push({ id: c.id, managed: false, title: c.manifest?.title ?? c.id, pitch: c.manifest?.pitch ?? null, accent: c.manifest?.accent ?? null, publishedStatus: c.status, message: readOnlyMessage(c) });
const message = readOnlyMessage(c);
courses.push({
id: c.id, managed: false, title: c.manifest?.title ?? c.id, pitch: c.manifest?.pitch ?? null, accent: c.manifest?.accent ?? null, publishedStatus: c.status, message,
imported: message === IMPORTED_MESSAGE, convertible: convertible(user, c.id),
});
}
}
return json({ courses, canCreate: all });
@@ -212,12 +259,42 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = {
if (!store.isManaged(courseId)) {
const c = ctx.courses.get(courseId);
if (!c) return fail(404, "no such course");
return json({ readOnly: true, message: readOnlyMessage(c), course: { id: c.id, title: c.manifest?.title ?? c.id, managed: false, root: c.root }, diagnostics: c.diagnostics });
const message = readOnlyMessage(c);
return json({ readOnly: true, message, imported: message === IMPORTED_MESSAGE, convertible: convertible(user, c.id), course: { id: c.id, title: c.manifest?.title ?? c.id, managed: false, root: c.root }, diagnostics: c.diagnostics });
}
return json(await courseState(courseId, user));
}),
},
"/api/studio/courses/:courseId/convert": {
POST: guard(async (_req, rc, user) => {
const courseId = rc.params.courseId ?? "";
if (!isOwner(user)) return fail(403, "only the owner can make an imported course editable in the studio");
// An incomplete conversion (failed after its commit) is finished by a new job: it recovers first.
const incomplete = store.conversionIncomplete(courseId);
if (store.isManaged(courseId) && !incomplete) return fail(409, `course "${courseId}" is already a studio course`);
if (!incomplete && !store.isConvertibleImport(courseId)) {
const c = ctx.courses.get(courseId);
return c ? fail(409, `${readOnlyMessage(c) === IMPORTED_MESSAGE ? "this imported course" : OUTSIDE_MESSAGE}: only imported courses in ${importedRoot} can be made editable`) : fail(404, "no such imported course");
}
const pending = (await ctx.jobs.list(CONVERT_JOB)).find((j) => (j.payload as { courseId?: string } | null)?.courseId === courseId && (j.status === "queued" || j.status === "running"));
if (pending) return fail(409, "this course is already being made editable");
const blocking = await pendingImports(courseId);
if (blocking.length) return fail(409, importRunning(blocking));
const jobId = await ctx.jobs.enqueue(CONVERT_JOB, { courseId, actor: { id: user.id, name: who(user).name, email: who(user).email } });
return json({ jobId }, 202);
}),
},
"/api/studio/jobs/:jobId": {
GET: guard(async (_req, rc, user) => {
if (!isOwner(user)) return fail(403, "only the owner can follow studio jobs");
const j = await ctx.jobs.get(rc.params.jobId ?? "");
if (!j || !j.kind.startsWith("studio.")) return fail(404, "no such job");
return json({ id: j.id, kind: j.kind, status: j.status, progress: j.progress, message: j.message, error: j.error, result: j.result });
}),
},
"/api/studio/courses/:courseId/file": {
GET: course(async (_req, rc, user, courseId) => {
const path = rc.url.searchParams.get("path") ?? "";
@@ -364,5 +441,5 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = {
watcher.start();
}
return { name: "studio", routes, multipartRoutes: ["/api/studio/courses/:courseId/upload"], store, watcher, draftWriter, stop: () => watcher.stop() };
return { name: "studio", routes, jobs: { [CONVERT_JOB]: convertJob }, multipartRoutes: ["/api/studio/courses/:courseId/upload"], store, watcher, draftWriter, ready, stop: () => watcher.stop() };
}
-12
View File
@@ -55,15 +55,3 @@ export async function gitOk(cwd: string, args: string[], who?: Identity): Promis
export function splitZ(out: string): string[] {
return out.split("\0").filter(Boolean);
}
const locks = new Map<string, Promise<unknown>>();
/** Serialises git work per key (the course's bare repo path) across every store instance in the process. */
export function withLock<T>(key: string, fn: () => Promise<T>): Promise<T> {
const prev = locks.get(key) ?? Promise.resolve();
const next = prev.then(fn, fn);
const tail = next.catch(() => {});
locks.set(key, tail);
tail.then(() => { if (locks.get(key) === tail) locks.delete(key); });
return next;
}
@@ -0,0 +1,61 @@
import { extname } from "node:path";
import { checkRelPath } from "./relpath";
/**
* Large media of a converted import stay out of git (v1.1 S, docs/operations.md "Imported courses in the studio").
* The repo commits `MEDIA_REFS` (path, size and checksum per file, spec 5); the repo's `info/exclude` (not the
* author-editable `.gitignore`) gets one line per path, and the studio never counts them as changes. Every studio worktree gets the files as hard links to `<dataDir>/media/<courseId>/<path>`.
*/
export const MEDIA_REFS = ".stcourses/media.json";
export const LARGE_FILE_BYTES = 8 * 1024 * 1024;
const MEDIA_EXT = new Set(["mp4", "m4v", "webm", "mov", "mkv", "avi", "mp3", "m4a", "wav", "ogg", "oga", "opus", "flac"]);
export interface MediaRef { path: string; size: number; sha256: string }
/** Video and audio always; anything else only when it is over `LARGE_FILE_BYTES`. */
export function isLargeMedia(rel: string, size: number): boolean {
return MEDIA_EXT.has(extname(rel).slice(1).toLowerCase()) || size > LARGE_FILE_BYTES;
}
/** One anchored `.gitignore` pattern matching exactly `rel`. Throws for names a `.gitignore` line cannot hold. */
export function gitignoreLine(rel: string): string {
if (/[\n\r]/.test(rel)) throw new Error(`a file name with a line break cannot be kept out of git: ${JSON.stringify(rel)}`);
let s = rel.replace(/[\\*?[\]]/g, (c) => `\\${c}`);
const trail = /( +)$/.exec(s)?.[1]?.length ?? 0;
if (trail) s = s.slice(0, -trail) + "\\ ".repeat(trail);
return `/${s}`;
}
export const MEDIA_EXCLUDE_HEADER = "# Imported media kept outside git: see .stcourses/media.json (STCourses)";
/** Ignore-file content: `existing` (if any) first, then the header and one line per referenced file. */
export function mediaGitignore(paths: string[], existing: string | null): string {
const own = existing ? (existing.endsWith("\n") ? existing : existing + "\n") + "\n" : "";
return `${own}${MEDIA_EXCLUDE_HEADER}\n${paths.map(gitignoreLine).join("\n")}\n`;
}
export async function sha256File(abs: string): Promise<string> {
const h = new Bun.CryptoHasher("sha256");
for await (const chunk of Bun.file(abs).stream()) h.update(chunk);
return h.digest("hex");
}
export function stringifyMediaRefs(files: MediaRef[]): string {
return JSON.stringify({ schema: 1, files }, null, 2) + "\n";
}
/** The reference list, or null when the text is not a valid one (every path must pass `checkRelPath`). */
export function parseMediaRefs(text: string): MediaRef[] | null {
let raw: unknown;
try { raw = JSON.parse(text); } catch { return null; }
const r = raw as { schema?: unknown; files?: unknown } | null;
if (!r || r.schema !== 1 || !Array.isArray(r.files)) return null;
const out: MediaRef[] = [];
for (const f of r.files as Partial<MediaRef>[]) {
if (!f || typeof f.path !== "string" || typeof f.size !== "number" || !Number.isInteger(f.size) || f.size < 0) return null;
if (typeof f.sha256 !== "string" || !/^[0-9a-f]{64}$/.test(f.sha256)) return null;
try { if (checkRelPath(f.path) !== f.path) return null; } catch { return null; }
out.push({ path: f.path, size: f.size, sha256: f.sha256 });
}
return out;
}
@@ -0,0 +1,12 @@
import { isAbsolute, normalize, sep } from "node:path";
/** Normalised relative path, or throws. Rejects absolute paths, `..`, `.git` and NUL. */
export function checkRelPath(p: string): string {
if (typeof p !== "string" || p === "" || p.includes("\0") || isAbsolute(p) || p.includes("\\")) throw new Error(`invalid path: ${JSON.stringify(p)}`);
const n = normalize(p);
const parts = n.split(sep);
if (n === "." || parts[0] === ".." || parts.some((x) => x === ".." || x.toLowerCase() === ".git") || n.endsWith(sep)) {
throw new Error(`invalid path: ${JSON.stringify(p)}`);
}
return n;
}
+410 -20
View File
@@ -1,8 +1,14 @@
import { constants, existsSync, lstatSync, mkdirSync, readdirSync, realpathSync, rmSync, statSync } from "node:fs";
import { mkdir, open, rm, unlink } from "node:fs/promises";
import { dirname, isAbsolute, join, normalize, sep } from "node:path";
import { loadCourse, type Diagnostic } from "@stc/course-format";
import { git, gitOk, splitZ, withLock, type Identity } from "./git";
import { constants, existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs";
import { cp, copyFile, mkdir, open, readFile, rename as fsRename, rm, unlink, writeFile } from "node:fs/promises";
import { dirname, join, sep } from "node:path";
import { loadCourse, parseManifest, type Diagnostic } from "@stc/course-format";
import { withCourseLock } from "@stc/core";
import { fsyncTree, linkOrCopy, scanImport, SimulatedCrash, unreferencable } from "./convert";
import { git, gitOk, splitZ, type Identity } from "./git";
import { gitignoreLine, MEDIA_EXCLUDE_HEADER, MEDIA_REFS, parseMediaRefs, sha256File, stringifyMediaRefs, type MediaRef } from "./media-refs";
import { checkRelPath } from "./relpath";
export { checkRelPath } from "./relpath";
export type { Identity } from "./git";
@@ -19,17 +25,6 @@ export function safeAuthor(authorId: string): string {
return s === "" ? "_" : s;
}
/** Normalised relative path, or throws. Rejects absolute paths, `..`, `.git` and NUL. */
export function checkRelPath(p: string): string {
if (typeof p !== "string" || p === "" || p.includes("\0") || isAbsolute(p) || p.includes("\\")) throw new Error(`invalid path: ${JSON.stringify(p)}`);
const n = normalize(p);
const parts = n.split(sep);
if (n === "." || parts[0] === ".." || parts.some((x) => x === ".." || x.toLowerCase() === ".git") || n.endsWith(sep)) {
throw new Error(`invalid path: ${JSON.stringify(p)}`);
}
return n;
}
export interface FileWrite {
path: string;
content?: string | Uint8Array;
@@ -76,6 +71,20 @@ export type SyncResult =
| { ok: false; reason: "conflict"; conflicts: string[] }
| { ok: false; reason: "forbidden"; message: string };
export interface ConvertResult {
commit: string;
/** Files kept out of git and referenced from MEDIA_REFS. */
media: number;
/** Files committed. */
files: number;
/** Paths skipped in the import folder (symlinks, special files, .git folders). */
skipped: string[];
/** True when an interrupted conversion was only finished (the repo already existed). */
resumed: boolean;
}
export type ConvertProgress = (fraction: number, message: string) => void | Promise<void>;
export class StudioError extends Error {
constructor(message: string, public status = 400) { super(message); }
}
@@ -129,9 +138,20 @@ export interface StudioStoreOptions {
importedDir?: string;
/** AppConfig.studioWorktreesDir (default `<dataDir>/worktrees`). */
worktreesDir?: string;
/**
* Media stores of converted imports (default `<dataDir>/media`): `<mediaDir>/<id>` is the former import folder.
* Not in AppConfig: production always uses the default; tests may point it elsewhere.
*/
mediaDir?: string;
publishedBranch?: string;
/** Tests only: allow local-path remotes (the HTTP feature never sets this). */
allowLocalRemotes?: boolean;
/**
* Tests only, read on every use (a test may change them between calls): simulate a crash (`crashAt`) or an ordinary
* failure (`failAt`) at a named conversion step, a cross-filesystem media move (`forceExdev`), or a copy whose
* bytes do not match MEDIA_REFS (`corruptStaging`).
*/
testHooks?: { crashAt?: string; failAt?: string; forceExdev?: boolean; corruptStaging?: boolean };
}
export function createStudioStore(opts: StudioStoreOptions) {
@@ -141,6 +161,7 @@ export function createStudioStore(opts: StudioStoreOptions) {
const reposDir = opts.reposDir ?? join(dataDir, "studio", "repos");
const importedDir = opts.importedDir ?? join(dataDir, "courses");
const worktreesDir = opts.worktreesDir ?? join(dataDir, "worktrees");
const mediaDir = opts.mediaDir ?? join(dataDir, "media");
const remoteGuard = opts.allowLocalRemotes ? ["-c", "protocol.ext.allow=never", "-c", "protocol.fd.allow=never"] : REMOTE_GUARD;
const paths = (courseId: string) => {
@@ -152,6 +173,8 @@ export function createStudioStore(opts: StudioStoreOptions) {
published: join(root, "published"),
draftsDir: join(root, "drafts"),
draft: (authorId: string) => join(root, "drafts", safeAuthor(authorId)),
media: join(mediaDir, courseId),
imported: join(importedDir, courseId),
};
};
const branchOf = (authorId: string) => `draft/${safeAuthor(authorId)}`;
@@ -165,14 +188,30 @@ export function createStudioStore(opts: StudioStoreOptions) {
if (!isManaged(courseId)) throw new StudioError(`course ${courseId} is not managed by the studio`, 404);
return paths(courseId);
};
const lock = <T>(courseId: string, fn: () => Promise<T>) => withLock(paths(courseId).bare, fn);
/** The process-wide course lock shared with the importer's jobs (core `withCourseLock`). */
const lock = <T>(courseId: string, fn: () => Promise<T>) => { paths(courseId); return withCourseLock(courseId, fn); };
const convertingDir = (courseId: string) => join(importedDir, `.converting-${courseId}`);
const listDir = (dir: string): string[] => { try { return readdirSync(dir); } catch { return []; } };
const crash = async (step: string) => {
if (opts.testHooks?.crashAt === step) throw new SimulatedCrash(step);
if (opts.testHooks?.failAt === step) throw new Error(`injected failure at ${step}`);
};
const rename = async (from: string, to: string) => fsRename(from, to);
const rev = async (cwd: string, ref: string) => (await gitOk(cwd, ["rev-parse", ref])).trim();
const count = async (cwd: string, range: string) => Number((await gitOk(cwd, ["rev-list", "--count", range])).trim());
const unmerged = async (cwd: string) => [...new Set(splitZ(await gitOk(cwd, ["diff", "--name-only", "-z", "--diff-filter=U"])))];
const isMerging = async (cwd: string) => (await git(cwd, ["rev-parse", "-q", "--verify", "MERGE_HEAD"])).code === 0;
const dirtyFiles = async (cwd: string) =>
splitZ(await gitOk(cwd, ["status", "--porcelain", "-z", "--untracked-files=all"])).map((l) => l.slice(3)).filter(Boolean);
/**
* Uncommitted changes, except untracked referenced media: whatever a `.gitignore` (editable by authors and remotes)
* says, the files listed in MEDIA_REFS never make a worktree dirty.
*/
const dirtyFiles = async (cwd: string) => {
const media = new Set((await mediaRefsIn(cwd)).map((r) => r.path));
return splitZ(await gitOk(cwd, ["status", "--porcelain", "-z", "--untracked-files=all"]))
.filter((l) => !(l.startsWith("?? ") && media.has(l.slice(3))))
.map((l) => l.slice(3)).filter(Boolean);
};
const remoteUrl = async (bare: string) => {
const r = await git(bare, ["remote", "get-url", "origin"]);
return r.code === 0 ? r.stdout.trim() : null;
@@ -241,9 +280,185 @@ export function createStudioStore(opts: StudioStoreOptions) {
const branch = branchOf(authorId);
const exists = (await git(p.bare, ["rev-parse", "-q", "--verify", `refs/heads/${branch}`])).code === 0;
await gitOk(p.bare, exists ? ["worktree", "add", dir, branch] : ["worktree", "add", "-b", branch, dir, main]);
await materializeMedia(dir, p.media, p.bare);
return dir;
}
/** The draft's or worktree's media references (empty for courses that never were imports). */
async function mediaRefsIn(dir: string): Promise<MediaRef[]> {
const f = await readFileIn(dir, MEDIA_REFS);
return f ? parseMediaRefs(f.content) ?? [] : [];
}
/**
* Puts every referenced file into `dir` as a hard link to `fromRoot/<path>` (a copy across filesystems).
* Idempotent; existing destinations are left alone. A reference must name a regular file whose real path is inside
* `fromRoot`, and the destination path may not contain a symlink. Returns the references it could not satisfy.
*/
async function materializeMedia(dir: string, fromRoot: string, bare: string): Promise<{ linked: number; missing: string[] }> {
const refs = await mediaRefsIn(dir);
if (refs.length) await writeExclude(bare, refs.map((r) => r.path));
const missing: string[] = [];
let linked = 0;
let realFrom: string;
try { realFrom = realpathSync(fromRoot); } catch { return { linked, missing: refs.map((r) => r.path) }; }
for (const r of refs) {
try {
const src = join(realFrom, r.path);
const st = lstatSync(src);
const real = realpathSync(src);
if (!st.isFile() || !real.startsWith(realFrom + sep)) throw new Error("not a regular file inside the media store");
const dest = await safeWritePath(dir, r.path);
if (lexists(dest)) continue;
await linkOrCopy(real, dest);
linked++;
} catch {
missing.push(r.path);
}
}
if (missing.length) console.warn(`[studio] media missing for ${dir}: ${missing.join(", ")}`);
return { linked, missing };
}
/**
* Keeps referenced media out of `git status` through the repo's `info/exclude` (shared by every worktree, not
* reachable by authors or remotes, unlike `.gitignore`). Lines are only ever added.
*/
async function writeExclude(bare: string, paths: string[]): Promise<void> {
const file = join(bare, "info", "exclude");
const cur = await readFile(file, "utf8").catch(() => "");
const lines = new Set(cur.split("\n"));
const add: string[] = [];
for (const x of paths) {
let line: string;
try { line = gitignoreLine(x); } catch { continue; }
if (!lines.has(line)) { add.push(line); lines.add(line); }
}
if (!add.length) return;
const head = cur.includes(MEDIA_EXCLUDE_HEADER) ? "" : `${MEDIA_EXCLUDE_HEADER}\n`;
await mkdir(dirname(file), { recursive: true });
await writeFile(file, `${cur}${cur && !cur.endsWith("\n") ? "\n" : ""}${head}${add.join("\n")}\n`);
}
/** Every reference present under `root` with its size (and, when asked, its sha256). */
async function mediaComplete(root: string, refs: MediaRef[], hash: boolean): Promise<boolean> {
for (const r of refs) {
try {
const st = lstatSync(join(root, r.path));
if (!st.isFile() || st.size !== r.size) return false;
if (hash && (await sha256File(join(root, r.path))) !== r.sha256) return false;
} catch {
return false;
}
}
return true;
}
/** Renames the old import folder aside (atomic, never discovered: dot-folder) and deletes it. */
async function retireSource(courseId: string, from: string): Promise<void> {
const gone = join(importedDir, `.converted-${courseId}.${Date.now()}`);
await rename(from, gone);
await crash("source-retired");
await rm(gone, { recursive: true, force: true });
}
/**
* Makes `from` (the hidden import folder) the media store `<mediaDir>/<id>`. Same filesystem: one rename.
* Across filesystems: copy into a staging dir, fsync, verify against MEDIA_REFS, rename into place, then retire the
* source. An existing store that matches MEDIA_REFS means an earlier attempt got that far: only the source is retired.
*/
async function finishMove(courseId: string, from: string): Promise<void> {
const p = paths(courseId);
const refs = await mediaRefsIn(p.published);
if (lexists(p.media)) {
if (!(await mediaComplete(p.media, refs, true))) throw new StudioError(`the media store ${p.media} exists but does not match ${MEDIA_REFS}; move it away and retry`, 409);
await retireSource(courseId, from);
return;
}
await crash("before-move");
await mkdir(mediaDir, { recursive: true });
try {
if (opts.testHooks?.forceExdev) throw Object.assign(new Error("EXDEV (test)"), { code: "EXDEV" });
await rename(from, p.media);
return;
} catch (e) {
if ((e as NodeJS.ErrnoException).code !== "EXDEV") throw e;
}
const staging = join(mediaDir, `.staging-${courseId}.${Bun.randomUUIDv7()}`);
try {
await cp(from, staging, { recursive: true, verbatimSymlinks: true, errorOnExist: true, force: false });
if (opts.testHooks?.corruptStaging && refs[0]) await writeFile(join(staging, refs[0].path), "corrupted copy");
await crash("staging-copied");
await fsyncTree(staging);
if (!(await mediaComplete(staging, refs, true))) throw new Error(`the copied media in ${staging} do not match ${MEDIA_REFS}`);
await rename(staging, p.media);
await fsyncTree(mediaDir, { shallow: true });
} catch (e) {
if (!(e instanceof SimulatedCrash)) await rm(staging, { recursive: true, force: true });
throw e;
}
await crash("staging-placed");
await retireSource(courseId, from);
}
/**
* Deterministic recovery of one course's conversion, under its lock:
* - staging copies are deleted (a move is always redone from the source);
* - a hidden import folder without a finished repo is restored and the partial repo removed (the next attempt
* starts over); with a finished repo, the published worktree is reset to its commit and the move finished;
* - retired sources are deleted once the media store matches MEDIA_REFS.
*/
async function recoverUnlocked(courseId: string): Promise<void> {
const p = paths(courseId);
for (const n of listDir(mediaDir)) if (n.startsWith(`.staging-${courseId}.`)) await rm(join(mediaDir, n), { recursive: true, force: true });
const conv = convertingDir(courseId);
if (lexists(conv)) {
if (!isManaged(courseId)) {
rmSync(p.bare, { recursive: true, force: true });
rmSync(p.root, { recursive: true, force: true });
if (lexists(p.imported)) throw new StudioError(`both ${conv} and ${p.imported} exist; remove one of them by hand`, 409);
await rename(conv, p.imported);
} else {
await gitOk(p.published, ["reset", "-q", "--hard", "HEAD"]);
await finishMove(courseId, conv);
await materializeMedia(p.published, p.media, p.bare);
}
}
const retired = listDir(importedDir).filter((n) => n.startsWith(`.converted-${courseId}.`));
if (retired.length) {
const refs = isManaged(courseId) ? await mediaRefsIn(p.published) : null;
if (refs && lexists(p.media) && (await mediaComplete(p.media, refs, false))) {
for (const n of retired) await rm(join(importedDir, n), { recursive: true, force: true });
} else {
console.warn(`[studio] ${retired.join(", ")} kept: the media store of ${courseId} is not complete`);
}
}
}
/**
* Referenced media paths that `treeish` (a draft branch, the remote's branch) tracks in git. Merging those would
* silently replace the media-store file with the commit's content, so publish and Sync refuse them.
*/
async function mediaClash(bare: string, treeish: string): Promise<string[]> {
const refs = new Set<string>();
for (const t of [main, treeish]) {
const text = await showText(bare, `${t}:${MEDIA_REFS}`);
for (const r of (text && parseMediaRefs(text)) || []) refs.add(r.path);
}
if (!refs.size) return [];
const r = await git(bare, ["--literal-pathspecs", "ls-tree", "-r", "-z", "--name-only", treeish, "--", ...refs]);
return r.code === 0 ? splitZ(r.stdout) : [];
}
/** Referenced media and the reference file are not editable in the studio (they are not in git). */
async function refuseMediaWrites(dir: string, paths: string[]): Promise<void> {
const locked = new Set([MEDIA_REFS, ...(await mediaRefsIn(dir)).map((r) => r.path)]);
const hit = paths.map((x) => checkRelPath(x)).filter((x) => locked.has(x));
if (hit.length) {
throw new StudioError(`${hit.join(", ")}: imported media are kept outside git and cannot be changed in the studio; upload a file under a new name instead`, 409);
}
}
async function readFileIn(dir: string, rel: string): Promise<{ content: string; hash: string } | null> {
let abs: string;
try { abs = inside(dir, rel); } catch { return null; }
@@ -268,6 +483,8 @@ export function createStudioStore(opts: StudioStoreOptions) {
if (lexists(abs)) await unlink(abs);
} else {
const abs = await safeWritePath(dir, f.path);
// never write into an inode shared with another path (imported media, an owner's original): replace the file
if (lexists(abs) && lstatSync(abs).nlink > 1) await unlink(abs);
// O_NOFOLLOW: a symlink swapped in after the check makes the open fail instead of writing through it
const fh = await open(abs, constants.O_WRONLY | constants.O_CREAT | constants.O_TRUNC | constants.O_NOFOLLOW, 0o644);
try { await fh.writeFile(f.content ?? ""); } finally { await fh.close(); }
@@ -323,16 +540,182 @@ export function createStudioStore(opts: StudioStoreOptions) {
});
},
/** An importer folder `<importedDir>/<id>` holding an imported (not authored) course the owner may convert. */
isConvertibleImport(courseId: string): boolean {
if (!ID_RE.test(courseId) || isManaged(courseId)) return false;
const p = paths(courseId);
try {
if (!lstatSync(p.imported).isDirectory()) return false;
const m = parseLoose(readFileSync(join(p.imported, MANIFEST), "utf8"));
return !!m?.source?.kind && m.source.kind !== "authored";
} catch {
return false;
}
},
/**
* A conversion that got past its commit but did not finish (the media move failed or was interrupted): the course
* is served from the published worktree, and `convertImported` (which recovers first) completes it.
*/
conversionIncomplete(courseId: string): boolean {
if (!ID_RE.test(courseId)) return false;
if (lexists(convertingDir(courseId))) return true;
return listDir(mediaDir).some((n) => n.startsWith(`.staging-${courseId}.`)) || listDir(importedDir).some((n) => n.startsWith(`.converted-${courseId}.`));
},
/** Re-links referenced media into the published worktree (after an outside checkout removed them, say). */
repairMedia: (courseId: string) => lock(courseId, async () => {
const p = requireManaged(courseId);
return materializeMedia(p.published, p.media, p.bare);
}),
/**
* Finishes or undoes every interrupted conversion (server start; `convertImported` also does it for its course).
* Returns the ids it touched. Never throws: a course it cannot repair is logged and left for the owner.
*/
async recoverConversions(): Promise<string[]> {
const ids = new Set<string>();
const scan = (dir: string, re: RegExp) => { for (const n of listDir(dir)) { const m = re.exec(n); if (m && ID_RE.test(m[1]!)) ids.add(m[1]!); } };
scan(mediaDir, /^\.staging-([^.]+)\./);
scan(importedDir, /^\.converting-([^.]+)$/);
scan(importedDir, /^\.converted-([^.]+)\./);
const done: string[] = [];
for (const id of [...ids].sort()) {
try { await lock(id, () => recoverUnlocked(id)); done.push(id); } catch (e) { console.error(`[studio] could not recover the conversion of ${id}:`, e); }
}
return done;
},
/**
* Turns the imported course `<importedDir>/<id>` into a studio course with the same id (v1.1 S, ruling R9):
* small files are committed as they are (manifest unchanged: visibility and source.kind stay), large media are
* referenced from MEDIA_REFS and hard-linked into each worktree, and the import folder ends up as `<mediaDir>/<id>`.
* Owner-only at the HTTP layer; holds the shared course lock, so importer jobs on the course wait or fail cleanly.
*
* Crash safety: the import folder is hidden (`<importedDir>/.converting-<id>`, never discovered) before the
* published worktree appears, so the id is never loaded twice. Until the worktree is checked out, a failure undoes
* everything; after that the conversion is committed and `recoverUnlocked` (run first by every attempt and at
* server start) finishes the move. Across filesystems the media are copied into `<mediaDir>/.staging-<id>.<n>`,
* fsynced, verified against MEDIA_REFS (size and sha256) and only then renamed into place.
*/
convertImported(courseId: string, who: Identity, opts: { onProgress?: ConvertProgress } = {}): Promise<ConvertResult> {
if (!ID_RE.test(courseId)) return Promise.reject(new StudioError(`invalid course id: ${courseId}`, 404));
const progress = opts.onProgress ?? (() => {});
return lock(courseId, async () => {
const p = paths(courseId);
await recoverUnlocked(courseId);
if (isManaged(courseId)) {
const refsFile = await readFileIn(p.published, MEDIA_REFS);
const media = refsFile ? (parseMediaRefs(refsFile.content) ?? []).length : 0;
const head = async () => rev(p.published, "HEAD");
// already converted (a job re-run after a restart, say): report success
if (refsFile && lexists(p.media) && !lexists(p.imported)) return { commit: await head(), media, files: 0, skipped: [], resumed: true };
// a conversion made before the import folder was hidden first: finish its move
if (refsFile && lexists(p.imported)) {
await finishMove(courseId, p.imported);
await materializeMedia(p.published, p.media, p.bare);
return { commit: await head(), media, files: 0, skipped: [], resumed: true };
}
throw new StudioError(`course "${courseId}" is already a studio course`, 409);
}
const srcManifest = join(p.imported, MANIFEST);
if (!lexists(srcManifest)) throw new StudioError(`there is no imported course "${courseId}" in ${importedDir}`, 404);
const top = lstatSync(p.imported);
if (top.isSymbolicLink() || !top.isDirectory() || lstatSync(srcManifest).isSymbolicLink()) throw new StudioError("the import folder and its manifest must not be symlinks", 400);
if (lexists(p.media)) throw new StudioError(`the media store ${p.media} already exists; move it away and retry`, 409);
const parsed = parseManifest(await Bun.file(srcManifest).text());
if (!parsed.ok) throw new StudioError(`the imported manifest is invalid: ${parsed.errors.join("; ")}`, 400);
const m = parsed.manifest;
if (m.id !== courseId) throw new StudioError(`the imported manifest has id "${m.id}", not "${courseId}"`, 400);
if (m.root) throw new StudioError("an imported course with root: cannot become a studio course", 400);
if (!m.source?.kind || m.source.kind === "authored") throw new StudioError("only imported courses can be made editable this way", 400);
await progress(0.02, "Reading the import folder");
const scan = await scanImport(p.imported);
const bad = unreferencable(scan.large);
if (bad.length) throw new StudioError(`these large files have names the studio cannot keep outside git; rename them first: ${bad.map((b) => JSON.stringify(b)).join(", ")}`, 400);
// A previous attempt that never got as far as the published worktree: start again.
rmSync(p.bare, { recursive: true, force: true });
rmSync(p.root, { recursive: true, force: true });
const total = scan.large.reduce((n, f) => n + f.size, 0) || 1;
let done = 0;
const refs: MediaRef[] = [];
for (const f of scan.large) {
await progress(0.05 + 0.75 * (done / total), `Checksumming ${f.path}`);
refs.push({ path: f.path, size: f.size, sha256: await sha256File(join(p.imported, f.path)) });
done += f.size;
}
const conv = convertingDir(courseId);
const work = join(dataDir, "tmp", `convert-${courseId}-${Bun.randomUUIDv7()}`);
const stage = join(work, "tree");
const list = [...new Set([...scan.small, MEDIA_REFS])];
let hidden = false;
try {
await progress(0.82, "Committing the course files");
for (const f of scan.small) {
await mkdir(dirname(join(stage, f)), { recursive: true });
await copyFile(join(p.imported, f), join(stage, f));
}
await mkdir(join(stage, dirname(MEDIA_REFS)), { recursive: true });
await writeFile(join(stage, MEDIA_REFS), stringifyMediaRefs(refs));
await writeFile(join(work, "paths"), list.join("\0") + "\0");
mkdirSync(reposDir, { recursive: true });
await gitOk(dataDir, ["init", "-q", "--bare", "-b", main, p.bare]);
await writeExclude(p.bare, refs.map((r) => r.path));
const gd = ["--literal-pathspecs", "--git-dir", p.bare, "--work-tree", stage];
// -f: a nested .gitignore from the course's own code must not drop course files
await gitOk(stage, [...gd, "add", "-f", `--pathspec-from-file=${join(work, "paths")}`, "--pathspec-file-nul"]);
await gitOk(stage, [...gd, "commit", "-q", "-m", `Make ${courseId} editable in the studio\n\nConverted from the import in ${p.imported}. ${refs.length} large media files stay outside git (${MEDIA_REFS}).`], who);
await crash("committed");
// Hide the import folder before the published worktree exists: the id is never discoverable twice.
await rename(p.imported, conv);
hidden = true;
await crash("hidden");
mkdirSync(p.root, { recursive: true });
await gitOk(p.bare, ["worktree", "add", p.published, main]);
await progress(0.9, "Linking media into the published course");
const mat = await materializeMedia(p.published, conv, p.bare);
if (mat.missing.length) throw new Error(`could not link media: ${mat.missing.join(", ")}`);
const dirty = await dirtyFiles(p.published);
if (dirty.length) throw new Error(`the converted course is not clean after checkout: ${dirty.join(", ")}`);
await crash("worktree");
} catch (e) {
if (!(e instanceof SimulatedCrash)) {
rmSync(p.bare, { recursive: true, force: true });
rmSync(p.root, { recursive: true, force: true });
if (hidden && !lexists(p.imported)) await rename(conv, p.imported);
}
throw e;
} finally {
await rm(work, { recursive: true, force: true });
}
// Committed: from here on a failure leaves a state that the next attempt (or a restart) finishes.
await progress(0.95, "Moving the import folder to the media store");
await finishMove(courseId, conv);
await materializeMedia(p.published, p.media, p.bare);
return { commit: await rev(p.published, "HEAD"), media: refs.length, files: list.length, skipped: scan.skipped, resumed: false };
});
},
ensureDraft: (courseId: string, authorId: string) => lock(courseId, () => ensureDraftUnlocked(courseId, authorId)),
async listFiles(courseId: string, authorId: string): Promise<string[]> {
const dir = await api.ensureDraft(courseId, authorId);
const tracked = splitZ(await gitOk(dir, ["ls-files", "-z", "--cached", "--others", "--exclude-standard"]));
return [...new Set(tracked)].filter((f) => existsSync(join(dir, f))).sort();
const media = (await mediaRefsIn(dir)).map((r) => r.path);
return [...new Set([...tracked, ...media])].filter((f) => existsSync(join(dir, f))).sort();
},
async readFile(courseId: string, authorId: string, rel: string) {
const dir = await api.ensureDraft(courseId, authorId);
// referenced media can be gigabytes and are not text: never read them whole (the raw route streams them)
if ((await mediaRefsIn(dir)).some((r) => r.path === rel)) return null;
return readFileIn(dir, rel);
},
@@ -348,6 +731,7 @@ export function createStudioStore(opts: StudioStoreOptions) {
return lock(courseId, async () => {
const dir = await ensureDraftUnlocked(courseId, who.id);
if (await isMerging(dir)) throw new StudioError("the draft has a merge in progress: resolve or abort it first", 409);
await refuseMediaWrites(dir, files.map((f) => f.path));
const conflicts: FileConflict[] = [];
for (const f of files) {
if (f.baseHash === undefined) continue;
@@ -419,6 +803,8 @@ export function createStudioStore(opts: StudioStoreOptions) {
const diags = (await api.validateDraftUnlocked(courseId, dir)).filter((d) => d.level === "error");
if (diags.length) return { ok: false, reason: "invalid", diagnostics: diags };
if ((await count(p.bare, `${main}..${branch}`)) === 0) return { ok: true, commit: await rev(p.published, "HEAD"), published: false };
const clash = await mediaClash(p.bare, branch);
if (clash.length) return { ok: false, reason: "invalid", diagnostics: clash.map((path) => ({ level: "error" as const, courseId, path, message: `${path} is imported media kept outside git; the draft must not commit a file there` })) };
const baseManifest = await showText(p.published, `HEAD:${MANIFEST}`);
// Merge without committing, so the merged result is checked before it becomes the published version.
const r = await git(p.published, ["merge", "--no-ff", "--no-commit", branch], who);
@@ -440,6 +826,7 @@ export function createStudioStore(opts: StudioStoreOptions) {
return { ok: false, reason: "invalid", diagnostics: mergedErrors };
}
await gitOk(p.published, ["commit", "-q", "-m", `Publish ${branch}`], who);
await materializeMedia(p.published, p.media, p.bare);
// Bring the draft level with what was just published, so "behind" stays meaningful.
await git(dir, ["merge", "-q", "--ff-only", main]);
return { ok: true, commit: await rev(p.published, "HEAD"), published: true };
@@ -501,6 +888,7 @@ export function createStudioStore(opts: StudioStoreOptions) {
return lock(courseId, async () => {
const dir = await ensureDraftUnlocked(courseId, who.id);
if (!(await isMerging(dir))) throw new StudioError("no merge in progress", 409);
await refuseMediaWrites(dir, files.map((f) => f.path));
const m = files.find((f) => checkRelPath(f.path) === MANIFEST);
if (m && !who.owner) {
const needOwner = ownerRequiredFor(await showText(requireManaged(courseId).bare, `${main}:${MANIFEST}`), m.content);
@@ -562,6 +950,8 @@ export function createStudioStore(opts: StudioStoreOptions) {
const local = await rev(p.bare, main);
const remote = await rev(p.bare, remoteRef);
if (local !== remote) {
const clash = await mediaClash(p.bare, remoteRef);
if (clash.length) return { ok: false, reason: "forbidden", message: `the remote commits files over imported media kept outside git: ${clash.join(", ")}` };
const localBehind = (await git(p.bare, ["merge-base", "--is-ancestor", local, remote])).code === 0;
const remoteBehind = (await git(p.bare, ["merge-base", "--is-ancestor", remote, local])).code === 0;
const baseManifest = await showText(p.bare, `${main}:${MANIFEST}`);
@@ -0,0 +1,102 @@
import { useEffect, useRef, useState } from "react";
import { Pill, ProgressBar } from "@stc/ui";
import { api, coursePath } from "./api";
export type ConvertPhase =
| { kind: "idle" }
| { kind: "running"; progress: number; message: string | null }
| { kind: "failed"; error: string };
export interface ReadOnlyInfo { title: string; message: string; root: string; imported?: boolean; convertible?: boolean }
/** The owner's "Make editable in the studio" panel for an imported course (v1.1 S). Presentational. */
export function ConvertPanelView({ phase, onConvert }: { phase: ConvertPhase; onConvert: () => void }) {
const running = phase.kind === "running";
return (
<section className="studio-section" aria-labelledby="convert-head">
<h2 id="convert-head">Make editable in the studio</h2>
<p className="studio-lede" style={{ margin: "0 0 12px" }}>
The course becomes a studio course with the same id, so learners keep their progress and notes. The manifest,
lessons and small files go into a git repository you edit on a draft and publish. Videos and other large files
stay outside git, on this server, and keep playing at the same paths. Visibility does not change.
</p>
{running ? (
<div className="studio-convert-progress" aria-live="polite">
<ProgressBar value={phase.progress} label="Conversion progress" />
<span className="faint">{phase.message ?? "Waiting to start…"}</span>
</div>
) : null}
{phase.kind === "failed" ? <p className="studio-errors" role="alert">{phase.error}</p> : null}
<div className="studio-actions">
<Pill variant="primary" disabled={running} onClick={onConvert}>{phase.kind === "failed" ? "Try again" : "Make editable in the studio"}</Pill>
</div>
</section>
);
}
/** Starts the conversion job (`POST …/convert`) and polls it; calls `onDone` once it has finished. */
function useConvertJob(courseId: string, onDone: () => void) {
const [phase, setPhase] = useState<ConvertPhase>({ kind: "idle" });
const live = useRef(true);
useEffect(() => () => { live.current = false; }, []);
const start = async () => {
setPhase({ kind: "running", progress: 0, message: null });
const { status, data } = await api("POST", coursePath(courseId, "/convert"));
if (status !== 202) return setPhase({ kind: "failed", error: data?.error ?? `Could not start (${status})` });
for (;;) {
await new Promise((r) => setTimeout(r, 700));
if (!live.current) return;
const j = await api("GET", `/api/studio/jobs/${encodeURIComponent(data.jobId)}`);
if (j.status !== 200) return setPhase({ kind: "failed", error: j.data?.error ?? `Could not follow the conversion (${j.status})` });
if (j.data.status === "done") return onDone();
if (j.data.status === "failed") return setPhase({ kind: "failed", error: j.data.error ?? "The conversion failed" });
setPhase({ kind: "running", progress: j.data.progress ?? 0, message: j.data.message });
}
};
return { phase, start };
}
export function ConvertPanel({ courseId, onDone }: { courseId: string; onDone: () => void }) {
const { phase, start } = useConvertJob(courseId, onDone);
return <ConvertPanelView phase={phase} onConvert={start} />;
}
/** A conversion that failed after its commit: the course is served, but its media move did not finish. */
export function IncompleteConversionView({ phase, canRetry, onRetry }: { phase: ConvertPhase; canRetry: boolean; onRetry: () => void }) {
return (
<div className="studio-note studio-note--warn" role="status">
<strong>Conversion incomplete.</strong> Learners see this course, but moving its imported media into place did not
finish. {canRetry ? "Retry to complete it." : "Ask the owner to retry it."}
{phase.kind === "running" ? (
<div className="studio-convert-progress" aria-live="polite" style={{ marginTop: 8 }}>
<ProgressBar value={phase.progress} label="Conversion progress" />
<span className="faint">{phase.message ?? "Waiting to start…"}</span>
</div>
) : null}
{phase.kind === "failed" ? <p className="studio-errors" role="alert">{phase.error}</p> : null}
{canRetry ? <div className="studio-actions" style={{ marginTop: 8 }}><Pill disabled={phase.kind === "running"} onClick={onRetry}>Retry</Pill></div> : null}
</div>
);
}
export function IncompleteConversion({ courseId, canRetry, onDone }: { courseId: string; canRetry: boolean; onDone: () => void }) {
const { phase, start } = useConvertJob(courseId, onDone);
return <IncompleteConversionView phase={phase} canRetry={canRetry} onRetry={start} />;
}
/** What the studio shows for a course it does not manage. */
export function ReadOnlyView({ courseId, info, onConverted }: { courseId: string; info: ReadOnlyInfo; onConverted: () => void }) {
if (info.imported) {
return (
<>
<div className="studio-note studio-note--warn" role="note">
<strong>Imported course.</strong>{" "}
{info.convertible ? "It is read-only until you make it editable in the studio." : "It is read-only here. Only the owner can make it editable in the studio."}
</div>
{info.convertible ? <ConvertPanel courseId={courseId} onDone={onConverted} /> : null}
</>
);
}
return <div className="studio-note studio-note--warn" role="note"><strong>{info.message}.</strong> It is read-only here; edit it in its own repository at <code>{info.root}</code>.</div>;
}
@@ -3,6 +3,7 @@ import { Link, useParams, useSearchParams } from "react-router-dom";
import { CourseCover, Pill } from "@stc/ui";
import { api, coursePath, uniqueId, type Diagnostic, type ManifestLesson } from "./api";
import { AccessMessage } from "./StudioHome";
import { IncompleteConversion, ReadOnlyView } from "./ConvertPanel";
import { clone, useWorkspace, type Workspace } from "./workspace";
import { CourseSettings } from "./editors/CourseSettings";
import { LessonEditor } from "./editors/LessonEditor";
@@ -15,7 +16,7 @@ type Notice = { kind: "ok" | "warn" | "bad"; text: string; files?: string[]; dia
export default function CourseStudio() {
const { courseId = "" } = useParams();
const { ws, readOnly, error } = useWorkspace(courseId);
const { ws, readOnly, error, reload } = useWorkspace(courseId);
if (error) return <div className="studio studio-page"><p className="chap"><Link to="/studio">Studio</Link></p>{error.status === 404 ? <div className="studio-note studio-note--bad"><strong>No such course.</strong></div> : <AccessMessage {...error} />}</div>;
if (readOnly) {
@@ -23,7 +24,7 @@ export default function CourseStudio() {
<div className="studio studio-page">
<p className="chap"><Link to="/studio">Studio</Link></p>
<h1>{readOnly.title}</h1>
<div className="studio-note studio-note--warn" role="note"><strong>{readOnly.message}.</strong> It is read-only here; edit it in its own repository at <code>{readOnly.root}</code>.</div>
<ReadOnlyView courseId={courseId} info={readOnly} onConverted={reload} />
</div>
);
}
@@ -106,6 +107,8 @@ function Workbench({ ws }: { ws: Workspace }) {
<Pill variant="primary" disabled={busy || st.merging || !st.ahead || hasErrors} title={hasErrors ? "Fix the errors in the draft first" : undefined} onClick={publish}>Publish</Pill>
</div>
{ws.state.conversionIncomplete ? <IncompleteConversion courseId={ws.courseId} canRetry={!!ws.state.canRetryConversion} onDone={ws.reload} /> : null}
{notice ? (
<div className={`studio-note studio-note--${notice.kind}`} role="status">
{notice.text}
@@ -6,7 +6,7 @@ import "./studio.css";
interface CourseRow {
id: string; title: string; pitch: string | null; accent: string | null; managed: boolean; publishedStatus: "ok" | "error";
ahead?: number; behind?: number; merging?: boolean; remote?: string | null; message?: string;
ahead?: number; behind?: number; merging?: boolean; remote?: string | null; message?: string; imported?: boolean; convertible?: boolean;
}
export function AccessMessage({ status, error }: { status: number; error?: string }) {
@@ -99,7 +99,7 @@ export default function StudioHome() {
{outside.length ? (
<section className="studio-section" aria-labelledby="outside-courses">
<h2 id="outside-courses">Managed outside STCourses</h2>
<p className="faint" style={{ margin: 0 }}>These courses come from repositories the platform does not own. Edit them in their own repository.</p>
<p className="faint" style={{ margin: 0 }}>These courses come from repositories the platform does not own, or from imports. Edit them in their own repository; the owner can make an imported course editable here.</p>
<ul className="studio-rows">
{outside.map((c) => (
<li className="studio-row" key={c.id}>
@@ -108,7 +108,7 @@ export default function StudioHome() {
<Link to={`/studio/${encodeURIComponent(c.id)}`}>{c.title}</Link>
<span className="s"><code>{c.id}</code> · {c.message}</span>
</div>
<span className="studio-tag">Read-only</span>
{c.convertible ? <Link className="studio-tag studio-tag--accent" to={`/studio/${encodeURIComponent(c.id)}`}>Make editable</Link> : <span className="studio-tag">{c.imported ? "Imported" : "Read-only"}</span>}
</li>
))}
</ul>
+4 -2
View File
@@ -42,10 +42,12 @@ export interface DraftStatus {
export interface Diagnostic { level: "error" | "warning"; lessonId?: string; path?: string; message: string }
export interface CourseState {
readOnly: false; course: { id: string; title: string }; status: DraftStatus; manifestText: string; manifestHash: string | null;
readOnly: false; course: { id: string; title: string }; status: DraftStatus;
/** A conversion from an import that did not finish its media move (v1.1 S); the owner may retry it. */
conversionIncomplete?: boolean; canRetryConversion?: boolean; manifestText: string; manifestHash: string | null;
manifestRaw: ManifestDoc | null; manifestErrors: string[]; diagnostics: Diagnostic[]; files: string[];
}
export interface ReadOnlyState { readOnly: true; message: string; course: { id: string; title: string; root: string }; diagnostics: Diagnostic[] }
export interface ReadOnlyState { readOnly: true; message: string; imported?: boolean; convertible?: boolean; course: { id: string; title: string; root: string }; diagnostics: Diagnostic[] }
export interface FileWrite { path: string; content?: string; baseHash?: string | null; delete?: boolean }
export interface FileConflict { path: string; current: string | null; currentHash: string | null }
@@ -96,3 +96,8 @@
.studio-content { padding: 0 16px 48px; width: 100%; }
.studio-bar { margin: 0 -16px 20px; padding: 10px 16px; }
}
/* "Make editable in the studio" (imported courses) */
.studio-convert-progress { display: flex; align-items: center; gap: 12px; margin: 0 0 12px; max-width: 760px; }
.studio-convert-progress .stc-bar { flex: 1; }
a.studio-tag { text-decoration: none; }
+4 -2
View File
@@ -1,4 +1,5 @@
import { useCallback, useEffect, useState } from "react";
import type { ReadOnlyInfo } from "./ConvertPanel";
import { api, coursePath, rawUrl, type CourseState, type FileConflict, type FileWrite, type ManifestDoc } from "./api";
export type SaveOutcome = { ok: true } | { ok: false; conflicts?: FileConflict[]; error?: string };
@@ -51,7 +52,7 @@ export type OpenFile = ReturnType<typeof useFile>;
export function useWorkspace(courseId: string) {
const [state, setState] = useState<CourseState | null>(null);
const [readOnly, setReadOnly] = useState<{ message: string; title: string; root: string } | null>(null);
const [readOnly, setReadOnly] = useState<ReadOnlyInfo | null>(null);
const [error, setError] = useState<{ status: number; error?: string } | null>(null);
const [doc, setDocState] = useState<ManifestDoc | null>(null);
const [docDirty, setDocDirty] = useState(false);
@@ -61,7 +62,8 @@ export function useWorkspace(courseId: string) {
const { status, data } = await api("GET", coursePath(courseId));
if (status !== 200) { setError({ status, error: data?.error }); return; }
setError(null);
if (data.readOnly) { setReadOnly({ message: data.message, title: data.course.title, root: data.course.root }); return; }
if (data.readOnly) { setReadOnly({ message: data.message, title: data.course.title, root: data.course.root, imported: !!data.imported, convertible: !!data.convertible }); return; }
setReadOnly(null);
setState(data);
setDocState(data.manifestRaw ? clone(data.manifestRaw) : null);
setDocDirty(false);
+20 -7
View File
@@ -1,9 +1,9 @@
import { resolve } from "node:path";
import type { AppConfig, AuthUser, ServerContext } from "@stc/contracts";
import { createApp, resolveConfig } from "@stc/core";
import type { AppConfig, AuthUser, JobQueue, ServerContext } from "@stc/contracts";
import { createApp, createJobQueue, resolveConfig } from "@stc/core";
import { createCourseRepository } from "@stc/course-format";
import { connect, migrate } from "@stc/db";
import { studioFeature } from "../src/server/feature";
import { studioFeature, type StudioFeatureOptions } from "../src/server/feature";
import { studioCourseDirs } from "../src/server/paths";
import { tempDir } from "./helpers";
@@ -15,7 +15,7 @@ export const users: Record<string, AuthUser> = {
};
/** A server context like main.ts builds, with header-selected users (`x-user: ada`). */
export async function setupApp(extraCourseDirs: string[] = [], dbUrl = "sqlite://:memory:", prepare?: (dataDir: string) => void, configOverrides: Partial<AppConfig> = {}) {
export async function setupApp(extraCourseDirs: string[] = [], dbUrl = "sqlite://:memory:", prepare?: (dataDir: string) => void, configOverrides: Partial<AppConfig> = {}, featureOpts: StudioFeatureOptions = {}) {
const dataDir = tempDir();
prepare?.(dataDir);
let dropSchema = async () => {};
@@ -34,10 +34,14 @@ export async function setupApp(extraCourseDirs: string[] = [], dbUrl = "sqlite:/
const config = resolveConfig({ port: 0, databaseUrl: "sqlite://:memory:", dataDir, courseDirs: extraCourseDirs, secret: "t", publicUrl: "http://x", ...configOverrides });
const courses = await createCourseRepository(studioCourseDirs(config));
const ctx = {
config, db, courses, jobs: null as never,
config, db, courses, jobs: null as unknown as JobQueue,
auth: { userFromRequest: async (req: Request) => users[req.headers.get("x-user") ?? ""] ?? null },
} satisfies ServerContext;
const feature = studioFeature(ctx, { watch: false });
const feature = studioFeature(ctx, { watch: false, ...featureOpts });
// the real queue, as composeServer builds it from the features' jobs
const queue = createJobQueue(db, feature.jobs ?? {}, { pollMs: 20 });
ctx.jobs = queue;
queue.start();
const app = createApp(ctx, [feature]);
const call = async (as: string | null, method: string, path: string, body?: unknown) => {
const headers: Record<string, string> = { origin: "http://x" };
@@ -51,5 +55,14 @@ export async function setupApp(extraCourseDirs: string[] = [], dbUrl = "sqlite:/
try { json = JSON.parse(text); } catch { json = text; }
return { status: r.status, json };
};
return { ctx, db, dataDir, feature, app, call, stop: async () => { feature.stop(); await db.close(); await dropSchema(); } };
/** Polls a job until it is done or failed. */
const waitJob = async (id: string) => {
for (let i = 0; i < 500; i++) {
const j = await queue.get(id);
if (j && (j.status === "done" || j.status === "failed")) return j;
await Bun.sleep(20);
}
throw new Error(`job ${id} did not finish`);
};
return { ctx, db, dataDir, feature, app, call, waitJob, stop: async () => { feature.stop(); await queue.stop(); await db.close(); await dropSchema(); } };
}
@@ -0,0 +1,231 @@
import { afterEach, describe, expect, test } from "bun:test";
import { existsSync, readdirSync, readFileSync } from "node:fs";
import { join, resolve } from "node:path";
import { studioFeature } from "../src/server/feature";
import { SimulatedCrash } from "../src/server/convert";
import { createStudioStore } from "../src/server/store";
import { REPO, setupApp } from "./app-helpers";
import { makeImportedCourse, VIDEO, videoBytes } from "./import-fixture";
let env: Awaited<ReturnType<typeof setupApp>>;
afterEach(() => env?.stop());
const PG = process.env.STC_TEST_PG_URL;
const dbs: [string, string][] = [["sqlite", "sqlite://:memory:"], ...(PG ? [["postgres", PG] as [string, string]] : [])];
async function convert(as = "owner", id = "imp") {
const r = await env.call(as, "POST", `/api/studio/courses/${id}/convert`);
expect(r.status).toBe(202);
const job = await env.waitJob(r.json.jobId);
expect(job.error).toBeNull();
expect(job.status).toBe("done");
return job;
}
describe.each(dbs)("convert routes [%s]", (_name, dbUrl) => {
test("the owner converts; the course is then served from the published worktree, still restricted, ACL kept", async () => {
env = await setupApp([], dbUrl, (d) => makeImportedCourse(d));
const { call, ctx, db } = env;
await db.sql`INSERT INTO course_acl (course_id, principal_type, principal_id) VALUES ('imp', 'user', 'u-lea')`;
expect(ctx.courses.get("imp")?.root).toBe(join(env.dataDir, "courses/imp"));
const ro = (await call("owner", "GET", "/api/studio/courses/imp")).json;
expect(ro).toMatchObject({ readOnly: true, convertible: true });
expect((await call("ada", "GET", "/api/studio/courses/imp")).json).toMatchObject({ readOnly: true, convertible: false });
expect((await call("owner", "GET", "/api/studio/courses")).json.courses).toContainEqual(expect.objectContaining({ id: "imp", managed: false, imported: true, convertible: true }));
const job = await convert();
expect(job.result).toMatchObject({ courseId: "imp", media: 3, resumed: false });
const c = ctx.courses.get("imp")!;
expect(c.root).toBe(join(env.dataDir, "worktrees/imp/published"));
expect(c.status).toBe("ok");
expect(c.manifest?.visibility).toBe("restricted");
expect(ctx.courses.list().filter((x) => x.id === "imp")).toHaveLength(1);
expect(await db.sql`SELECT principal_id FROM course_acl WHERE course_id = 'imp'`).toEqual([{ principal_id: "u-lea" }]);
// media through core's route, Range included
const r = await env.app.fetch(new Request(`http://x/api/media/imp/${VIDEO.split("/").map(encodeURIComponent).join("/")}`, { headers: { "x-user": "lea", range: "bytes=10-19" } }));
expect(r.status).toBe(206);
expect(new Uint8Array(await r.arrayBuffer())).toEqual(videoBytes(4096, 7).slice(10, 20));
// anonymous users still see nothing
expect((await env.app.fetch(new Request(`http://x/api/media/imp/${encodeURI(VIDEO)}`))).status).toBe(404);
// the studio now edits it
const st = (await call("owner", "GET", "/api/studio/courses/imp")).json;
expect(st.readOnly).toBe(false);
expect(st.files).toContain(VIDEO);
expect((await call("owner", "GET", "/api/studio/courses")).json.courses).toContainEqual(expect.objectContaining({ id: "imp", managed: true }));
});
test("an author edits and publishes a lesson title but cannot make the course public; the owner can", async () => {
env = await setupApp([], dbUrl, (d) => makeImportedCourse(d));
const { call, ctx } = env;
await convert();
let st = (await call("ada", "GET", "/api/studio/courses/imp")).json;
const m = structuredClone(st.manifestRaw);
m.modules[0].lessons[0].title = "Welcome aboard";
expect((await call("ada", "PUT", "/api/studio/courses/imp/files", { message: "Rename", manifest: m, manifestBaseHash: st.manifestHash })).status).toBe(200);
expect((await call("ada", "POST", "/api/studio/courses/imp/publish")).json).toMatchObject({ ok: true, published: true });
expect(ctx.courses.get("imp")?.lessonIndex.get("l1")?.lesson.title).toBe("Welcome aboard");
st = (await call("ada", "GET", "/api/studio/courses/imp")).json;
const pub = structuredClone(st.manifestRaw);
pub.visibility = "public";
expect((await call("ada", "PUT", "/api/studio/courses/imp/files", { message: "Go public", manifest: pub, manifestBaseHash: st.manifestHash })).status).toBe(403);
st = (await call("owner", "GET", "/api/studio/courses/imp")).json;
expect((await call("owner", "PUT", "/api/studio/courses/imp/files", { message: "Go public", manifest: { ...st.manifestRaw, visibility: "public" }, manifestBaseHash: st.manifestHash })).status).toBe(200);
expect((await call("owner", "POST", "/api/studio/courses/imp/publish")).status).toBe(200);
expect(ctx.courses.get("imp")?.manifest?.visibility).toBe("public");
});
});
describe.each(dbs)("conversion and importer jobs [%s]", (_name, dbUrl) => {
test("409 while an import job for the course is queued; the job itself re-checks", async () => {
env = await setupApp([], dbUrl, (d) => makeImportedCourse(d));
// no importer feature in this app: its jobs stay queued
await env.ctx.jobs.enqueue("import.code-repo", { kind: "code-repo", path: "/x", options: { courseId: "imp" } });
const blocker = await env.ctx.jobs.list("import.code-repo");
const r = await env.call("owner", "POST", "/api/studio/courses/imp/convert");
expect(r.status).toBe(409);
expect(r.json.error).toContain("an import job for this course is still running");
expect(r.json.error).toContain(`import.code-repo ${blocker[0]!.id}`);
// a convert job that was queued anyway (before the import was) fails cleanly and changes nothing
const id = await env.ctx.jobs.enqueue("studio.convert-import", { courseId: "imp", actor: { id: "u-own", name: "O", email: "o@x" } });
const j = await env.waitJob(id);
expect(j.status).toBe("failed");
expect(j.error).toContain("an import job for this course is still running");
expect(existsSync(join(env.dataDir, "courses/imp/stcourse.yaml"))).toBe(true);
expect(existsSync(join(env.dataDir, "studio/repos/imp.git"))).toBe(false);
// an import job for another course does not block
await env.db.sql`DELETE FROM jobs WHERE kind = 'import.code-repo'`;
await env.ctx.jobs.enqueue("import.transcribe", { courseId: "other" });
expect((await env.call("owner", "POST", "/api/studio/courses/imp/convert")).status).toBe(202);
});
test("server start recovers a conversion that crashed mid-move; learners see the course once", async () => {
env = await setupApp([], dbUrl, (d) => makeImportedCourse(d));
const crashing = createStudioStore({ dataDir: env.dataDir, testHooks: { crashAt: "staging-copied", forceExdev: true } });
await expect(crashing.convertImported("imp", { name: "O", email: "o@x" })).rejects.toBeInstanceOf(SimulatedCrash);
await env.ctx.courses.reload();
expect(env.ctx.courses.list().filter((c) => c.id === "imp").length).toBeLessThanOrEqual(1);
// a restart: a new studio feature recovers before serving
const restarted = studioFeature(env.ctx, { watch: false });
await restarted.ready;
restarted.stop();
const loaded = env.ctx.courses.list().filter((c) => c.id === "imp");
expect(loaded).toHaveLength(1);
expect(loaded[0]!.root).toBe(join(env.dataDir, "worktrees/imp/published"));
expect(existsSync(join(env.dataDir, "courses/imp"))).toBe(false);
const r = await env.app.fetch(new Request(`http://x/api/media/imp/${VIDEO.split("/").map(encodeURIComponent).join("/")}`, { headers: { "x-user": "owner", range: "bytes=0-9" } }));
expect(r.status).toBe(206);
// the requeued job (or the owner) retrying reports success
expect((await env.call("owner", "POST", "/api/studio/courses/imp/convert")).status).toBe(409);
expect(await env.feature.store.convertImported("imp", { name: "O", email: "o@x" })).toMatchObject({ resumed: true });
});
});
describe.each(dbs)("a conversion that fails without a restart [%s]", (_name, dbUrl) => {
const media = (as: string) => env.app.fetch(new Request(`http://x/api/media/imp/${VIDEO.split("/").map(encodeURIComponent).join("/")}`, { headers: { "x-user": as, range: "bytes=10-19" } }));
const learnerView = async (where: "import" | "studio") => {
const all = env.ctx.courses.list().filter((c) => c.id === "imp");
expect(all).toHaveLength(1);
expect(all[0]!.status).toBe("ok");
expect(all[0]!.root).toBe(join(env.dataDir, where === "import" ? "courses/imp" : "worktrees/imp/published"));
const r = await media("lea");
expect(r.status).toBe(206);
expect(new Uint8Array(await r.arrayBuffer())).toEqual(videoBytes(4096, 7).slice(10, 20));
};
const cases: [string, { forceExdev?: boolean; failAt?: string; corruptStaging?: boolean }, "import" | "studio", RegExp][] = [
["the cross-filesystem copy fails (disk full)", { forceExdev: true, failAt: "staging-copied" }, "studio", /injected failure at staging-copied/],
["the copied media fail their checksum", { forceExdev: true, corruptStaging: true }, "studio", /do not match/],
["the checkout fails before the commit point", { failAt: "worktree" }, "import", /injected failure at worktree/],
];
test.each(cases)("%s: learners keep a working course and the owner's retry completes", async (_c, hooks, after, error) => {
env = await setupApp([], dbUrl, (d) => makeImportedCourse(d), {}, { storeTestHooks: hooks });
await env.db.sql`INSERT INTO course_acl (course_id, principal_type, principal_id) VALUES ('imp', 'user', 'u-lea')`;
await learnerView("import");
const r = await env.call("owner", "POST", "/api/studio/courses/imp/convert");
expect(r.status).toBe(202);
const j = await env.waitJob(r.json.jobId);
expect(j.status).toBe("failed");
expect(j.error).toMatch(error);
// no restart: learners already get the intact import or the finished studio course
await learnerView(after);
if (after === "studio") {
const st = (await env.call("owner", "GET", "/api/studio/courses/imp")).json;
expect(st).toMatchObject({ readOnly: false, conversionIncomplete: true, canRetryConversion: true });
expect((await env.call("ada", "GET", "/api/studio/courses/imp")).json).toMatchObject({ conversionIncomplete: true, canRetryConversion: false });
} else {
expect((await env.call("owner", "GET", "/api/studio/courses/imp")).json).toMatchObject({ readOnly: true, convertible: true });
}
// the cause goes away (space freed, disk fixed); the owner retries through the same route
delete hooks.failAt;
delete hooks.corruptStaging;
expect((await env.call("ada", "POST", "/api/studio/courses/imp/convert")).status).toBe(403);
const again = await env.call("owner", "POST", "/api/studio/courses/imp/convert");
expect(again.status).toBe(202);
const j2 = await env.waitJob(again.json.jobId);
expect(j2.error).toBeNull();
expect(j2.status).toBe("done");
await learnerView("studio");
expect((await env.call("owner", "GET", "/api/studio/courses/imp")).json.conversionIncomplete).toBe(false);
expect(readdirSync(join(env.dataDir, "courses"))).toEqual([]);
expect(readdirSync(join(env.dataDir, "media"))).toEqual(["imp"]);
expect((await env.call("owner", "POST", "/api/studio/courses/imp/convert")).status).toBe(409);
});
});
describe("convert route refusals", () => {
test("only the owner; cross-site requests are refused; nothing changes", async () => {
env = await setupApp([], undefined, (d) => makeImportedCourse(d));
for (const who of ["ada", "lea"]) expect((await env.call(who, "POST", "/api/studio/courses/imp/convert")).status).toBe(403);
expect((await env.call(null, "POST", "/api/studio/courses/imp/convert")).status).toBe(401);
const forged = await env.app.fetch(new Request("http://x/api/studio/courses/imp/convert", { method: "POST", headers: { "x-user": "owner", origin: "http://evil.example" } }));
expect(forged.status).toBe(403);
const noOrigin = await env.app.fetch(new Request("http://x/api/studio/courses/imp/convert", { method: "POST", headers: { "x-user": "owner" } }));
expect(noOrigin.status).toBe(403);
expect(existsSync(join(env.dataDir, "courses/imp/stcourse.yaml"))).toBe(true);
expect(existsSync(join(env.dataDir, "studio/repos/imp.git"))).toBe(false);
expect(await env.ctx.jobs.list("studio.convert-import")).toEqual([]);
});
test("overlay, studio and unknown courses; a second request while one is pending", async () => {
env = await setupApp([resolve(REPO, "fixtures/courses")], undefined, (d) => makeImportedCourse(d));
expect((await env.call("owner", "POST", "/api/studio/courses/sample/convert")).status).toBe(409);
expect((await env.call("owner", "POST", "/api/studio/courses/nope/convert")).status).toBe(404);
await env.call("owner", "POST", "/api/studio/courses", { id: "c1", title: "C" });
expect((await env.call("owner", "POST", "/api/studio/courses/c1/convert")).status).toBe(409);
await env.ctx.jobs.enqueue("studio.convert-import", { courseId: "imp", actor: { id: "u-own", name: "O", email: "o@x" } });
// whether the first job is still queued, running or already done, a second request is refused
const r = await env.call("owner", "POST", "/api/studio/courses/imp/convert");
expect(r.status).toBe(409);
});
test("job status is owner-only and limited to studio jobs", async () => {
env = await setupApp([], undefined, (d) => makeImportedCourse(d));
const { jobId } = (await env.call("owner", "POST", "/api/studio/courses/imp/convert")).json;
expect((await env.call("ada", "GET", `/api/studio/jobs/${jobId}`)).status).toBe(403);
await env.waitJob(jobId);
const j = (await env.call("owner", "GET", `/api/studio/jobs/${jobId}`)).json;
expect(j).toMatchObject({ id: jobId, status: "done", progress: 1, error: null });
const other = await env.ctx.jobs.enqueue("import.transcribe", { courseId: "x" });
expect((await env.call("owner", "GET", `/api/studio/jobs/${other}`)).status).toBe(404);
});
test("a failed conversion reports the error and leaves the import as it was", async () => {
env = await setupApp([], undefined, (d) => makeImportedCourse(d));
const manifest = join(env.dataDir, "courses/imp/stcourse.yaml");
// the route only checks that this is an imported course; the job validates the manifest fully
await Bun.write(manifest, readFileSync(manifest, "utf8").replace("id: imp", "id: other"));
const { jobId } = (await env.call("owner", "POST", "/api/studio/courses/imp/convert")).json;
const j = await env.waitJob(jobId);
expect(j.status).toBe("failed");
expect(j.error).toContain('"other"');
expect(existsSync(join(env.dataDir, "studio/repos/imp.git"))).toBe(false);
});
});
@@ -0,0 +1,51 @@
import { expect, test } from "bun:test";
import { renderToString } from "react-dom/server";
import { ConvertPanelView, IncompleteConversionView, ReadOnlyView } from "../src/ui/ConvertPanel";
const noop = () => {};
test("the owner sees the action and what it does", () => {
const html = renderToString(<ConvertPanelView phase={{ kind: "idle" }} onConvert={noop} />);
expect(html).toContain("Make editable in the studio");
expect(html).toContain("same id");
expect(html).toContain("progress and notes");
expect(html).toContain("outside git");
expect(html).toContain("<button");
});
test("progress and failure states", () => {
const running = renderToString(<ConvertPanelView phase={{ kind: "running", progress: 0.4, message: "Checksumming media/a.mp4" }} onConvert={noop} />);
expect(running).toContain('role="progressbar"');
expect(running).toContain('aria-valuenow="40"');
expect(running).toContain("Checksumming media/a.mp4");
expect(running).toContain("disabled");
const failed = renderToString(<ConvertPanelView phase={{ kind: "failed", error: "the imported manifest is invalid" }} onConvert={noop} />);
expect(failed).toContain('role="alert"');
expect(failed).toContain("the imported manifest is invalid");
expect(failed).toContain("Try again");
});
test("read-only view: imported for the owner offers the action; for others explains who can", () => {
const owner = renderToString(<ReadOnlyView courseId="imp" info={{ title: "Imp", message: "Imported course", root: "/d/courses/imp", imported: true, convertible: true }} onConverted={noop} />);
expect(owner).toContain("Make editable in the studio");
expect(owner).not.toContain("/d/courses/imp");
const author = renderToString(<ReadOnlyView courseId="imp" info={{ title: "Imp", message: "Imported course", root: "/d/courses/imp", imported: true, convertible: false }} onConverted={noop} />);
expect(author).not.toContain("<button");
expect(author).toContain("Only the owner");
const outside = renderToString(<ReadOnlyView courseId="o" info={{ title: "O", message: "This course is managed outside STCourses", root: "/repos/o", imported: false, convertible: false }} onConverted={noop} />);
expect(outside).toContain("/repos/o");
expect(outside).not.toContain("<button");
});
test("an incomplete conversion: the owner can retry, others are told who can", () => {
const owner = renderToString(<IncompleteConversionView phase={{ kind: "idle" }} canRetry onRetry={noop} />);
expect(owner).toContain("Conversion incomplete");
expect(owner).toContain(">Retry</button>");
const author = renderToString(<IncompleteConversionView phase={{ kind: "idle" }} canRetry={false} onRetry={noop} />);
expect(author).toContain("Conversion incomplete");
expect(author).not.toContain("<button");
expect(author).toContain("Ask the owner");
const failed = renderToString(<IncompleteConversionView phase={{ kind: "failed", error: "disk full" }} canRetry onRetry={noop} />);
expect(failed).toContain('role="alert"');
expect(failed).toContain("disk full");
});
@@ -0,0 +1,299 @@
import { describe, expect, test } from "bun:test";
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { loadAll, loadCourse } from "@stc/course-format";
import { createStudioStore } from "../src/server/store";
import { SimulatedCrash } from "../src/server/convert";
import { gitignoreLine, MEDIA_REFS, parseMediaRefs } from "../src/server/media-refs";
import { BIG_PDF, makeImportedCourse, VIDEO, VIDEO2, videoBytes } from "./import-fixture";
import { ada, extGit, newStore, tempDir } from "./helpers";
const ino = (p: string) => statSync(p).ino;
const tracked = async (dir: string) => (await extGit(dir, "-c", "core.quotepath=false", "ls-files")).split("\n").filter(Boolean).sort();
describe("convertImported", () => {
test("builds the repo and published worktree; large media stay out of git as hard links", async () => {
const { dataDir, store } = await newStore();
const src = makeImportedCourse(dataDir, { hostile: true });
const srcVideoIno = ino(join(src, VIDEO));
const r = await store.convertImported("imp", ada);
expect(r.media).toBe(3);
expect(store.isManaged("imp")).toBe(true);
expect(existsSync(join(dataDir, "studio/repos/imp.git/HEAD"))).toBe(true);
expect(existsSync(src)).toBe(false);
const mediaRoot = join(dataDir, "media/imp");
expect(existsSync(join(mediaRoot, "stcourse.yaml"))).toBe(true);
const pub = store.paths("imp").published;
expect(await tracked(pub)).toEqual([
MEDIA_REFS, "code/S01 - Basics/.gitignore", "code/S01 - Basics/README.md", "media/cover.jpg", "stcourse.yaml", "transcripts/l1.md",
]);
// the symlink was skipped, never followed; .git contents never copied
expect(existsSync(join(pub, "code/leak.txt"))).toBe(false);
expect(existsSync(join(pub, "code/.git"))).toBe(false);
expect(await extGit(pub, "status", "--porcelain", "--untracked-files=all")).toBe("");
// the exclusions live in the repo's info/exclude, which authors and remotes cannot change
const exclude = readFileSync(join(dataDir, "studio/repos/imp.git/info/exclude"), "utf8");
for (const p of [VIDEO, VIDEO2, BIG_PDF]) expect(exclude).toContain(gitignoreLine(p));
const refs = parseMediaRefs(readFileSync(join(pub, MEDIA_REFS), "utf8"))!;
expect(refs.map((x) => x.path).sort()).toEqual([BIG_PDF, VIDEO2, VIDEO].sort());
const v = refs.find((x) => x.path === VIDEO)!;
expect(v.size).toBe(4096);
expect(v.sha256).toBe(new Bun.CryptoHasher("sha256").update(videoBytes(4096, 7)).digest("hex"));
for (const p of [VIDEO, VIDEO2, BIG_PDF]) expect(ino(join(pub, p))).toBe(ino(join(mediaRoot, p)));
expect(ino(join(mediaRoot, VIDEO))).toBe(srcVideoIno);
// the course loads from the published worktree exactly as before
const c = await loadCourse(join(pub, "stcourse.yaml"));
expect(c.status).toBe("ok");
expect(c.manifest?.visibility).toBe("restricted");
expect(c.manifest?.source?.kind).toBe("video-folder");
// the commit is attributed to the converting owner
expect(await extGit(pub, "log", "-1", "--format=%an <%ae>")).toBe(`${ada.name} <${ada.email}>`);
});
test("a draft gets the media; listFiles shows them; the media and reference file are read-only", async () => {
const { dataDir, store } = await newStore();
makeImportedCourse(dataDir);
await store.convertImported("imp", ada);
const draft = await store.ensureDraft("imp", ada.id);
expect(ino(join(draft, VIDEO))).toBe(ino(join(dataDir, "media/imp", VIDEO)));
const files = await store.listFiles("imp", ada.id);
expect(files).toContain(VIDEO);
expect(files).toContain(BIG_PDF);
for (const path of [VIDEO, MEDIA_REFS]) {
await expect(store.saveFiles("imp", ada, [{ path, content: "x" }], "x")).rejects.toMatchObject({ status: 409 });
await expect(store.saveFiles("imp", ada, [{ path, delete: true }], "x")).rejects.toMatchObject({ status: 409 });
}
expect(readFileSync(join(dataDir, "media/imp", VIDEO))).toEqual(Buffer.from(videoBytes(4096, 7)));
});
test("a studio write to a hard-linked file breaks the link instead of writing through it", async () => {
const { dataDir, store } = await newStore();
makeImportedCourse(dataDir);
await store.convertImported("imp", ada);
const draft = await store.ensureDraft("imp", ada.id);
// a planted hard link at an ordinary (non-referenced) path
const outside = join(dataDir, "media/imp/transcripts/l1.md");
rmSync(join(draft, "transcripts/l1.md"));
const { linkSync } = await import("node:fs");
linkSync(outside, join(draft, "transcripts/l1.md"));
expect((await store.saveFiles("imp", ada, [{ path: "transcripts/l1.md", content: "# changed\n" }], "edit")).ok).toBe(true);
expect(readFileSync(outside, "utf8")).toBe("# Intro\n\n[0:00] Hello.\n");
expect(readFileSync(join(draft, "transcripts/l1.md"), "utf8")).toBe("# changed\n");
});
test("edit and publish: the published worktree keeps its media and stays clean", async () => {
const { dataDir, store } = await newStore();
makeImportedCourse(dataDir);
await store.convertImported("imp", ada);
const m = (await store.readFile("imp", ada.id, "stcourse.yaml"))!;
const next = m.content.replace("title: Intro", "title: Welcome aboard");
expect((await store.saveFiles("imp", ada, [{ path: "stcourse.yaml", content: next, baseHash: m.hash }], "Rename lesson")).ok).toBe(true);
const r = await store.publish("imp", ada);
expect(r).toMatchObject({ ok: true, published: true });
const pub = store.paths("imp").published;
expect(readFileSync(join(pub, "stcourse.yaml"), "utf8")).toContain("title: Welcome aboard");
expect(ino(join(pub, VIDEO))).toBe(ino(join(dataDir, "media/imp", VIDEO)));
expect(await extGit(pub, "status", "--porcelain", "--untracked-files=all")).toBe("");
});
test("a non-owner cannot publish the converted course as public; the owner can", async () => {
const { dataDir, store } = await newStore();
makeImportedCourse(dataDir);
await store.convertImported("imp", ada);
const m = (await store.readFile("imp", ada.id, "stcourse.yaml"))!;
await store.saveFiles("imp", ada, [{ path: "stcourse.yaml", content: m.content.replace("visibility: restricted", "visibility: public"), baseHash: m.hash }], "go public");
expect(await store.publish("imp", ada)).toMatchObject({ ok: false, reason: "forbidden" });
expect(await store.publish("imp", { ...ada, owner: true })).toMatchObject({ ok: true, published: true });
});
describe("refusals leave the import untouched and no repo behind", () => {
const cases: [string, (dataDir: string) => void, number][] = [
["no such import", () => {}, 404],
["an authored course", (d) => makeImportedCourse(d, { kind: "authored" }), 400],
["a manifest id mismatch", (d) => makeImportedCourse(d, { manifestId: "other" }), 400],
["a root: field", (d) => makeImportedCourse(d, { root: "." }), 400],
["a media store already there", (d) => { makeImportedCourse(d); mkdirSync(join(d, "media/imp"), { recursive: true }); }, 409],
];
test.each(cases)("%s", async (_name, prepare, status) => {
const { dataDir, store } = await newStore();
prepare(dataDir);
const before = existsSync(join(dataDir, "courses/imp/stcourse.yaml")) ? readFileSync(join(dataDir, "courses/imp/stcourse.yaml"), "utf8") : null;
await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status });
expect(existsSync(join(dataDir, "studio/repos/imp.git"))).toBe(false);
expect(existsSync(join(dataDir, "worktrees/imp"))).toBe(false);
if (before !== null) expect(readFileSync(join(dataDir, "courses/imp/stcourse.yaml"), "utf8")).toBe(before);
});
test("a studio course with the same id", async () => {
const { store } = await newStore();
await store.createCourse({ id: "imp", title: "Studio" }, ada);
await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status: 409 });
expect(store.isManaged("imp")).toBe(true);
});
test("a large file whose name has a line break", async () => {
const { dataDir, store } = await newStore();
const src = makeImportedCourse(dataDir);
writeFileSync(join(src, "media/bad\nname.mp4"), "x");
await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status: 400 });
expect(existsSync(join(dataDir, "studio/repos/imp.git"))).toBe(false);
expect(existsSync(join(src, "stcourse.yaml"))).toBe(true);
});
});
describe("restart safety", () => {
test("a leftover partial repo (no published worktree) is rebuilt", async () => {
const { dataDir, store } = await newStore();
makeImportedCourse(dataDir);
mkdirSync(join(dataDir, "studio/repos/imp.git"), { recursive: true });
writeFileSync(join(dataDir, "studio/repos/imp.git/junk"), "x");
await store.convertImported("imp", ada);
expect(store.isManaged("imp")).toBe(true);
expect(existsSync(join(dataDir, "studio/repos/imp.git/junk"))).toBe(false);
});
test("a managed course whose import folder is still there only finishes the move", async () => {
const { dataDir, store } = await newStore();
makeImportedCourse(dataDir);
await store.convertImported("imp", ada);
const head = await store.publishedHead("imp");
// simulate a crash between the checkout and the move
const { renameSync } = await import("node:fs");
renameSync(join(dataDir, "media/imp"), join(dataDir, "courses/imp"));
const r = await store.convertImported("imp", ada);
expect(r.resumed).toBe(true);
expect(await store.publishedHead("imp")).toBe(head);
expect(existsSync(join(dataDir, "courses/imp"))).toBe(false);
expect(existsSync(join(dataDir, "media/imp", VIDEO))).toBe(true);
});
});
});
describe("an author's or a remote's .gitignore never blocks the course", () => {
const editManifest = async (store: Awaited<ReturnType<typeof newStore>>["store"], title: string) => {
const m = (await store.readFile("imp", ada.id, "stcourse.yaml"))!;
const text = m.content.replace(/title: (Intro|Welcome \d+)\n/, `title: ${title}\n`);
expect((await store.saveFiles("imp", ada, [{ path: "stcourse.yaml", content: text, baseHash: m.hash }], title)).ok).toBe(true);
};
test("un-ignoring the media in .gitignore: publish keeps working, again and again", async () => {
const { dataDir, store } = await newStore();
makeImportedCourse(dataDir);
await store.convertImported("imp", ada);
expect((await store.saveFiles("imp", ada, [{ path: ".gitignore", content: "!*.mp4\n!*.pdf\n!/media/**\n" }], "tidy .gitignore")).ok).toBe(true);
expect(await store.publish("imp", ada)).toMatchObject({ ok: true, published: true });
const pub = store.paths("imp").published;
// git itself now sees the media as untracked; the studio does not count them
expect(await extGit(pub, "-c", "core.quotepath=false", "status", "--porcelain")).toContain("media/");
expect((await store.status("imp", ada.id)).publishedDirty).toEqual([]);
await editManifest(store, "Welcome 2");
expect(await store.publish("imp", ada)).toMatchObject({ ok: true, published: true });
await editManifest(store, "Welcome 3");
expect(await store.publish("imp", ada)).toMatchObject({ ok: true, published: true });
expect(readFileSync(join(pub, "stcourse.yaml"), "utf8")).toContain("title: Welcome 3");
});
test("a remote commit that rewrites .gitignore: Sync and publish keep working", async () => {
const { dataDir, store } = await newStore({ allowLocalRemotes: true });
makeImportedCourse(dataDir);
await store.convertImported("imp", ada);
const remote = join(tempDir(), "remote.git");
await extGit(tempDir(), "init", "-q", "--bare", "-b", "main", remote);
await store.setRemote("imp", remote);
expect(await store.sync("imp", ada)).toMatchObject({ ok: true, pushed: true });
const other = join(tempDir(), "other");
await extGit(tempDir(), "clone", "-q", remote, other);
writeFileSync(join(other, ".gitignore"), "!*\n");
await extGit(other, "add", ".gitignore");
await extGit(other, "commit", "-qm", "unignore everything");
await extGit(other, "push", "-q", "origin", "main");
expect(await store.sync("imp", ada)).toMatchObject({ ok: true, pulled: true });
expect(await store.updateDraft("imp", ada)).toMatchObject({ ok: true });
await editManifest(store, "Welcome 2");
expect(await store.publish("imp", ada)).toMatchObject({ ok: true, published: true });
expect(await store.sync("imp", ada)).toMatchObject({ ok: true, pushed: true });
});
test("a remote commit that puts a file over referenced media is refused at Sync; the media stay", async () => {
const { dataDir, store } = await newStore({ allowLocalRemotes: true });
makeImportedCourse(dataDir);
await store.convertImported("imp", ada);
const remote = join(tempDir(), "remote.git");
await extGit(tempDir(), "init", "-q", "--bare", "-b", "main", remote);
await store.setRemote("imp", remote);
await store.sync("imp", ada);
const other = join(tempDir(), "other");
await extGit(tempDir(), "clone", "-q", remote, other);
mkdirSync(join(other, "media"), { recursive: true });
writeFileSync(join(other, VIDEO), "not the video");
await extGit(other, "add", "-f", VIDEO);
await extGit(other, "commit", "-qm", "replace media");
await extGit(other, "push", "-q", "origin", "main");
expect(await store.sync("imp", ada)).toMatchObject({ ok: false, reason: "forbidden" });
expect(ino(join(store.paths("imp").published, VIDEO))).toBe(ino(join(dataDir, "media/imp", VIDEO)));
});
test("referenced media are not read whole into the editor", async () => {
const { dataDir, store } = await newStore();
makeImportedCourse(dataDir);
await store.convertImported("imp", ada);
expect(await store.readFile("imp", ada.id, VIDEO)).toBeNull();
expect(await store.readFile("imp", ada.id, "transcripts/l1.md")).not.toBeNull();
});
});
describe("crash safety: a retry after any crash point succeeds and the id is never loaded twice", () => {
const loaded = async (dataDir: string) => (await loadAll([join(dataDir, "courses"), join(dataDir, "worktrees")])).filter((c) => c.id === "imp");
const sameFs = ["committed", "hidden", "worktree", "before-move"];
const crossFs = [...sameFs, "staging-copied", "staging-placed", "source-retired"];
const cases: [string, boolean][] = [...sameFs.map((s) => [s, false] as [string, boolean]), ...crossFs.map((s) => [s, true] as [string, boolean])];
test.each(cases)("crash at %s (cross-filesystem: %p)", async (step, exdev) => {
const dataDir = tempDir();
makeImportedCourse(dataDir);
const crashing = createStudioStore({ dataDir, testHooks: { crashAt: step, forceExdev: exdev } });
await expect(crashing.convertImported("imp", ada)).rejects.toBeInstanceOf(SimulatedCrash);
expect((await loaded(dataDir)).length).toBeLessThanOrEqual(1);
// the process restarts: recovery first, then the owner (or the requeued job) retries
const fresh = createStudioStore({ dataDir, testHooks: { forceExdev: exdev } });
await fresh.recoverConversions();
expect((await loaded(dataDir)).length).toBeLessThanOrEqual(1);
await fresh.convertImported("imp", ada);
// and once more: a re-run of a finished conversion reports success
expect(await fresh.convertImported("imp", ada)).toMatchObject({ resumed: true });
expect(fresh.isManaged("imp")).toBe(true);
const all = await loaded(dataDir);
expect(all).toHaveLength(1);
expect(all[0]!.status).toBe("ok");
expect(all[0]!.root).toBe(fresh.paths("imp").published);
expect(readdirSync(join(dataDir, "courses"))).toEqual([]);
expect(readdirSync(join(dataDir, "media"))).toEqual(["imp"]);
const pub = fresh.paths("imp").published;
for (const p of [VIDEO, VIDEO2, BIG_PDF]) {
expect(statSync(join(pub, p)).size).toBe(statSync(join(dataDir, "media/imp", p)).size);
if (!exdev) expect(ino(join(pub, p))).toBe(ino(join(dataDir, "media/imp", p)));
}
expect(readFileSync(join(dataDir, "media/imp", VIDEO))).toEqual(Buffer.from(videoBytes(4096, 7)));
expect(await extGit(pub, "status", "--porcelain", "--untracked-files=all")).toBe("");
});
test("a failed cross-filesystem copy leaves no staging dir and a retry finishes", async () => {
const dataDir = tempDir();
makeImportedCourse(dataDir);
const store = createStudioStore({ dataDir, testHooks: { forceExdev: true } });
// the media store's parent is a file: the copy fails in-process
writeFileSync(join(dataDir, "media"), "not a dir");
await expect(store.convertImported("imp", ada)).rejects.toThrow();
expect((await loaded(dataDir)).length).toBeLessThanOrEqual(1);
rmSync(join(dataDir, "media"));
await store.convertImported("imp", ada);
expect(readdirSync(join(dataDir, "media"))).toEqual(["imp"]);
expect(readdirSync(join(dataDir, "courses"))).toEqual([]);
expect(await loaded(dataDir)).toHaveLength(1);
});
});
@@ -0,0 +1,86 @@
// A synthetic imported course, laid out as the importer leaves it in <dataDir>/courses/<id>. Nothing here is real material.
import { mkdirSync, symlinkSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { LARGE_FILE_BYTES } from "../src/server/media-refs";
import { tempDir } from "./helpers";
export const VIDEO = "media/01 Intro – Übersicht.mp4";
export const VIDEO2 = "media/[2] Next step.mp4";
export const BIG_PDF = "decks/big deck.pdf";
/** Fake MP4 bytes: the server only streams them, nothing decodes them. */
export const videoBytes = (n = 4096, seed = 7) => new Uint8Array(n).map((_, i) => (i * seed) % 256);
export interface ImportFixtureOpts {
id?: string;
kind?: string;
manifestId?: string;
root?: string;
visibility?: string;
/** Extra hostile bits: a symlink to outside, a .git dir, a nested .gitignore that hides a lesson. */
hostile?: boolean;
}
function put(dir: string, rel: string, content: string | Uint8Array) {
mkdirSync(dirname(join(dir, rel)), { recursive: true });
writeFileSync(join(dir, rel), content);
}
export function manifestYaml(o: ImportFixtureOpts = {}): string {
const id = o.manifestId ?? o.id ?? "imp";
return [
"schema: 1",
`id: ${id}`,
"title: Imported Course",
`visibility: ${o.visibility ?? "restricted"}`,
...(o.root ? [`root: ${o.root}`] : []),
"cover: media/cover.jpg",
"source:",
` kind: ${o.kind ?? "video-folder"}`,
" origin: /import/synthetic",
"decks:",
" - id: d1",
` file: ${BIG_PDF}`,
" attachTo: l1",
"modules:",
" - id: m1",
" title: Videos",
" lessons:",
" - id: l1",
" title: Intro",
" type: video",
` file: "${VIDEO}"`,
" transcript: transcripts/l1.md",
" - id: l2",
" title: Next step",
" type: video",
` file: "${VIDEO2}"`,
" - id: notes",
" title: Reading",
" type: text",
" file: code/S01 - Basics/README.md",
"",
].join("\n");
}
/** Writes the course into `<dataDir>/courses/<id>` and returns that folder. */
export function makeImportedCourse(dataDir: string, o: ImportFixtureOpts = {}): string {
const id = o.id ?? "imp";
const dir = join(dataDir, "courses", id);
mkdirSync(dir, { recursive: true });
put(dir, "stcourse.yaml", manifestYaml(o));
put(dir, VIDEO, videoBytes(4096, 7));
put(dir, VIDEO2, videoBytes(2048, 11));
put(dir, "media/cover.jpg", new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0, 0x10, 0x4a, 0x46, 0x49, 0x46, 0, 1, 0xff, 0xd9]));
put(dir, BIG_PDF, new Uint8Array(LARGE_FILE_BYTES + 100).fill(0x25));
put(dir, "transcripts/l1.md", "# Intro\n\n[0:00] Hello.\n");
put(dir, "code/S01 - Basics/README.md", "# Basics\n\nRead me.\n");
if (o.hostile) {
// the companion code repo's own .gitignore hides Markdown: the lesson must still be committed
put(dir, "code/S01 - Basics/.gitignore", "*.md\n");
put(dir, "code/.git/config", "[core]\n");
const outside = join(tempDir(), "secret.txt");
writeFileSync(outside, "outside\n");
symlinkSync(outside, join(dir, "code/leak.txt"));
}
return dir;
}
@@ -0,0 +1,77 @@
import { describe, expect, test } from "bun:test";
import { mkdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { $ } from "bun";
import { gitignoreLine, isLargeMedia, LARGE_FILE_BYTES, mediaGitignore, parseMediaRefs, sha256File } from "../src/server/media-refs";
import { tempDir } from "./helpers";
describe("isLargeMedia", () => {
test("video and audio by extension, whatever the size", () => {
for (const p of ["a.mp4", "x/B.MOV", "c.webm", "d.mkv", "e.mp3", "f.m4a", "g.wav", "h.flac", "i.avi", "j.m4v"]) expect(isLargeMedia(p, 10)).toBe(true);
});
test("other files only above the size limit", () => {
expect(isLargeMedia("deck.pdf", 1000)).toBe(false);
expect(isLargeMedia("deck.pdf", LARGE_FILE_BYTES + 1)).toBe(true);
expect(isLargeMedia("lesson.md", LARGE_FILE_BYTES)).toBe(false);
});
});
describe("gitignoreLine", () => {
test("anchors and escapes", () => {
expect(gitignoreLine("media/a.mp4")).toBe("/media/a.mp4");
expect(gitignoreLine("media/[1] *?.mp4")).toBe("/media/\\[1\\] \\*\\?.mp4");
expect(gitignoreLine("a\\b.mp4")).toBe("/a\\\\b.mp4");
expect(gitignoreLine("trail.mp4 ")).toBe("/trail.mp4\\ \\ ");
});
test("refuses names a .gitignore cannot express", () => {
expect(() => gitignoreLine("a\nb.mp4")).toThrow();
expect(() => gitignoreLine("a\rb.mp4")).toThrow();
});
test("git ignores exactly the listed paths", async () => {
const dir = tempDir();
await $`git init -q ${dir}`.quiet();
const listed = ["media/01 Intro – Übersicht.mp4", "media/[draft] *star?.mp4", "#hash.mp4", "!bang.mp4", "media/trailing.mp4 ", "deep/a b/c.mkv"];
const siblings = ["media/01 Intro – Übersicht.mp4.txt", "media/d.mp4", "hash.mp4", "bang.mp4", "media/trailing.mp4", "deep/a b/c.mkv2", "sub/#hash.mp4"];
for (const p of [...listed, ...siblings]) {
mkdirSync(join(dir, p, ".."), { recursive: true });
writeFileSync(join(dir, p), "x");
}
writeFileSync(join(dir, ".gitignore"), mediaGitignore(listed, null));
const input = Buffer.from([...listed, ...siblings].join("\0") + "\0");
const r = await $`git -c core.quotepath=false check-ignore -z --no-index --stdin < ${input}`.cwd(dir).quiet().nothrow();
const ignored = r.stdout.toString().split("\0").filter(Boolean).sort();
expect(ignored).toEqual([...listed].sort());
});
test("an existing .gitignore is kept above the media block", () => {
const out = mediaGitignore(["a.mp4"], "node_modules/\n");
expect(out.startsWith("node_modules/\n")).toBe(true);
expect(out).toContain("/a.mp4\n");
expect(mediaGitignore(["a.mp4"], "dist")).toStartWith("dist\n");
});
});
test("sha256File streams the file", async () => {
const dir = tempDir();
const bytes = new Uint8Array(3 * 1024 * 1024 + 7).map((_, i) => i % 251);
writeFileSync(join(dir, "f.bin"), bytes);
expect(await sha256File(join(dir, "f.bin"))).toBe(new Bun.CryptoHasher("sha256").update(bytes).digest("hex"));
});
describe("parseMediaRefs", () => {
const sha = "a".repeat(64);
test("accepts the schema", () => {
expect(parseMediaRefs(JSON.stringify({ schema: 1, files: [{ path: "media/a.mp4", size: 3, sha256: sha }] }))).toEqual([{ path: "media/a.mp4", size: 3, sha256: sha }]);
});
test.each([
"not json", "null", "{}", JSON.stringify({ schema: 2, files: [] }),
JSON.stringify({ schema: 1, files: [{ path: "../x.mp4", size: 1, sha256: sha }] }),
JSON.stringify({ schema: 1, files: [{ path: "/abs.mp4", size: 1, sha256: sha }] }),
JSON.stringify({ schema: 1, files: [{ path: ".git/config", size: 1, sha256: sha }] }),
JSON.stringify({ schema: 1, files: [{ path: "a.mp4", size: -1, sha256: sha }] }),
JSON.stringify({ schema: 1, files: [{ path: "a.mp4", size: 1, sha256: "zz" }] }),
])("rejects %p", (text) => {
expect(parseMediaRefs(text)).toBeNull();
});
});
@@ -1,6 +1,7 @@
import { afterEach, describe, expect, test } from "bun:test";
import { existsSync, mkdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { IMPORTED_MESSAGE } from "../src/server/feature";
import { isAllowedRemote, ownerRequiredFor } from "../src/server/store";
import { ada, extGit, newStore, tempDir } from "./helpers";
import { setupApp } from "./app-helpers";
@@ -229,11 +230,11 @@ describe("I4: layout and imported courses", () => {
});
expect(env.ctx.courses.get("imp")?.status).toBe("ok");
const list = (await env.call("ada", "GET", "/api/studio/courses")).json;
expect(list.courses).toContainEqual(expect.objectContaining({ id: "imp", managed: false, message: "Imported course: read-only in the studio for now" }));
expect((await env.call("ada", "GET", "/api/studio/courses/imp")).json.message).toBe("Imported course: read-only in the studio for now");
expect(list.courses).toContainEqual(expect.objectContaining({ id: "imp", managed: false, message: IMPORTED_MESSAGE }));
expect((await env.call("ada", "GET", "/api/studio/courses/imp")).json.message).toBe(IMPORTED_MESSAGE);
const w = await env.call("ada", "PUT", "/api/studio/courses/imp/files", { message: "x", files: [{ path: "a.md", content: "x" }] });
expect(w.status).toBe(403);
expect(w.json.error).toBe("Imported course: read-only in the studio for now");
expect(w.json.error).toBe(IMPORTED_MESSAGE);
expect((await env.call("ada", "POST", "/api/studio/courses", { id: "imp", title: "Dup" })).status).toBe(409);
// studio repos are not in the importer's folder
expect((await env.call("ada", "POST", "/api/studio/courses", { id: "c1", title: "C" })).status).toBe(201);