merge: v1.1 edit imported courses in the studio
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
@@ -0,0 +1,153 @@
|
||||
// End to end for v1.1 S: import -> make editable in the studio -> edit -> publish, through the production feature set,
|
||||
// on SQLite and Postgres. The learner must see the change and keep progress and notes; media must still answer Range.
|
||||
// Synthetic material only (ffmpeg test pattern videos).
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { existsSync, mkdirSync, mkdtempSync, statSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { loadConfig } from "@stc/core";
|
||||
import { makeVideo } from "../../../packages/feature-import/test/fixtures";
|
||||
import { packageMigrations, productFeatures } from "../src/features";
|
||||
import { startServer } from "../src/main";
|
||||
import { PG, pgSchema } from "./pg";
|
||||
|
||||
const ORIGIN = "http://localhost:3000";
|
||||
const cookieFrom = (r: Response) => r.headers.getSetCookie().map((c) => c.split(";")[0]).filter((c) => !c.endsWith("=")).join("; ");
|
||||
const CLIP1 = "01 Intro – Übersicht.mp4";
|
||||
const CLIP2 = "02 Next step.mp4";
|
||||
|
||||
async function run(databaseUrl: string) {
|
||||
const dataDir = mkdtempSync(join(tmpdir(), "stc-e2e-si-"));
|
||||
const inbox = join(dataDir, "import-inbox");
|
||||
const folder = join(inbox, "Synthetic Course");
|
||||
mkdirSync(folder, { recursive: true });
|
||||
await makeVideo(join(folder, CLIP1), 2);
|
||||
await makeVideo(join(folder, CLIP2), 1);
|
||||
const config = loadConfig({
|
||||
STC_PORT: "0", STC_DATABASE_URL: databaseUrl, STC_SECRET: "e2e-secret-0123456789abcdef0123456789",
|
||||
STC_DATA_DIR: dataDir, STC_COURSE_DIRS: "", STC_PUBLIC_URL: ORIGIN, STC_IMPORT_ROOTS: inbox,
|
||||
});
|
||||
const s = await startServer(config, productFeatures({ studioWatch: false }), { migrations: packageMigrations });
|
||||
const base = s.server.url.toString().replace(/\/$/, "");
|
||||
type Opts = { cookie?: string; body?: unknown; headers?: Record<string, string> };
|
||||
const call = (method: string, path: string, opts: Opts = {}) => {
|
||||
const headers: Record<string, string> = { origin: ORIGIN, ...opts.headers };
|
||||
if (opts.cookie) headers.cookie = opts.cookie;
|
||||
if (opts.body !== undefined) headers["content-type"] = "application/json";
|
||||
return fetch(base + path, { method, headers, body: opts.body === undefined ? undefined : JSON.stringify(opts.body) });
|
||||
};
|
||||
const json = async (method: string, path: string, opts: Opts = {}) => (await call(method, path, opts)).json() as Promise<any>;
|
||||
const signUp = async (email: string, name: string) => {
|
||||
const r = await call("POST", "/api/auth/sign-up/email", { body: { email, name, password: "correct horse battery" } });
|
||||
expect(r.status).toBe(200);
|
||||
return cookieFrom(r);
|
||||
};
|
||||
const waitJob = async (path: string, cookie: string) => {
|
||||
for (let i = 0; i < 600; i++) {
|
||||
const j = await json("GET", path, { cookie });
|
||||
if (j.status === "done" || j.status === "failed") return j;
|
||||
await Bun.sleep(50);
|
||||
}
|
||||
throw new Error(`timeout waiting for ${path}`);
|
||||
};
|
||||
|
||||
try {
|
||||
const owner = await signUp("[email protected]", "Olive Owner");
|
||||
const lea = await signUp("[email protected]", "Lea Learner");
|
||||
const leaId = (await json("GET", "/api/me", { cookie: lea })).user.id as string;
|
||||
|
||||
// 1. import a folder of videos (restricted by default)
|
||||
const imp = await call("POST", "/api/import/jobs", { cookie: owner, body: { kind: "video-folder", path: folder, options: { courseId: "synth", title: "Synthetic Course" } } });
|
||||
expect(imp.status).toBe(202);
|
||||
const impId = ((await imp.json()) as { id: string }).id;
|
||||
let done: any = null;
|
||||
for (let i = 0; i < 600 && !done; i++) {
|
||||
const j = ((await json("GET", "/api/import/jobs", { cookie: owner })) as any[]).find((x) => x.id === impId);
|
||||
if (j && (j.status === "done" || j.status === "failed")) done = j;
|
||||
else await Bun.sleep(50);
|
||||
}
|
||||
expect(done?.error ?? null).toBeNull();
|
||||
expect(done.status).toBe("done");
|
||||
expect(existsSync(join(dataDir, "courses/synth/stcourse.yaml"))).toBe(true);
|
||||
|
||||
// 2. the owner grants the learner access; the learner makes progress and writes a note
|
||||
expect((await call("PUT", "/api/admin/courses/synth/access", { cookie: owner, body: { acl: [{ principal_type: "user", principal_id: leaId }] } })).status).toBe(200);
|
||||
const before = await json("GET", "/api/courses/synth", { cookie: lea });
|
||||
expect(before.modules[0].lessons.map((l: { id: string; title: string }) => [l.id, l.title])).toEqual([["l1", "Intro – Übersicht"], ["l2", "Next step"]]);
|
||||
expect((await call("PUT", "/api/courses/synth/progress/l1", { cookie: lea, body: { state: "done" } })).status).toBe(200);
|
||||
expect((await call("PUT", "/api/courses/synth/progress/l2", { cookie: lea, body: { state: "in-progress", positionSeconds: 1 } })).status).toBe(200);
|
||||
expect((await call("POST", "/api/courses/synth/notes", { cookie: lea, body: { lessonId: "l2", anchorKind: "video", anchorValue: 1, body: "Remember this bit" } })).status).toBe(201);
|
||||
const progressBefore = await json("GET", "/api/courses/synth/progress", { cookie: lea });
|
||||
const notesBefore = await json("GET", "/api/courses/synth/notes", { cookie: lea });
|
||||
expect(progressBefore.l1.state).toBe("done");
|
||||
expect(notesBefore.map((n: { body: string }) => n.body)).toEqual(["Remember this bit"]);
|
||||
|
||||
// 3. only the owner may convert; the learner's attempt changes nothing
|
||||
expect((await call("POST", "/api/studio/courses/synth/convert", { cookie: lea })).status).toBe(403);
|
||||
const ro = await json("GET", "/api/studio/courses/synth", { cookie: owner });
|
||||
expect(ro).toMatchObject({ readOnly: true, imported: true, convertible: true });
|
||||
const conv = await call("POST", "/api/studio/courses/synth/convert", { cookie: owner });
|
||||
expect(conv.status).toBe(202);
|
||||
const job = await waitJob(`/api/studio/jobs/${((await conv.json()) as { jobId: string }).jobId}`, owner);
|
||||
expect(job.error).toBeNull();
|
||||
expect(job.result).toMatchObject({ courseId: "synth", media: 2 });
|
||||
expect(existsSync(join(dataDir, "courses/synth"))).toBe(false);
|
||||
expect(existsSync(join(dataDir, "studio/repos/synth.git/HEAD"))).toBe(true);
|
||||
const pubClip = join(dataDir, "worktrees/synth/published/media", CLIP1);
|
||||
expect(statSync(pubClip).ino).toBe(statSync(join(dataDir, "media/synth/media", CLIP1)).ino);
|
||||
|
||||
// 4. edit a lesson title in the studio and publish
|
||||
const st = await json("GET", "/api/studio/courses/synth", { cookie: owner });
|
||||
expect(st.readOnly).toBe(false);
|
||||
const m = structuredClone(st.manifestRaw);
|
||||
expect(m.visibility).toBe("restricted");
|
||||
m.modules[0].lessons[0].title = "Welcome aboard";
|
||||
expect((await call("PUT", "/api/studio/courses/synth/files", { cookie: owner, body: { message: "Rename the first lesson", manifest: m, manifestBaseHash: st.manifestHash } })).status).toBe(200);
|
||||
// drafts never reach learners
|
||||
expect((await json("GET", "/api/courses/synth", { cookie: lea })).modules[0].lessons[0].title).toBe("Intro – Übersicht");
|
||||
const pub = await call("POST", "/api/studio/courses/synth/publish", { cookie: owner });
|
||||
expect(pub.status).toBe(200);
|
||||
expect(await pub.json()).toMatchObject({ ok: true, published: true });
|
||||
|
||||
// 5. the learner sees the change and keeps progress and notes
|
||||
const after = await json("GET", "/api/courses/synth", { cookie: lea });
|
||||
expect(after.modules[0].lessons.map((l: { title: string }) => l.title)).toEqual(["Welcome aboard", "Next step"]);
|
||||
expect(await json("GET", "/api/courses/synth/progress", { cookie: lea })).toEqual(progressBefore);
|
||||
expect(await json("GET", "/api/courses/synth/notes", { cookie: lea })).toEqual(notesBefore);
|
||||
expect(after.progress.l1.state).toBe("done");
|
||||
|
||||
// 6. media still play: Range gives 206, an unsatisfiable range 416; the URL is unchanged
|
||||
const lesson = await json("GET", "/api/courses/synth/lessons/l1", { cookie: lea });
|
||||
expect(lesson.lesson.title).toBe("Welcome aboard");
|
||||
expect(lesson.video.file).toBe(`media/${CLIP1}`);
|
||||
const size = statSync(pubClip).size;
|
||||
const part = await call("GET", lesson.video.url, { cookie: lea, headers: { range: "bytes=0-99" } });
|
||||
expect(part.status).toBe(206);
|
||||
expect(part.headers.get("content-range")).toBe(`bytes 0-99/${size}`);
|
||||
expect((await part.arrayBuffer()).byteLength).toBe(100);
|
||||
expect((await call("GET", lesson.video.url, { cookie: lea, headers: { range: `bytes=${size + 10}-` } })).status).toBe(416);
|
||||
const full = await call("GET", lesson.video.url, { cookie: lea });
|
||||
expect(full.status).toBe(200);
|
||||
expect((await full.arrayBuffer()).byteLength).toBe(size);
|
||||
|
||||
// 7. still restricted: anonymous users and other accounts see nothing
|
||||
expect((await call("GET", "/api/courses/synth")).status).toBe(404);
|
||||
expect((await call("GET", lesson.video.url)).status).toBe(404);
|
||||
const other = await signUp("[email protected]", "Max Other");
|
||||
expect((await call("GET", "/api/courses/synth", { cookie: other })).status).toBe(404);
|
||||
expect((await call("GET", "/api/media/synth/..%2F..%2Fmedia%2Fsynth%2Fstcourse.yaml", { cookie: lea })).status).toBe(404);
|
||||
} finally {
|
||||
await s.stop();
|
||||
}
|
||||
}
|
||||
|
||||
describe("studio import end to end", () => {
|
||||
test("import, make editable, edit, publish [sqlite]", () => run("sqlite://:memory:"), 120_000);
|
||||
const pg = PG;
|
||||
if (pg) {
|
||||
test("import, make editable, edit, publish [postgres]", async () => {
|
||||
const db = await pgSchema(pg);
|
||||
try { await run(db.url); } finally { await db.drop(); }
|
||||
}, 120_000);
|
||||
}
|
||||
});
|
||||
@@ -22,7 +22,7 @@ Layout after the review (controller ruling on I4):
|
||||
|
||||
Proposal: main.ts calls `createCourseRepository(studioCourseDirs(config))`, or `loadConfig` appends both dirs.
|
||||
|
||||
Imported courses (`source.kind` other than `authored`, with their root under `<dataDir>/courses`) are read-only in the studio for v1, with the message "Imported course: read-only in the studio for now". There is a `TODO(studio)` in `feature.ts` (`readOnlyMessage`) for converting one into a studio-managed repo. The studio refuses to create a course whose id already exists as `<dataDir>/courses/<id>`. WP-B should likewise refuse an import id that `<dataDir>/studio/repos/<id>.git` already uses.
|
||||
Imported courses (`source.kind` other than `authored`, with their root under `<dataDir>/courses`) are read-only in the studio for v1, with the message "Imported course: read-only in the studio for now". There was a `TODO(studio)` in `feature.ts` (`readOnlyMessage`) for converting one into a studio-managed repo; v1.1 S resolved it with the owner-only "Make editable in the studio" action (see `docs/operations.md`, "Imported courses in the studio"). The studio refuses to create a course whose id already exists as `<dataDir>/courses/<id>`. WP-B should likewise refuse an import id that `<dataDir>/studio/repos/<id>.git` already uses.
|
||||
|
||||
## 3. Live preview: swap in the learner `LessonView`
|
||||
|
||||
|
||||
+76
-1
@@ -22,7 +22,7 @@ Mounts (all course mounts are read-only; STCourses never changes your originals)
|
||||
|
||||
| Host | Container | What |
|
||||
|---|---|---|
|
||||
| volume `stc-data` | `/data` | database (SQLite), imported courses (`/data/courses`), studio repos (`/data/studio/repos`) and their published worktrees (`/data/worktrees`), caches (`/data/cache`), AI drafts for non-studio courses (`/data/drafts`) |
|
||||
| volume `stc-data` | `/data` | database (SQLite), imported courses (`/data/courses`), studio repos (`/data/studio/repos`) and their published worktrees (`/data/worktrees`), media of imports made editable in the studio (`/data/media`), caches (`/data/cache`), AI drafts for non-studio courses (`/data/drafts`) |
|
||||
| `STC_COURSES_HOST` (default `./courses`) | `/courses` | course repos, one folder with a `stcourse.yaml` per course |
|
||||
| `STC_IMPORT_HOST` (default `./import-inbox`) | `/import` | files the owner may import (Packt zips, video folders, PDFs, code zips) |
|
||||
|
||||
@@ -78,6 +78,81 @@ Without Docker: `bun install`, then `STC_SECRET=… NODE_ENV=production bun apps
|
||||
- Course media (`/api/media/…`) is sent with `X-Content-Type-Options: nosniff`, and SVG, HTML and JavaScript files
|
||||
with `Content-Security-Policy: sandbox`, so author content cannot run script on the app's origin.
|
||||
|
||||
## Imported courses in the studio
|
||||
|
||||
Imported courses (`<data>/courses/<id>`) are read-only in the studio until the owner chooses **Make editable in the
|
||||
studio** on the course's studio page. That runs a background job (`studio.convert-import`) which:
|
||||
|
||||
1. commits the manifest, lessons, transcripts, small images, small PDFs and code into a new studio repo,
|
||||
`<data>/studio/repos/<id>.git`, checked out as `<data>/worktrees/<id>/published`. The manifest is committed
|
||||
unchanged: the course keeps its id, its visibility (restricted after an import) and its `source.kind`, so only the
|
||||
owner can later make it public or signed-in. ACL entries, progress and notes are keyed by course and lesson id
|
||||
and are kept;
|
||||
2. keeps **large media out of git**: every video or audio file (`mp4 m4v webm mov mkv avi mp3 m4a wav ogg oga opus
|
||||
flac`) and any other file over 8 MiB. Each is listed with its path, size and SHA-256 in `.stcourses/media.json`
|
||||
in the repo. They are kept out of `git status` by the repo's `info/exclude` (shared by every worktree), not by
|
||||
the course's `.gitignore`, and the studio never counts a listed file as a change. So an author or a remote
|
||||
rewriting `.gitignore` cannot block publishing or Sync. Git LFS is not used: it is not in the image and would
|
||||
need an LFS server;
|
||||
3. **moves the import folder to `<data>/media/<id>`** (decided over keeping it in place: a manifest left in
|
||||
`<data>/courses/<id>` would be found as a second course with the same id, and the importer's "add to course" and
|
||||
transcription jobs would keep changing a folder learners no longer see). `<data>/media` is never scanned for
|
||||
courses and nothing writes to it afterwards. File permissions are left alone, because an import made in `link`
|
||||
mode shares its files with your originals;
|
||||
4. puts every referenced file into the published worktree, and later into each author's draft worktree, as a
|
||||
**hard link** to `<data>/media/<id>/<path>`. The manifest paths are unchanged, so `/api/media/<id>/…` serves the
|
||||
same URLs with Range support, and no extra disk space is used. If `<data>/worktrees` and `<data>/media` are on
|
||||
different filesystems, each worktree gets a **full copy** instead: the published one plus one per author who
|
||||
opens a draft. Keep the data directory on one filesystem.
|
||||
|
||||
In the studio, referenced media cannot be replaced or deleted (save, upload and conflict resolution refuse them with
|
||||
409); upload a new file under another name instead. A studio write never goes through a hard link: a file with more
|
||||
than one link is replaced, not overwritten. Publish and Sync refuse a commit that tracks a file at a referenced media
|
||||
path, since it would silently replace the media.
|
||||
|
||||
**Importer jobs and conversion.** Both use the same per-course lock. Conversion is refused (409, "an import job for
|
||||
this course is still running") while an importer job for that course is queued or running. An importer job that
|
||||
starts after the conversion fails with "this course is now edited in the studio" and writes nothing.
|
||||
|
||||
**Crash safety.** The import folder is first renamed to `<data>/courses/.converting-<id>` (dot-folders are never
|
||||
scanned), before the published worktree is checked out, so the course is never loaded twice. Until that checkout is
|
||||
done, a failure undoes everything. After it, the conversion is committed and only the move remains:
|
||||
|
||||
- on one filesystem, one `rename` to `<data>/media/<id>`;
|
||||
- across filesystems, a copy into `<data>/media/.staging-<id>.<n>`, `fsync`, a size and SHA-256 check against
|
||||
`media.json`, a `rename` into place, and then the source is renamed to `<data>/courses/.converted-<id>.<n>` and
|
||||
deleted.
|
||||
|
||||
At server start and before every conversion, STCourses recovers each course deterministically:
|
||||
|
||||
- staging folders are deleted;
|
||||
- a `.converting-` folder without a finished repo is renamed back (the next attempt starts over);
|
||||
- a `.converting-` folder with a finished repo has its move finished;
|
||||
- `.converted-` folders are deleted once `<data>/media/<id>` is complete.
|
||||
|
||||
A job left running by a restart is requeued, and re-running a finished conversion reports success.
|
||||
|
||||
Things to know:
|
||||
|
||||
- Back up `<data>/media` together with `<data>/studio`: the repo alone does not contain the videos.
|
||||
- **Sync** pushes only the git repo. A clone of the remote has the manifest, lessons and `.stcourses/media.json`, but
|
||||
not the media files themselves; copy `<data>/media/<id>` alongside if you move the course to another server.
|
||||
- If a worktree loses a media file (for example after a manual `git clean -x`), the next publish links it back into
|
||||
the published worktree, and a new draft worktree gets every file. A file missing from `<data>/media/<id>` shows
|
||||
as a course diagnostic ("lesson file not found"), never a crash.
|
||||
- After conversion the importer can no longer add decks, code or transcripts to that course (it only edits
|
||||
`<data>/courses`). Add them in the studio.
|
||||
- `<data>/media` is fixed under the data directory (there is no separate setting).
|
||||
- **Discovery skips dot-folders.** Every course directory (`STC_COURSE_DIRS`, `<data>/courses`, `<data>/worktrees`)
|
||||
is scanned to depth 2 for `stcourse.yaml`, and folders whose name starts with `.` are never entered. Do not give a
|
||||
course folder a name that starts with a dot. The conversion's `.converting-<id>` and `.converted-<id>.<n>` folders
|
||||
rely on this.
|
||||
- **A failed conversion.** If the media move fails after the commit (disk full, or a copy that fails its checksum),
|
||||
learners are switched to the studio course right away; its media are hard links into the hidden import folder.
|
||||
The course's studio page shows **Conversion incomplete** with a **Retry** button for the owner. A retry (or the
|
||||
next server start) finishes the move. A failure before the commit is rolled back, and learners keep the import
|
||||
unchanged.
|
||||
|
||||
## Memory budget
|
||||
|
||||
The deployment must stay under 2 GB idle. Measured with this compose file (`--profile postgres`), the image built
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
# STCourses v1.1 — S: Edit imported courses in the studio
|
||||
|
||||
> **For agentic workers:** executed inline (superpowers:executing-plans) by the S package owner, TDD, one commit per task. Steps use checkbox (`- [ ]`) syntax.
|
||||
|
||||
**Goal:** an owner-only action, "Make editable in the studio", turns an imported course (`<dataDir>/courses/<id>`) into a studio course (`<dataDir>/studio/repos/<id>.git` plus `<dataDir>/worktrees/<id>/published`) with the same id, so authors can edit and publish it and learners keep their progress and notes. Resolves `TODO(studio): convert imported course to a studio-managed repo` (ruling R9).
|
||||
|
||||
**Requirements:** `docs/superpowers/plans/2026-10-30-v1-1.md` ("Global Constraints", package S); spec sections 5, 8, 9.1, 12, 13; `CLAUDE.md` project rules.
|
||||
|
||||
## Decisions
|
||||
|
||||
1. **Large media stay out of git, referenced by path, size and checksum (spec 5).** No Git LFS (not installed in the image, and it would need a remote). A file is "large media" when its extension is video/audio (`mp4 m4v webm mov mkv avi mp3 m4a wav ogg oga opus flac`) or it is over 8 MiB. Everything else (manifest, Markdown, small images, small PDFs, code) is committed.
|
||||
- The repo gets `.stcourses/media.json` (`{ schema: 1, files: [{ path, size, sha256 }] }`) and a root `.gitignore` with one anchored, escaped line per referenced path (an existing root `.gitignore` is kept above our block).
|
||||
- Every studio worktree (published and each author's draft) gets the referenced files materialised at their manifest paths as **hard links** to the media store (copy with `--reflink=auto` across filesystems). Manifest paths do not change; core's media route (Range 206/416, 404 outside the root) serves them unchanged. No symlinks: the loader's `resolveInside` and the studio's write rules both refuse paths that leave the course root.
|
||||
2. **The import folder is moved to `<dataDir>/media/<id>`** (a `rename`, same filesystem) rather than kept in place. Reasons: a manifest left in `<dataDir>/courses/<id>` would be discovered as a second course with the same id (and would win: imported dirs are scanned before worktrees); the importer's "add to course" and transcription jobs would keep editing a folder learners no longer see. `<dataDir>/media` is never scanned. Nothing writes there after the move; permissions are not changed, because the importer's `link` mode shares inodes with the owner's originals.
|
||||
3. **Conversion runs as a job** (`studio.convert-import`), because hashing gigabytes of video can exceed Bun's 10 s idle timeout. The route checks everything it can first and answers `202 { jobId }`; the UI polls `GET /api/studio/jobs/:jobId`. The job is restart-safe: a conversion interrupted before the move is rolled back and redone; one interrupted after the repo exists only finishes the move.
|
||||
4. **Same id, same manifest.** The manifest is committed unchanged: `visibility` (restricted by default for imports) and `source.kind` stay, so the v1 owner-only rule at publish still applies (only the owner can make it public or signed-in, or change `source.kind`). ACL rows, progress and notes are keyed by course and lesson ids, which do not change.
|
||||
5. **Referenced media are read-only in the studio.** Saves, uploads, deletes and conflict resolutions that touch a referenced path or `.stcourses/media.json` get 409 with a message to upload under a new name. As defence in depth, any studio write to a file with more than one hard link unlinks it first, so a write can never reach a shared inode.
|
||||
6. **Hostile input:** symlinks and special files in the import folder are skipped (never followed), `.git` folders are skipped, a large file whose name contains a newline (inexpressible in `.gitignore`) refuses the conversion, and the published worktree must be clean after checkout or the conversion rolls back. Materialisation checks each reference with `checkRelPath`, requires a regular file whose real path is inside `<dataDir>/media/<id>`, and writes through `safeWritePath` (refuses symlinks in the destination path).
|
||||
|
||||
## Global Constraints
|
||||
- Everything in `CLAUDE.md` "Project rules" and the v1.1 "Global Constraints" applies. No DB change is needed (no migration).
|
||||
- v1 safeguards stay: symlink-refusing writes, the remote allowlist, the owner-only visibility rule at publish, core's CSRF Origin check (applies to the new POST route automatically), repos at `<dataDir>/studio/repos/<id>.git`.
|
||||
- Synthetic fixtures only. `bun test`, `bun run typecheck` and the Postgres run (`STC_TEST_PG_URL`) stay green.
|
||||
|
||||
## File structure
|
||||
- Create `packages/feature-studio/src/server/media-refs.ts`: classification, `.gitignore` escaping, hashing, reference file read/write.
|
||||
- Create `packages/feature-studio/src/server/convert.ts`: `convertImported` (stage, commit, worktree, materialise, move).
|
||||
- Modify `packages/feature-studio/src/server/store.ts`: `mediaDir` option, `materializeMedia`, draft materialisation, write guards, `listFiles` includes referenced media, `convertImported` wiring.
|
||||
- Modify `packages/feature-studio/src/server/feature.ts`: `POST /api/studio/courses/:courseId/convert`, `GET /api/studio/jobs/:jobId`, job handler, `convertible` flag; remove the TODO.
|
||||
- Modify `packages/feature-studio/src/ui/*`: convert panel in the read-only view, imported tag on the studio home.
|
||||
- Create tests: `packages/feature-studio/test/media-refs.test.ts`, `convert.test.ts`, `convert-routes.test.ts`, `convert-ui.test.tsx`; `apps/server/test/studio-import-e2e.test.ts`.
|
||||
- Modify `docs/operations.md` (media decision, layout).
|
||||
|
||||
### Task 1: Media references
|
||||
- [ ] Tests: `isLargeMedia` by extension and size; `gitignoreLine` escapes `\ * ? [ ] # !` and trailing spaces, and `git check-ignore` matches exactly the listed paths for names with spaces, brackets, `#`, `!`, non-ASCII and trailing spaces (and does not match siblings); `sha256File` matches `Bun.CryptoHasher`; `parseMediaRefs` rejects bad shapes.
|
||||
- [ ] Implement `media-refs.ts`. Run tests, typecheck. Commit.
|
||||
|
||||
### Task 2: Conversion in the store
|
||||
- [ ] Tests (`convert.test.ts`, synthetic import folder built in the test: manifest, lesson Markdown, a fake `.mp4` with space and non-ASCII name, a > 8 MiB PDF, a nested `.gitignore` that ignores a lesson file, a symlink pointing outside, a `.git` dir):
|
||||
- the bare repo, published worktree and media store exist; the import folder is gone; `isManaged`;
|
||||
- git tracks the manifest, lessons (including the nested-ignored one) and small files, never the video or big PDF; the symlink and `.git` contents are not tracked;
|
||||
- `.stcourses/media.json` lists path, size, sha256; the published worktree is clean; the video in the worktree is a hard link (same inode) to `<dataDir>/media/<id>/…`;
|
||||
- a draft worktree gets the media too; `listFiles` includes them;
|
||||
- saving, uploading or deleting a referenced path or the reference file is refused (409); a write to a hard-linked file never changes the other link;
|
||||
- refusals: not under the imports dir, `source.kind: authored`, manifest id mismatch, `root:` set, already a studio course, media store already present; each leaves the import folder untouched and no repo behind;
|
||||
- resume: a leftover partial repo is rebuilt; a managed course whose import folder still exists only finishes the move;
|
||||
- publish after an edit keeps the media in the published worktree.
|
||||
- [ ] Implement `convert.ts` and the store changes. Run studio tests, typecheck. Commit.
|
||||
|
||||
### Task 3: Routes and job
|
||||
- [ ] Tests (`convert-routes.test.ts`, real job queue from `@stc/core`):
|
||||
- owner `POST …/convert` gives 202 and a job id; polling reaches `done`; the course is then studio-managed, served from the published worktree, still restricted, with ACL rows unchanged;
|
||||
- an author or learner gets 403; a foreign Origin gets 403 (CSRF); an overlay or studio course gets 409; an unknown course 404; a second request while one is queued gets 409;
|
||||
- `GET /api/studio/courses/:id` for an imported course says `convertible: true` for the owner only;
|
||||
- after conversion a non-owner author can edit and publish a lesson title, but cannot publish `visibility: public` (403); the owner can.
|
||||
- [ ] Implement. Run tests, typecheck. Commit.
|
||||
|
||||
### Task 4: Studio UI
|
||||
- [ ] Test (`convert-ui.test.tsx`, server render): the read-only panel shows "Make editable in the studio" and the explanation only when `convertible`; progress and error states render.
|
||||
- [ ] Implement `ConvertPanel` (button, job polling, then reload into the workbench) and the imported tag on the home list. Existing tokens and classes only; no new colours, fonts or motion. Commit.
|
||||
|
||||
### Task 5: End-to-end
|
||||
- [ ] `apps/server/test/studio-import-e2e.test.ts`, SQLite and Postgres: production feature set; owner signs up; a synthetic video folder is imported through `/api/import/jobs`; a learner is granted access by ACL, marks a lesson done and writes a note; owner converts; edits a lesson title in the studio; publishes. The learner sees the new title, the same progress and notes, media answer `Range` with 206 and a bad range with 416; anonymous users still get 404.
|
||||
- [ ] Commit.
|
||||
|
||||
### Task 6: Docs, smoke run, report
|
||||
- [ ] `docs/operations.md`: the media decision and the new `<dataDir>/media` folder; backups; that media are not pushed to studio remotes.
|
||||
- [ ] Dev server on port 3492 with `STC_DATA_DIR=/tmp/stc-v11-studio`: import, convert, edit, publish by curl.
|
||||
- [ ] Full `bun test`, typecheck, Postgres run. Commit. Write the report.
|
||||
@@ -181,6 +181,7 @@ Phase 0 (section 10) pins down the exact schema. These principles are fixed:
|
||||
- The two reference courses must load by adding only a manifest and front matter, without moving or rewriting their existing files.
|
||||
- Their existing tooling (test scripts, `tools/*.mjs`) must keep working.
|
||||
- **Derived data lives in a cache directory that can be rebuilt from the files:** search index, slide images and text, transcripts, video posters and AI embeddings.
|
||||
- **Discovery** (v1.1 note): the platform finds `stcourse.yaml` at depth 0–2 under each course directory and skips folders whose name starts with `.` (and `node_modules`). A course folder must therefore not be a dot-folder. The studio relies on this to hide an import folder while it is converted (`.converting-<id>`, `.converted-<id>.<n>`).
|
||||
|
||||
## 6. Accounts, roles and visibility
|
||||
|
||||
|
||||
@@ -3,3 +3,4 @@ export { createJobQueue } from "./jobs";
|
||||
export { serveMedia, contentType, contentDisposition, mediaSecurityHeaders, MEDIA_SANDBOX_CSP } from "./media";
|
||||
export { createApp, canView, crossSiteRefusal, type Routes } from "./app";
|
||||
export { composeServer, type ComposeOptions, type ComposedServer } from "./compose";
|
||||
export { withCourseLock } from "./locks";
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
const locks = new Map<string, Promise<unknown>>();
|
||||
|
||||
/**
|
||||
* Serialises work on one course across features in this process: the studio (saves, publish, conversion of an import)
|
||||
* and the importer's jobs that add to an existing course. A failed holder never blocks the next one.
|
||||
*/
|
||||
export function withCourseLock<T>(courseId: string, fn: () => Promise<T>): Promise<T> {
|
||||
const prev = locks.get(courseId) ?? Promise.resolve();
|
||||
const next = prev.then(fn, fn);
|
||||
const tail = next.catch(() => {});
|
||||
locks.set(courseId, tail);
|
||||
tail.then(() => { if (locks.get(courseId) === tail) locks.delete(courseId); });
|
||||
return next;
|
||||
}
|
||||
@@ -39,7 +39,7 @@ export function resolveInside(root: string, rel: string): string | null {
|
||||
}
|
||||
}
|
||||
|
||||
/** Finds stcourse.yaml at depth 0-2 under each dir. */
|
||||
/** Finds stcourse.yaml at depth 0-2 under each dir. Dot-folders are skipped (hidden, or retired by the studio). */
|
||||
export async function discoverManifests(courseDirs: string[]): Promise<string[]> {
|
||||
const found: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
@@ -60,7 +60,7 @@ export async function discoverManifests(courseDirs: string[]): Promise<string[]>
|
||||
}
|
||||
entries.sort((a, b) => a.name.localeCompare(b.name));
|
||||
for (const e of entries) {
|
||||
if (e.isDirectory() && !SKIP_DIRS.has(e.name)) await walk(join(dir, e.name), depth + 1);
|
||||
if (e.isDirectory() && !SKIP_DIRS.has(e.name) && !e.name.startsWith(".")) await walk(join(dir, e.name), depth + 1);
|
||||
}
|
||||
}
|
||||
for (const d of courseDirs) await walk(resolve(d), 0);
|
||||
|
||||
@@ -45,6 +45,13 @@ describe("discovery", () => {
|
||||
const found = (await discoverManifests([d])).map((p) => p.slice(d.length + 1));
|
||||
expect(found.sort()).toEqual(["a/b/stcourse.yaml", "a/stcourse.yaml", "stcourse.yaml"]);
|
||||
});
|
||||
|
||||
test("skips dot-folders (hidden or retired course folders, e.g. an import being converted)", async () => {
|
||||
const d = await tmp();
|
||||
for (const p of ["a/stcourse.yaml", ".converting-a/stcourse.yaml", "x/.hidden/stcourse.yaml"]) await Bun.write(join(d, p), "x");
|
||||
const found = (await discoverManifests([d])).map((p) => p.slice(d.length + 1));
|
||||
expect(found).toEqual(["a/stcourse.yaml"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("overlays and errors", () => {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { mkdir, readdir, rename, stat } from "node:fs/promises";
|
||||
import { join, relative, resolve } from "node:path";
|
||||
import type { ServerContext } from "@stc/contracts";
|
||||
import { withCourseLock } from "@stc/core";
|
||||
import { readRawManifest, writeManifest, type RawManifest } from "./util/manifest";
|
||||
import { slugify, uniqueId } from "./util/text";
|
||||
import { extractFrame, type Tools } from "./media";
|
||||
@@ -65,9 +66,21 @@ export async function publishCourse(d: Deps, stage: string, id: string): Promise
|
||||
|
||||
export interface ManagedCourse { id: string; dir: string; manifest: RawManifest }
|
||||
|
||||
export const STUDIO_OWNED = "this course is now edited in the studio; add files there instead";
|
||||
|
||||
/**
|
||||
* Runs importer work that changes an existing course under the course lock shared with the studio (core
|
||||
* `withCourseLock`), after checking, inside the lock, that the course is still an importer folder. A conversion to a
|
||||
* studio course therefore either waits for the job, or the job fails cleanly with STUDIO_OWNED and writes nothing.
|
||||
*/
|
||||
export function withManagedCourse<T>(d: Deps, courseId: string | undefined, fn: (c: ManagedCourse) => Promise<T>): Promise<T> {
|
||||
return withCourseLock(courseId ?? "", async () => fn(await managedCourse(d, courseId)));
|
||||
}
|
||||
|
||||
/** An existing course the importer may add to. Only folders directly under <dataDir>/courses qualify: overlays and authored repos are never edited. */
|
||||
export async function managedCourse(d: Deps, courseId: string | undefined): Promise<ManagedCourse> {
|
||||
if (!courseId) throw new Error("options.courseId is required: choose the course to add to");
|
||||
if (isPlainId(courseId) && (await exists(studioRepo(d, courseId)))) throw new Error(STUDIO_OWNED);
|
||||
const dir = join(coursesDir(d), courseId);
|
||||
const rel = relative(coursesDir(d), dir);
|
||||
if (!isPlainId(courseId) || rel.startsWith("..") || !(await exists(join(dir, "stcourse.yaml")))) {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { cp, lstat, mkdir, readdir } from "node:fs/promises";
|
||||
import { basename, extname, join } from "node:path";
|
||||
import type { ImportResult, JobContext } from "@stc/contracts";
|
||||
import { finish, managedCourse, saveManifest, stageDir, type Deps } from "../context";
|
||||
import { finish, saveManifest, stageDir, withManagedCourse, type Deps, type ManagedCourse } from "../context";
|
||||
import { attachDeck } from "../deck";
|
||||
import { putFile } from "../media";
|
||||
import { run } from "../util/exec";
|
||||
@@ -48,9 +48,13 @@ async function findPdfs(dir: string, depth = 0, out: string[] = []): Promise<str
|
||||
}
|
||||
|
||||
export async function importCodeRepo(d: Deps, payload: unknown, jc: JobContext): Promise<ImportResult> {
|
||||
const { path, options } = parsePayload(payload, "code-repo");
|
||||
const { options } = parsePayload(payload, "code-repo");
|
||||
return withManagedCourse(d, optStr(options, "courseId"), (course) => importCodeRepoInto(d, course, payload, jc));
|
||||
}
|
||||
|
||||
async function importCodeRepoInto(d: Deps, course: ManagedCourse, payload: unknown, jc: JobContext): Promise<ImportResult> {
|
||||
const { path } = parsePayload(payload, "code-repo");
|
||||
const warnings: string[] = [];
|
||||
const course = await managedCourse(d, optStr(options, "courseId"));
|
||||
if (!GIT_URL.test(path)) await assertAllowedPath(d, path);
|
||||
const stage = await stageDir(d, "repo");
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { stat } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import type { ImportResult, JobContext } from "@stc/contracts";
|
||||
import { finish, managedCourse, saveManifest, type Deps } from "../context";
|
||||
import { finish, saveManifest, withManagedCourse, type Deps } from "../context";
|
||||
import { assertAllowedPath } from "../roots";
|
||||
import { attachDeck } from "../deck";
|
||||
import { optStr, parsePayload } from "./common";
|
||||
@@ -11,7 +11,7 @@ export async function importPdfDeck(d: Deps, payload: unknown, jc: JobContext):
|
||||
const warnings: string[] = [];
|
||||
await assertAllowedPath(d, path);
|
||||
if (!(await stat(path).catch(() => null))?.isFile() || !/\.pdf$/i.test(path)) throw new Error(`not a PDF file: ${path}`);
|
||||
const course = await managedCourse(d, optStr(options, "courseId"));
|
||||
return withManagedCourse(d, optStr(options, "courseId"), async (course) => {
|
||||
const attachTo = optStr(options, "attachTo");
|
||||
if (!attachTo) throw new Error("options.attachTo is required: the lesson or module id the deck belongs to");
|
||||
await jc.reportProgress(0.05, "Copying the deck");
|
||||
@@ -25,4 +25,5 @@ export async function importPdfDeck(d: Deps, payload: unknown, jc: JobContext):
|
||||
await finish(d, course.id, warnings);
|
||||
await jc.reportProgress(1, r.pages ? `Deck attached (${r.pages} slides)` : "Deck attached");
|
||||
return { courseId: course.id, manifestPath: join(course.dir, "stcourse.yaml"), warnings };
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import type { JobContext } from "@stc/contracts";
|
||||
import { finish, managedCourse, saveManifest, stageDir, transcriptCacheDir, type Deps } from "../context";
|
||||
import { finish, saveManifest, stageDir, transcriptCacheDir, withManagedCourse, type Deps, type ManagedCourse } from "../context";
|
||||
import { run } from "../util/exec";
|
||||
import { optStr, withStage } from "./common";
|
||||
|
||||
@@ -36,7 +36,10 @@ export async function transcribeCourse(d: Deps, payload: unknown, jc: JobContext
|
||||
const url = d.transcribeUrl;
|
||||
const courseId = optStr(p, "courseId") ?? "";
|
||||
if (!url) return { courseId, skipped: true, reason: "STC_TRANSCRIBE_URL is not set", transcribed: 0, warnings: [] };
|
||||
const course = await managedCourse(d, courseId);
|
||||
return withManagedCourse(d, courseId, (course) => transcribeInto(d, course, url, p, jc));
|
||||
}
|
||||
|
||||
async function transcribeInto(d: Deps, course: ManagedCourse, url: string, p: Record<string, unknown>, jc: JobContext): Promise<TranscribeResult> {
|
||||
const only = Array.isArray(p.lessonIds) ? new Set(p.lessonIds.map(String)) : null;
|
||||
const lessons = (course.manifest.modules as any[]).flatMap((m) => m.lessons as any[]).filter((l) => l.type === "video" && !l.transcript && (!only || only.has(l.id)));
|
||||
const warnings: string[] = [];
|
||||
|
||||
@@ -3,7 +3,7 @@ import { join, relative, resolve } from "node:path";
|
||||
import type { Feature, RouteHandler } from "@stc/contracts";
|
||||
import { serveMedia } from "@stc/core";
|
||||
import { assertAllowedPath, importRoots, OUTSIDE } from "./roots";
|
||||
import { coverCacheDir, coursesDir, exists, isPlainId, managedCourse, saveManifest, type Deps } from "./context";
|
||||
import { coverCacheDir, coursesDir, exists, isPlainId, saveManifest, STUDIO_OWNED, withManagedCourse, type Deps } from "./context";
|
||||
|
||||
export const KINDS = ["packt-video", "code-repo", "pdf-deck", "video-folder"] as const;
|
||||
const OPTION_KEYS = ["title", "courseId", "attachTo", "media", "transcribe"] as const;
|
||||
@@ -90,13 +90,18 @@ export function importRoutes(deps: () => Deps): Feature["routes"] {
|
||||
if (!isPlainId(rc.params.courseId!)) return json({ error: "Invalid course id" }, 400);
|
||||
const src = join(coverCacheDir(d, rc.params.courseId!), name);
|
||||
if (!(await exists(src))) return json({ error: "No such candidate" }, 404);
|
||||
let course;
|
||||
try { course = await managedCourse(d, rc.params.courseId); } catch (e) { return json({ error: (e as Error).message }, 404); }
|
||||
await mkdir(join(course.dir, "media"), { recursive: true });
|
||||
await copyFile(src, join(course.dir, "media", "cover.jpg"));
|
||||
course.manifest.cover = "media/cover.jpg";
|
||||
await saveManifest(course.dir, course.manifest);
|
||||
await d.ctx.courses.reload(course.id);
|
||||
try {
|
||||
await withManagedCourse(d, rc.params.courseId, async (course) => {
|
||||
await mkdir(join(course.dir, "media"), { recursive: true });
|
||||
await copyFile(src, join(course.dir, "media", "cover.jpg"));
|
||||
course.manifest.cover = "media/cover.jpg";
|
||||
await saveManifest(course.dir, course.manifest);
|
||||
await d.ctx.courses.reload(course.id);
|
||||
});
|
||||
} catch (e) {
|
||||
const msg = (e as Error).message;
|
||||
return json({ error: msg }, msg === STUDIO_OWNED ? 409 : 404);
|
||||
}
|
||||
return json({ cover: "media/cover.jpg" });
|
||||
}),
|
||||
},
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
// v1.1 S review fix: importer jobs and "Make editable in the studio" share the course lock. Either order is safe:
|
||||
// a job that started first finishes and its files are converted with the course; a job that comes second fails
|
||||
// cleanly and never recreates <dataDir>/courses/<id>.
|
||||
import { expect, test } from "bun:test";
|
||||
import { existsSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { $ } from "bun";
|
||||
import { loadAll } from "@stc/course-format";
|
||||
import { createStudioStore } from "@stc/feature-studio";
|
||||
import { STUDIO_OWNED } from "../src/context";
|
||||
import { importCodeRepo } from "../src/jobs/code-repo";
|
||||
import { importPacktVideo } from "../src/jobs/packt-video";
|
||||
import { transcribeCourse } from "../src/jobs/transcribe";
|
||||
import { makeEnv } from "./harness";
|
||||
import { makePacktZip, makeRepoZip, tmp } from "./fixtures";
|
||||
|
||||
const owner = { name: "Olive Owner", email: "[email protected]" };
|
||||
|
||||
async function imported(env: Record<string, string> = {}) {
|
||||
const e = await makeEnv(env);
|
||||
const r = await importPacktVideo(e.deps, { path: await makePacktZip(await tmp()), options: { title: "Race Test" } }, e.jc);
|
||||
const store = createStudioStore({ dataDir: e.dataDir });
|
||||
const count = async () => (await loadAll([join(e.dataDir, "courses"), join(e.dataDir, "worktrees")])).filter((c) => c.id === r.courseId).length;
|
||||
return { e, id: r.courseId, store, count };
|
||||
}
|
||||
|
||||
test("an importer job that started first finishes; the conversion waits and commits its files", async () => {
|
||||
const { e, id, store, count } = await imported();
|
||||
const zip = await makeRepoZip(await tmp());
|
||||
const job = importCodeRepo(e.deps, { kind: "code-repo", path: zip, options: { courseId: id } }, e.jc);
|
||||
const conv = store.convertImported(id, owner);
|
||||
const [res] = await Promise.all([job, conv]);
|
||||
expect(res.courseId).toBe(id);
|
||||
const pub = store.paths(id).published;
|
||||
const files = (await $`git -c core.quotepath=false ls-files`.cwd(pub).quiet()).stdout.toString();
|
||||
expect(files).toContain("code/S02 - Images (Sample)/app/main.py");
|
||||
expect(await Bun.file(join(pub, "stcourse.yaml")).text()).toContain("code/S02 - Images (Sample)");
|
||||
expect(existsSync(join(e.dataDir, "courses", id))).toBe(false);
|
||||
expect(await count()).toBe(1);
|
||||
});
|
||||
|
||||
test("an importer job that comes after the conversion fails cleanly and writes nothing", async () => {
|
||||
const { e, id, store, count } = await imported({ STC_TRANSCRIBE_URL: "http://127.0.0.1:9/never" });
|
||||
const zip = await makeRepoZip(await tmp());
|
||||
const conv = store.convertImported(id, owner);
|
||||
const job = importCodeRepo(e.deps, { kind: "code-repo", path: zip, options: { courseId: id } }, e.jc);
|
||||
const tr = transcribeCourse(e.deps, { courseId: id }, e.jc);
|
||||
const [c, j, t] = await Promise.allSettled([conv, job, tr]);
|
||||
expect(c.status).toBe("fulfilled");
|
||||
expect(j.status === "rejected" && (j.reason as Error).message).toBe(STUDIO_OWNED);
|
||||
expect(t.status === "rejected" && (t.reason as Error).message).toBe(STUDIO_OWNED);
|
||||
expect(existsSync(join(e.dataDir, "courses", id))).toBe(false);
|
||||
expect(await count()).toBe(1);
|
||||
// and later ones too
|
||||
await expect(importCodeRepo(e.deps, { kind: "code-repo", path: zip, options: { courseId: id } }, e.jc)).rejects.toThrow(STUDIO_OWNED);
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
// Server entry of @stc/feature-studio. The browser imports "@stc/feature-studio/web" instead (no server code).
|
||||
export { studioFeature, OUTSIDE_MESSAGE, IMPORTED_MESSAGE, uploadMaxBytes, UPLOAD_EXTENSIONS, type StudioFeature, type StudioFeatureOptions } from "./server/feature";
|
||||
export { studioFeature, OUTSIDE_MESSAGE, IMPORTED_MESSAGE, CONVERT_JOB, uploadMaxBytes, UPLOAD_EXTENSIONS, type StudioFeature, type StudioFeatureOptions } from "./server/feature";
|
||||
export { createDraftWriter, type DraftWriter, type DraftWriterOptions } from "./server/draft-writer";
|
||||
export { createStudioStore, contentHash, isAllowedRemote, ownerRequiredFor, type StudioStore, type Identity, type DraftStatus, type PublishResult, type SyncResult } from "./server/store";
|
||||
export { createPublishedWatcher, type PublishedWatcher } from "./server/watch";
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import { constants } from "node:fs";
|
||||
import { copyFile, link, mkdir, open, readdir, stat } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { $ } from "bun";
|
||||
import { gitignoreLine, isLargeMedia } from "./media-refs";
|
||||
import { checkRelPath } from "./relpath";
|
||||
|
||||
export interface ScannedImport {
|
||||
/** Committed into git. */
|
||||
small: string[];
|
||||
/** Kept out of git, referenced by path, size and checksum. */
|
||||
large: { path: string; size: number }[];
|
||||
/** Skipped and never followed: symlinks, sockets, `.git` folders. */
|
||||
skipped: string[];
|
||||
}
|
||||
|
||||
/** Walks an import folder with lstat semantics: symlinks and special files are skipped, never followed. */
|
||||
export async function scanImport(root: string): Promise<ScannedImport> {
|
||||
const out: ScannedImport = { small: [], large: [], skipped: [] };
|
||||
const walk = async (rel: string) => {
|
||||
const entries = await readdir(join(root, rel), { withFileTypes: true });
|
||||
entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
||||
for (const e of entries) {
|
||||
const r = rel ? `${rel}/${e.name}` : e.name;
|
||||
if (e.name.toLowerCase() === ".git") { out.skipped.push(r); continue; }
|
||||
if (e.isDirectory()) await walk(r);
|
||||
else if (e.isFile()) {
|
||||
const size = (await stat(join(root, r))).size;
|
||||
if (isLargeMedia(r, size)) out.large.push({ path: r, size });
|
||||
else out.small.push(r);
|
||||
} else out.skipped.push(r);
|
||||
}
|
||||
};
|
||||
await walk("");
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Problems that make a large file impossible to reference: a name git ignore files or the studio cannot express. */
|
||||
export function unreferencable(large: { path: string }[]): string[] {
|
||||
const bad: string[] = [];
|
||||
for (const f of large) {
|
||||
try { gitignoreLine(f.path); if (checkRelPath(f.path) !== f.path) throw new Error("not normal"); } catch { bad.push(f.path); }
|
||||
}
|
||||
return bad;
|
||||
}
|
||||
|
||||
/** Test-only marker for a simulated crash: cleanup code must not run for it (a real crash would not run it either). */
|
||||
export class SimulatedCrash extends Error {
|
||||
constructor(public step: string) { super(`simulated crash at ${step}`); }
|
||||
}
|
||||
|
||||
/** A hard link (no extra space), else a reflink or plain copy when linking is impossible (another filesystem, no
|
||||
* permission, too many links). Other errors (the destination exists, the source vanished) are real failures. */
|
||||
export async function linkOrCopy(src: string, dest: string): Promise<void> {
|
||||
await mkdir(dirname(dest), { recursive: true });
|
||||
try {
|
||||
await link(src, dest);
|
||||
return;
|
||||
} catch (e) {
|
||||
if (!["EXDEV", "EPERM", "EMLINK", "ENOTSUP", "EOPNOTSUPP"].includes((e as NodeJS.ErrnoException).code ?? "")) throw e;
|
||||
}
|
||||
const r = await $`cp --reflink=auto -- ${src} ${dest}`.quiet().nothrow();
|
||||
if (r.exitCode !== 0) await copyFile(src, dest, constants.COPYFILE_EXCL);
|
||||
}
|
||||
|
||||
/** fsyncs every file and directory under `root` (or only `root` itself with `shallow`), so a rename after it is durable. */
|
||||
export async function fsyncTree(root: string, o: { shallow?: boolean } = {}): Promise<void> {
|
||||
const sync = async (p: string) => { const fh = await open(p, "r"); try { await fh.sync(); } finally { await fh.close(); } };
|
||||
if (!o.shallow) {
|
||||
for (const e of await readdir(root, { withFileTypes: true })) {
|
||||
const p = join(root, e.name);
|
||||
if (e.isDirectory()) await fsyncTree(p);
|
||||
else if (e.isFile()) await sync(p);
|
||||
}
|
||||
}
|
||||
await sync(root);
|
||||
}
|
||||
@@ -1,16 +1,17 @@
|
||||
import { existsSync, realpathSync } from "node:fs";
|
||||
import { basename, extname, resolve, sep } from "node:path";
|
||||
import type { AuthUser, Feature, RequestContext, RouteHandler, ServerContext } from "@stc/contracts";
|
||||
import type { AuthUser, Feature, JobHandler, RequestContext, RouteHandler, ServerContext } from "@stc/contracts";
|
||||
import { canView, serveMedia } from "@stc/core";
|
||||
import { ManifestSchema, parseManifest, resolveInside, type LoadedCourse } from "@stc/course-format";
|
||||
import { aclRepo } from "@stc/db";
|
||||
import { createDraftWriter, type DraftWriter } from "./draft-writer";
|
||||
import { storeDirs, studioWorktreesDir } from "./paths";
|
||||
import { checkRelPath, createStudioStore, MANIFEST, ownerRequiredFor, StudioError, type FileWrite, type StudioStore } from "./store";
|
||||
import { checkRelPath, createStudioStore, MANIFEST, ownerRequiredFor, StudioError, type FileWrite, type StudioStore, type StudioStoreOptions } from "./store";
|
||||
import { createPublishedWatcher, type PublishedWatcher } from "./watch";
|
||||
|
||||
export const OUTSIDE_MESSAGE = "This course is managed outside STCourses";
|
||||
export const IMPORTED_MESSAGE = "Imported course: read-only in the studio for now";
|
||||
export const IMPORTED_MESSAGE = "Imported course: not editable in the studio until the owner makes it editable";
|
||||
export const CONVERT_JOB = "studio.convert-import";
|
||||
|
||||
/** Upload cap in bytes from AppConfig.uploadMaxMb (STC_UPLOAD_MAX_MB, default 100; review I2). */
|
||||
export function uploadMaxBytes(mb: number | undefined): number {
|
||||
@@ -44,9 +45,12 @@ async function readCapped(req: Request, max: number): Promise<Uint8Array<ArrayBu
|
||||
}
|
||||
|
||||
export interface StudioFeature extends Feature {
|
||||
jobs: Record<string, JobHandler>;
|
||||
store: StudioStore;
|
||||
watcher: PublishedWatcher;
|
||||
draftWriter: DraftWriter;
|
||||
/** Resolves once interrupted conversions are recovered (every studio route waits for it). */
|
||||
ready: Promise<void>;
|
||||
stop(): void;
|
||||
}
|
||||
|
||||
@@ -54,6 +58,8 @@ export interface StudioFeatureOptions {
|
||||
/** Poll the published worktrees for external commits (default true). */
|
||||
watch?: boolean;
|
||||
watchIntervalMs?: number;
|
||||
/** Tests only: conversion failure injection (see StudioStoreOptions.testHooks). */
|
||||
storeTestHooks?: StudioStoreOptions["testHooks"];
|
||||
}
|
||||
|
||||
const json = (body: unknown, status = 200) => Response.json(body, { status });
|
||||
@@ -72,7 +78,7 @@ function manifestToYaml(m: unknown): string {
|
||||
}
|
||||
|
||||
export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = {}): StudioFeature {
|
||||
const store = createStudioStore(storeDirs(ctx.config));
|
||||
const store = createStudioStore({ ...storeDirs(ctx.config), testHooks: opts.storeTestHooks });
|
||||
const watcher = createPublishedWatcher(store, ctx.courses, { intervalMs: opts.watchIntervalMs });
|
||||
const acl = aclRepo(ctx.db);
|
||||
const wtDir = resolve(ctx.config.studioWorktreesDir ?? studioWorktreesDir(ctx.config.dataDir));
|
||||
@@ -99,24 +105,59 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = {
|
||||
const readOnlyMessage = (c: LoadedCourse): string => {
|
||||
const kind = c.manifest?.source?.kind;
|
||||
const underImports = c.root === importedRoot || c.root.startsWith(importedRoot + sep);
|
||||
if (kind && kind !== "authored" && underImports) {
|
||||
// TODO(studio): convert imported course to a studio-managed repo (git init into <dataDir>/studio/repos/<id>.git
|
||||
// plus a published worktree), so authors can edit it and the owner-only visibility rule applies on publish.
|
||||
return IMPORTED_MESSAGE;
|
||||
}
|
||||
if (kind && kind !== "authored" && underImports) return IMPORTED_MESSAGE;
|
||||
return OUTSIDE_MESSAGE;
|
||||
};
|
||||
const isOwner = (u: AuthUser) => u.roles.includes("owner");
|
||||
/** v1.1 S: only the owner may turn an imported course into a studio course ("Make editable in the studio"). */
|
||||
const convertible = (u: AuthUser, courseId: string) => isOwner(u) && store.isConvertibleImport(courseId);
|
||||
|
||||
/** Importer jobs (queued or running) that would write into this course's import folder. */
|
||||
const pendingImports = async (courseId: string) =>
|
||||
(await ctx.jobs.list()).filter((j) => {
|
||||
if (!j.kind.startsWith("import.") || (j.status !== "queued" && j.status !== "running")) return false;
|
||||
const p = (j.payload ?? {}) as { courseId?: unknown; options?: { courseId?: unknown } };
|
||||
return p.courseId === courseId || p.options?.courseId === courseId;
|
||||
});
|
||||
/** 409 text naming the blocking importer jobs (id and kind), so the owner can find them on the import page. */
|
||||
const importRunning = (jobs: { id: string; kind: string }[]) =>
|
||||
`an import job for this course is still running (${jobs.map((j) => `${j.kind} ${j.id}`).join(", ")}); make it editable once it has finished`;
|
||||
|
||||
/** The conversion job: payload `{ courseId, actor }`, checked again by the store under the course lock. */
|
||||
const convertJob: JobHandler = async (payload, jc) => {
|
||||
await ready;
|
||||
const p = (payload ?? {}) as { courseId?: unknown; actor?: { id?: unknown; name?: unknown; email?: unknown } };
|
||||
const courseId = typeof p.courseId === "string" ? p.courseId : "";
|
||||
const blocking = await pendingImports(courseId);
|
||||
if (blocking.length) throw new StudioError(importRunning(blocking), 409);
|
||||
const actor = { name: String(p.actor?.name ?? "STCourses"), email: String(p.actor?.email ?? "[email protected]") };
|
||||
try {
|
||||
const r = await store.convertImported(courseId, actor, { onProgress: (f, m) => jc.reportProgress(f, m) });
|
||||
return { courseId, ...r };
|
||||
} finally {
|
||||
// Success or failure, learners must get whatever is on disk now: the intact import (failure before the commit,
|
||||
// rolled back) or the published worktree (after it), never a course whose folder has moved away.
|
||||
await ctx.courses.reload(courseId).catch((e) => console.error("[studio] reload after conversion", e));
|
||||
if (store.isManaged(courseId)) await store.publishedHead(courseId).then((h) => watcher.note(courseId, h), () => {});
|
||||
}
|
||||
};
|
||||
|
||||
const afterPublishedChange = async (courseId: string) => {
|
||||
await ctx.courses.reload(courseId);
|
||||
watcher.note(courseId, await store.publishedHead(courseId));
|
||||
};
|
||||
|
||||
// Crash recovery for conversions interrupted by a stop (v1.1 S); learners never see a course twice meanwhile.
|
||||
const ready: Promise<void> = store.recoverConversions()
|
||||
.then(async (ids) => { if (ids.length) await ctx.courses.reload(); })
|
||||
.catch((e) => console.error("[studio] recovery", e));
|
||||
|
||||
/** Wraps a handler: signed-in author/owner (or per-course author), error mapping. */
|
||||
const guard = (h: (req: Request, rc: RequestContext, user: AuthUser) => Promise<Response>): RouteHandler => async (req, rc) => {
|
||||
const user = rc.user;
|
||||
if (!user) return fail(401, "sign in to use the studio");
|
||||
try {
|
||||
await ready;
|
||||
if (!hasRole(user) && (await authoredCourses(user)).length === 0) return fail(403, "the studio is for authors");
|
||||
return await h(req, rc, user);
|
||||
} catch (e) {
|
||||
@@ -154,6 +195,8 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = {
|
||||
}
|
||||
return {
|
||||
readOnly: false,
|
||||
conversionIncomplete: store.conversionIncomplete(courseId),
|
||||
canRetryConversion: isOwner(user),
|
||||
course: { id: courseId, title: (manifestRaw as { title?: string } | null)?.title ?? courseId, managed: true },
|
||||
status,
|
||||
manifestText: m?.content ?? "",
|
||||
@@ -188,7 +231,11 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = {
|
||||
if (all) {
|
||||
for (const c of ctx.courses.list()) {
|
||||
if (isStudioCourse(c) || managed.includes(c.id)) continue;
|
||||
courses.push({ id: c.id, managed: false, title: c.manifest?.title ?? c.id, pitch: c.manifest?.pitch ?? null, accent: c.manifest?.accent ?? null, publishedStatus: c.status, message: readOnlyMessage(c) });
|
||||
const message = readOnlyMessage(c);
|
||||
courses.push({
|
||||
id: c.id, managed: false, title: c.manifest?.title ?? c.id, pitch: c.manifest?.pitch ?? null, accent: c.manifest?.accent ?? null, publishedStatus: c.status, message,
|
||||
imported: message === IMPORTED_MESSAGE, convertible: convertible(user, c.id),
|
||||
});
|
||||
}
|
||||
}
|
||||
return json({ courses, canCreate: all });
|
||||
@@ -212,12 +259,42 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = {
|
||||
if (!store.isManaged(courseId)) {
|
||||
const c = ctx.courses.get(courseId);
|
||||
if (!c) return fail(404, "no such course");
|
||||
return json({ readOnly: true, message: readOnlyMessage(c), course: { id: c.id, title: c.manifest?.title ?? c.id, managed: false, root: c.root }, diagnostics: c.diagnostics });
|
||||
const message = readOnlyMessage(c);
|
||||
return json({ readOnly: true, message, imported: message === IMPORTED_MESSAGE, convertible: convertible(user, c.id), course: { id: c.id, title: c.manifest?.title ?? c.id, managed: false, root: c.root }, diagnostics: c.diagnostics });
|
||||
}
|
||||
return json(await courseState(courseId, user));
|
||||
}),
|
||||
},
|
||||
|
||||
"/api/studio/courses/:courseId/convert": {
|
||||
POST: guard(async (_req, rc, user) => {
|
||||
const courseId = rc.params.courseId ?? "";
|
||||
if (!isOwner(user)) return fail(403, "only the owner can make an imported course editable in the studio");
|
||||
// An incomplete conversion (failed after its commit) is finished by a new job: it recovers first.
|
||||
const incomplete = store.conversionIncomplete(courseId);
|
||||
if (store.isManaged(courseId) && !incomplete) return fail(409, `course "${courseId}" is already a studio course`);
|
||||
if (!incomplete && !store.isConvertibleImport(courseId)) {
|
||||
const c = ctx.courses.get(courseId);
|
||||
return c ? fail(409, `${readOnlyMessage(c) === IMPORTED_MESSAGE ? "this imported course" : OUTSIDE_MESSAGE}: only imported courses in ${importedRoot} can be made editable`) : fail(404, "no such imported course");
|
||||
}
|
||||
const pending = (await ctx.jobs.list(CONVERT_JOB)).find((j) => (j.payload as { courseId?: string } | null)?.courseId === courseId && (j.status === "queued" || j.status === "running"));
|
||||
if (pending) return fail(409, "this course is already being made editable");
|
||||
const blocking = await pendingImports(courseId);
|
||||
if (blocking.length) return fail(409, importRunning(blocking));
|
||||
const jobId = await ctx.jobs.enqueue(CONVERT_JOB, { courseId, actor: { id: user.id, name: who(user).name, email: who(user).email } });
|
||||
return json({ jobId }, 202);
|
||||
}),
|
||||
},
|
||||
|
||||
"/api/studio/jobs/:jobId": {
|
||||
GET: guard(async (_req, rc, user) => {
|
||||
if (!isOwner(user)) return fail(403, "only the owner can follow studio jobs");
|
||||
const j = await ctx.jobs.get(rc.params.jobId ?? "");
|
||||
if (!j || !j.kind.startsWith("studio.")) return fail(404, "no such job");
|
||||
return json({ id: j.id, kind: j.kind, status: j.status, progress: j.progress, message: j.message, error: j.error, result: j.result });
|
||||
}),
|
||||
},
|
||||
|
||||
"/api/studio/courses/:courseId/file": {
|
||||
GET: course(async (_req, rc, user, courseId) => {
|
||||
const path = rc.url.searchParams.get("path") ?? "";
|
||||
@@ -364,5 +441,5 @@ export function studioFeature(ctx: ServerContext, opts: StudioFeatureOptions = {
|
||||
watcher.start();
|
||||
}
|
||||
|
||||
return { name: "studio", routes, multipartRoutes: ["/api/studio/courses/:courseId/upload"], store, watcher, draftWriter, stop: () => watcher.stop() };
|
||||
return { name: "studio", routes, jobs: { [CONVERT_JOB]: convertJob }, multipartRoutes: ["/api/studio/courses/:courseId/upload"], store, watcher, draftWriter, ready, stop: () => watcher.stop() };
|
||||
}
|
||||
|
||||
@@ -55,15 +55,3 @@ export async function gitOk(cwd: string, args: string[], who?: Identity): Promis
|
||||
export function splitZ(out: string): string[] {
|
||||
return out.split("\0").filter(Boolean);
|
||||
}
|
||||
|
||||
const locks = new Map<string, Promise<unknown>>();
|
||||
|
||||
/** Serialises git work per key (the course's bare repo path) across every store instance in the process. */
|
||||
export function withLock<T>(key: string, fn: () => Promise<T>): Promise<T> {
|
||||
const prev = locks.get(key) ?? Promise.resolve();
|
||||
const next = prev.then(fn, fn);
|
||||
const tail = next.catch(() => {});
|
||||
locks.set(key, tail);
|
||||
tail.then(() => { if (locks.get(key) === tail) locks.delete(key); });
|
||||
return next;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import { extname } from "node:path";
|
||||
import { checkRelPath } from "./relpath";
|
||||
|
||||
/**
|
||||
* Large media of a converted import stay out of git (v1.1 S, docs/operations.md "Imported courses in the studio").
|
||||
* The repo commits `MEDIA_REFS` (path, size and checksum per file, spec 5); the repo's `info/exclude` (not the
|
||||
* author-editable `.gitignore`) gets one line per path, and the studio never counts them as changes. Every studio worktree gets the files as hard links to `<dataDir>/media/<courseId>/<path>`.
|
||||
*/
|
||||
export const MEDIA_REFS = ".stcourses/media.json";
|
||||
export const LARGE_FILE_BYTES = 8 * 1024 * 1024;
|
||||
const MEDIA_EXT = new Set(["mp4", "m4v", "webm", "mov", "mkv", "avi", "mp3", "m4a", "wav", "ogg", "oga", "opus", "flac"]);
|
||||
|
||||
export interface MediaRef { path: string; size: number; sha256: string }
|
||||
|
||||
/** Video and audio always; anything else only when it is over `LARGE_FILE_BYTES`. */
|
||||
export function isLargeMedia(rel: string, size: number): boolean {
|
||||
return MEDIA_EXT.has(extname(rel).slice(1).toLowerCase()) || size > LARGE_FILE_BYTES;
|
||||
}
|
||||
|
||||
/** One anchored `.gitignore` pattern matching exactly `rel`. Throws for names a `.gitignore` line cannot hold. */
|
||||
export function gitignoreLine(rel: string): string {
|
||||
if (/[\n\r]/.test(rel)) throw new Error(`a file name with a line break cannot be kept out of git: ${JSON.stringify(rel)}`);
|
||||
let s = rel.replace(/[\\*?[\]]/g, (c) => `\\${c}`);
|
||||
const trail = /( +)$/.exec(s)?.[1]?.length ?? 0;
|
||||
if (trail) s = s.slice(0, -trail) + "\\ ".repeat(trail);
|
||||
return `/${s}`;
|
||||
}
|
||||
|
||||
export const MEDIA_EXCLUDE_HEADER = "# Imported media kept outside git: see .stcourses/media.json (STCourses)";
|
||||
|
||||
/** Ignore-file content: `existing` (if any) first, then the header and one line per referenced file. */
|
||||
export function mediaGitignore(paths: string[], existing: string | null): string {
|
||||
const own = existing ? (existing.endsWith("\n") ? existing : existing + "\n") + "\n" : "";
|
||||
return `${own}${MEDIA_EXCLUDE_HEADER}\n${paths.map(gitignoreLine).join("\n")}\n`;
|
||||
}
|
||||
|
||||
export async function sha256File(abs: string): Promise<string> {
|
||||
const h = new Bun.CryptoHasher("sha256");
|
||||
for await (const chunk of Bun.file(abs).stream()) h.update(chunk);
|
||||
return h.digest("hex");
|
||||
}
|
||||
|
||||
export function stringifyMediaRefs(files: MediaRef[]): string {
|
||||
return JSON.stringify({ schema: 1, files }, null, 2) + "\n";
|
||||
}
|
||||
|
||||
/** The reference list, or null when the text is not a valid one (every path must pass `checkRelPath`). */
|
||||
export function parseMediaRefs(text: string): MediaRef[] | null {
|
||||
let raw: unknown;
|
||||
try { raw = JSON.parse(text); } catch { return null; }
|
||||
const r = raw as { schema?: unknown; files?: unknown } | null;
|
||||
if (!r || r.schema !== 1 || !Array.isArray(r.files)) return null;
|
||||
const out: MediaRef[] = [];
|
||||
for (const f of r.files as Partial<MediaRef>[]) {
|
||||
if (!f || typeof f.path !== "string" || typeof f.size !== "number" || !Number.isInteger(f.size) || f.size < 0) return null;
|
||||
if (typeof f.sha256 !== "string" || !/^[0-9a-f]{64}$/.test(f.sha256)) return null;
|
||||
try { if (checkRelPath(f.path) !== f.path) return null; } catch { return null; }
|
||||
out.push({ path: f.path, size: f.size, sha256: f.sha256 });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { isAbsolute, normalize, sep } from "node:path";
|
||||
|
||||
/** Normalised relative path, or throws. Rejects absolute paths, `..`, `.git` and NUL. */
|
||||
export function checkRelPath(p: string): string {
|
||||
if (typeof p !== "string" || p === "" || p.includes("\0") || isAbsolute(p) || p.includes("\\")) throw new Error(`invalid path: ${JSON.stringify(p)}`);
|
||||
const n = normalize(p);
|
||||
const parts = n.split(sep);
|
||||
if (n === "." || parts[0] === ".." || parts.some((x) => x === ".." || x.toLowerCase() === ".git") || n.endsWith(sep)) {
|
||||
throw new Error(`invalid path: ${JSON.stringify(p)}`);
|
||||
}
|
||||
return n;
|
||||
}
|
||||
@@ -1,8 +1,14 @@
|
||||
import { constants, existsSync, lstatSync, mkdirSync, readdirSync, realpathSync, rmSync, statSync } from "node:fs";
|
||||
import { mkdir, open, rm, unlink } from "node:fs/promises";
|
||||
import { dirname, isAbsolute, join, normalize, sep } from "node:path";
|
||||
import { loadCourse, type Diagnostic } from "@stc/course-format";
|
||||
import { git, gitOk, splitZ, withLock, type Identity } from "./git";
|
||||
import { constants, existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, realpathSync, rmSync, statSync } from "node:fs";
|
||||
import { cp, copyFile, mkdir, open, readFile, rename as fsRename, rm, unlink, writeFile } from "node:fs/promises";
|
||||
import { dirname, join, sep } from "node:path";
|
||||
import { loadCourse, parseManifest, type Diagnostic } from "@stc/course-format";
|
||||
import { withCourseLock } from "@stc/core";
|
||||
import { fsyncTree, linkOrCopy, scanImport, SimulatedCrash, unreferencable } from "./convert";
|
||||
import { git, gitOk, splitZ, type Identity } from "./git";
|
||||
import { gitignoreLine, MEDIA_EXCLUDE_HEADER, MEDIA_REFS, parseMediaRefs, sha256File, stringifyMediaRefs, type MediaRef } from "./media-refs";
|
||||
import { checkRelPath } from "./relpath";
|
||||
|
||||
export { checkRelPath } from "./relpath";
|
||||
|
||||
export type { Identity } from "./git";
|
||||
|
||||
@@ -19,17 +25,6 @@ export function safeAuthor(authorId: string): string {
|
||||
return s === "" ? "_" : s;
|
||||
}
|
||||
|
||||
/** Normalised relative path, or throws. Rejects absolute paths, `..`, `.git` and NUL. */
|
||||
export function checkRelPath(p: string): string {
|
||||
if (typeof p !== "string" || p === "" || p.includes("\0") || isAbsolute(p) || p.includes("\\")) throw new Error(`invalid path: ${JSON.stringify(p)}`);
|
||||
const n = normalize(p);
|
||||
const parts = n.split(sep);
|
||||
if (n === "." || parts[0] === ".." || parts.some((x) => x === ".." || x.toLowerCase() === ".git") || n.endsWith(sep)) {
|
||||
throw new Error(`invalid path: ${JSON.stringify(p)}`);
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
export interface FileWrite {
|
||||
path: string;
|
||||
content?: string | Uint8Array;
|
||||
@@ -76,6 +71,20 @@ export type SyncResult =
|
||||
| { ok: false; reason: "conflict"; conflicts: string[] }
|
||||
| { ok: false; reason: "forbidden"; message: string };
|
||||
|
||||
export interface ConvertResult {
|
||||
commit: string;
|
||||
/** Files kept out of git and referenced from MEDIA_REFS. */
|
||||
media: number;
|
||||
/** Files committed. */
|
||||
files: number;
|
||||
/** Paths skipped in the import folder (symlinks, special files, .git folders). */
|
||||
skipped: string[];
|
||||
/** True when an interrupted conversion was only finished (the repo already existed). */
|
||||
resumed: boolean;
|
||||
}
|
||||
|
||||
export type ConvertProgress = (fraction: number, message: string) => void | Promise<void>;
|
||||
|
||||
export class StudioError extends Error {
|
||||
constructor(message: string, public status = 400) { super(message); }
|
||||
}
|
||||
@@ -129,9 +138,20 @@ export interface StudioStoreOptions {
|
||||
importedDir?: string;
|
||||
/** AppConfig.studioWorktreesDir (default `<dataDir>/worktrees`). */
|
||||
worktreesDir?: string;
|
||||
/**
|
||||
* Media stores of converted imports (default `<dataDir>/media`): `<mediaDir>/<id>` is the former import folder.
|
||||
* Not in AppConfig: production always uses the default; tests may point it elsewhere.
|
||||
*/
|
||||
mediaDir?: string;
|
||||
publishedBranch?: string;
|
||||
/** Tests only: allow local-path remotes (the HTTP feature never sets this). */
|
||||
allowLocalRemotes?: boolean;
|
||||
/**
|
||||
* Tests only, read on every use (a test may change them between calls): simulate a crash (`crashAt`) or an ordinary
|
||||
* failure (`failAt`) at a named conversion step, a cross-filesystem media move (`forceExdev`), or a copy whose
|
||||
* bytes do not match MEDIA_REFS (`corruptStaging`).
|
||||
*/
|
||||
testHooks?: { crashAt?: string; failAt?: string; forceExdev?: boolean; corruptStaging?: boolean };
|
||||
}
|
||||
|
||||
export function createStudioStore(opts: StudioStoreOptions) {
|
||||
@@ -141,6 +161,7 @@ export function createStudioStore(opts: StudioStoreOptions) {
|
||||
const reposDir = opts.reposDir ?? join(dataDir, "studio", "repos");
|
||||
const importedDir = opts.importedDir ?? join(dataDir, "courses");
|
||||
const worktreesDir = opts.worktreesDir ?? join(dataDir, "worktrees");
|
||||
const mediaDir = opts.mediaDir ?? join(dataDir, "media");
|
||||
const remoteGuard = opts.allowLocalRemotes ? ["-c", "protocol.ext.allow=never", "-c", "protocol.fd.allow=never"] : REMOTE_GUARD;
|
||||
|
||||
const paths = (courseId: string) => {
|
||||
@@ -152,6 +173,8 @@ export function createStudioStore(opts: StudioStoreOptions) {
|
||||
published: join(root, "published"),
|
||||
draftsDir: join(root, "drafts"),
|
||||
draft: (authorId: string) => join(root, "drafts", safeAuthor(authorId)),
|
||||
media: join(mediaDir, courseId),
|
||||
imported: join(importedDir, courseId),
|
||||
};
|
||||
};
|
||||
const branchOf = (authorId: string) => `draft/${safeAuthor(authorId)}`;
|
||||
@@ -165,14 +188,30 @@ export function createStudioStore(opts: StudioStoreOptions) {
|
||||
if (!isManaged(courseId)) throw new StudioError(`course ${courseId} is not managed by the studio`, 404);
|
||||
return paths(courseId);
|
||||
};
|
||||
const lock = <T>(courseId: string, fn: () => Promise<T>) => withLock(paths(courseId).bare, fn);
|
||||
/** The process-wide course lock shared with the importer's jobs (core `withCourseLock`). */
|
||||
const lock = <T>(courseId: string, fn: () => Promise<T>) => { paths(courseId); return withCourseLock(courseId, fn); };
|
||||
const convertingDir = (courseId: string) => join(importedDir, `.converting-${courseId}`);
|
||||
const listDir = (dir: string): string[] => { try { return readdirSync(dir); } catch { return []; } };
|
||||
const crash = async (step: string) => {
|
||||
if (opts.testHooks?.crashAt === step) throw new SimulatedCrash(step);
|
||||
if (opts.testHooks?.failAt === step) throw new Error(`injected failure at ${step}`);
|
||||
};
|
||||
const rename = async (from: string, to: string) => fsRename(from, to);
|
||||
|
||||
const rev = async (cwd: string, ref: string) => (await gitOk(cwd, ["rev-parse", ref])).trim();
|
||||
const count = async (cwd: string, range: string) => Number((await gitOk(cwd, ["rev-list", "--count", range])).trim());
|
||||
const unmerged = async (cwd: string) => [...new Set(splitZ(await gitOk(cwd, ["diff", "--name-only", "-z", "--diff-filter=U"])))];
|
||||
const isMerging = async (cwd: string) => (await git(cwd, ["rev-parse", "-q", "--verify", "MERGE_HEAD"])).code === 0;
|
||||
const dirtyFiles = async (cwd: string) =>
|
||||
splitZ(await gitOk(cwd, ["status", "--porcelain", "-z", "--untracked-files=all"])).map((l) => l.slice(3)).filter(Boolean);
|
||||
/**
|
||||
* Uncommitted changes, except untracked referenced media: whatever a `.gitignore` (editable by authors and remotes)
|
||||
* says, the files listed in MEDIA_REFS never make a worktree dirty.
|
||||
*/
|
||||
const dirtyFiles = async (cwd: string) => {
|
||||
const media = new Set((await mediaRefsIn(cwd)).map((r) => r.path));
|
||||
return splitZ(await gitOk(cwd, ["status", "--porcelain", "-z", "--untracked-files=all"]))
|
||||
.filter((l) => !(l.startsWith("?? ") && media.has(l.slice(3))))
|
||||
.map((l) => l.slice(3)).filter(Boolean);
|
||||
};
|
||||
const remoteUrl = async (bare: string) => {
|
||||
const r = await git(bare, ["remote", "get-url", "origin"]);
|
||||
return r.code === 0 ? r.stdout.trim() : null;
|
||||
@@ -241,9 +280,185 @@ export function createStudioStore(opts: StudioStoreOptions) {
|
||||
const branch = branchOf(authorId);
|
||||
const exists = (await git(p.bare, ["rev-parse", "-q", "--verify", `refs/heads/${branch}`])).code === 0;
|
||||
await gitOk(p.bare, exists ? ["worktree", "add", dir, branch] : ["worktree", "add", "-b", branch, dir, main]);
|
||||
await materializeMedia(dir, p.media, p.bare);
|
||||
return dir;
|
||||
}
|
||||
|
||||
/** The draft's or worktree's media references (empty for courses that never were imports). */
|
||||
async function mediaRefsIn(dir: string): Promise<MediaRef[]> {
|
||||
const f = await readFileIn(dir, MEDIA_REFS);
|
||||
return f ? parseMediaRefs(f.content) ?? [] : [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Puts every referenced file into `dir` as a hard link to `fromRoot/<path>` (a copy across filesystems).
|
||||
* Idempotent; existing destinations are left alone. A reference must name a regular file whose real path is inside
|
||||
* `fromRoot`, and the destination path may not contain a symlink. Returns the references it could not satisfy.
|
||||
*/
|
||||
async function materializeMedia(dir: string, fromRoot: string, bare: string): Promise<{ linked: number; missing: string[] }> {
|
||||
const refs = await mediaRefsIn(dir);
|
||||
if (refs.length) await writeExclude(bare, refs.map((r) => r.path));
|
||||
const missing: string[] = [];
|
||||
let linked = 0;
|
||||
let realFrom: string;
|
||||
try { realFrom = realpathSync(fromRoot); } catch { return { linked, missing: refs.map((r) => r.path) }; }
|
||||
for (const r of refs) {
|
||||
try {
|
||||
const src = join(realFrom, r.path);
|
||||
const st = lstatSync(src);
|
||||
const real = realpathSync(src);
|
||||
if (!st.isFile() || !real.startsWith(realFrom + sep)) throw new Error("not a regular file inside the media store");
|
||||
const dest = await safeWritePath(dir, r.path);
|
||||
if (lexists(dest)) continue;
|
||||
await linkOrCopy(real, dest);
|
||||
linked++;
|
||||
} catch {
|
||||
missing.push(r.path);
|
||||
}
|
||||
}
|
||||
if (missing.length) console.warn(`[studio] media missing for ${dir}: ${missing.join(", ")}`);
|
||||
return { linked, missing };
|
||||
}
|
||||
|
||||
/**
|
||||
* Keeps referenced media out of `git status` through the repo's `info/exclude` (shared by every worktree, not
|
||||
* reachable by authors or remotes, unlike `.gitignore`). Lines are only ever added.
|
||||
*/
|
||||
async function writeExclude(bare: string, paths: string[]): Promise<void> {
|
||||
const file = join(bare, "info", "exclude");
|
||||
const cur = await readFile(file, "utf8").catch(() => "");
|
||||
const lines = new Set(cur.split("\n"));
|
||||
const add: string[] = [];
|
||||
for (const x of paths) {
|
||||
let line: string;
|
||||
try { line = gitignoreLine(x); } catch { continue; }
|
||||
if (!lines.has(line)) { add.push(line); lines.add(line); }
|
||||
}
|
||||
if (!add.length) return;
|
||||
const head = cur.includes(MEDIA_EXCLUDE_HEADER) ? "" : `${MEDIA_EXCLUDE_HEADER}\n`;
|
||||
await mkdir(dirname(file), { recursive: true });
|
||||
await writeFile(file, `${cur}${cur && !cur.endsWith("\n") ? "\n" : ""}${head}${add.join("\n")}\n`);
|
||||
}
|
||||
|
||||
/** Every reference present under `root` with its size (and, when asked, its sha256). */
|
||||
async function mediaComplete(root: string, refs: MediaRef[], hash: boolean): Promise<boolean> {
|
||||
for (const r of refs) {
|
||||
try {
|
||||
const st = lstatSync(join(root, r.path));
|
||||
if (!st.isFile() || st.size !== r.size) return false;
|
||||
if (hash && (await sha256File(join(root, r.path))) !== r.sha256) return false;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Renames the old import folder aside (atomic, never discovered: dot-folder) and deletes it. */
|
||||
async function retireSource(courseId: string, from: string): Promise<void> {
|
||||
const gone = join(importedDir, `.converted-${courseId}.${Date.now()}`);
|
||||
await rename(from, gone);
|
||||
await crash("source-retired");
|
||||
await rm(gone, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* Makes `from` (the hidden import folder) the media store `<mediaDir>/<id>`. Same filesystem: one rename.
|
||||
* Across filesystems: copy into a staging dir, fsync, verify against MEDIA_REFS, rename into place, then retire the
|
||||
* source. An existing store that matches MEDIA_REFS means an earlier attempt got that far: only the source is retired.
|
||||
*/
|
||||
async function finishMove(courseId: string, from: string): Promise<void> {
|
||||
const p = paths(courseId);
|
||||
const refs = await mediaRefsIn(p.published);
|
||||
if (lexists(p.media)) {
|
||||
if (!(await mediaComplete(p.media, refs, true))) throw new StudioError(`the media store ${p.media} exists but does not match ${MEDIA_REFS}; move it away and retry`, 409);
|
||||
await retireSource(courseId, from);
|
||||
return;
|
||||
}
|
||||
await crash("before-move");
|
||||
await mkdir(mediaDir, { recursive: true });
|
||||
try {
|
||||
if (opts.testHooks?.forceExdev) throw Object.assign(new Error("EXDEV (test)"), { code: "EXDEV" });
|
||||
await rename(from, p.media);
|
||||
return;
|
||||
} catch (e) {
|
||||
if ((e as NodeJS.ErrnoException).code !== "EXDEV") throw e;
|
||||
}
|
||||
const staging = join(mediaDir, `.staging-${courseId}.${Bun.randomUUIDv7()}`);
|
||||
try {
|
||||
await cp(from, staging, { recursive: true, verbatimSymlinks: true, errorOnExist: true, force: false });
|
||||
if (opts.testHooks?.corruptStaging && refs[0]) await writeFile(join(staging, refs[0].path), "corrupted copy");
|
||||
await crash("staging-copied");
|
||||
await fsyncTree(staging);
|
||||
if (!(await mediaComplete(staging, refs, true))) throw new Error(`the copied media in ${staging} do not match ${MEDIA_REFS}`);
|
||||
await rename(staging, p.media);
|
||||
await fsyncTree(mediaDir, { shallow: true });
|
||||
} catch (e) {
|
||||
if (!(e instanceof SimulatedCrash)) await rm(staging, { recursive: true, force: true });
|
||||
throw e;
|
||||
}
|
||||
await crash("staging-placed");
|
||||
await retireSource(courseId, from);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deterministic recovery of one course's conversion, under its lock:
|
||||
* - staging copies are deleted (a move is always redone from the source);
|
||||
* - a hidden import folder without a finished repo is restored and the partial repo removed (the next attempt
|
||||
* starts over); with a finished repo, the published worktree is reset to its commit and the move finished;
|
||||
* - retired sources are deleted once the media store matches MEDIA_REFS.
|
||||
*/
|
||||
async function recoverUnlocked(courseId: string): Promise<void> {
|
||||
const p = paths(courseId);
|
||||
for (const n of listDir(mediaDir)) if (n.startsWith(`.staging-${courseId}.`)) await rm(join(mediaDir, n), { recursive: true, force: true });
|
||||
const conv = convertingDir(courseId);
|
||||
if (lexists(conv)) {
|
||||
if (!isManaged(courseId)) {
|
||||
rmSync(p.bare, { recursive: true, force: true });
|
||||
rmSync(p.root, { recursive: true, force: true });
|
||||
if (lexists(p.imported)) throw new StudioError(`both ${conv} and ${p.imported} exist; remove one of them by hand`, 409);
|
||||
await rename(conv, p.imported);
|
||||
} else {
|
||||
await gitOk(p.published, ["reset", "-q", "--hard", "HEAD"]);
|
||||
await finishMove(courseId, conv);
|
||||
await materializeMedia(p.published, p.media, p.bare);
|
||||
}
|
||||
}
|
||||
const retired = listDir(importedDir).filter((n) => n.startsWith(`.converted-${courseId}.`));
|
||||
if (retired.length) {
|
||||
const refs = isManaged(courseId) ? await mediaRefsIn(p.published) : null;
|
||||
if (refs && lexists(p.media) && (await mediaComplete(p.media, refs, false))) {
|
||||
for (const n of retired) await rm(join(importedDir, n), { recursive: true, force: true });
|
||||
} else {
|
||||
console.warn(`[studio] ${retired.join(", ")} kept: the media store of ${courseId} is not complete`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Referenced media paths that `treeish` (a draft branch, the remote's branch) tracks in git. Merging those would
|
||||
* silently replace the media-store file with the commit's content, so publish and Sync refuse them.
|
||||
*/
|
||||
async function mediaClash(bare: string, treeish: string): Promise<string[]> {
|
||||
const refs = new Set<string>();
|
||||
for (const t of [main, treeish]) {
|
||||
const text = await showText(bare, `${t}:${MEDIA_REFS}`);
|
||||
for (const r of (text && parseMediaRefs(text)) || []) refs.add(r.path);
|
||||
}
|
||||
if (!refs.size) return [];
|
||||
const r = await git(bare, ["--literal-pathspecs", "ls-tree", "-r", "-z", "--name-only", treeish, "--", ...refs]);
|
||||
return r.code === 0 ? splitZ(r.stdout) : [];
|
||||
}
|
||||
|
||||
/** Referenced media and the reference file are not editable in the studio (they are not in git). */
|
||||
async function refuseMediaWrites(dir: string, paths: string[]): Promise<void> {
|
||||
const locked = new Set([MEDIA_REFS, ...(await mediaRefsIn(dir)).map((r) => r.path)]);
|
||||
const hit = paths.map((x) => checkRelPath(x)).filter((x) => locked.has(x));
|
||||
if (hit.length) {
|
||||
throw new StudioError(`${hit.join(", ")}: imported media are kept outside git and cannot be changed in the studio; upload a file under a new name instead`, 409);
|
||||
}
|
||||
}
|
||||
|
||||
async function readFileIn(dir: string, rel: string): Promise<{ content: string; hash: string } | null> {
|
||||
let abs: string;
|
||||
try { abs = inside(dir, rel); } catch { return null; }
|
||||
@@ -268,6 +483,8 @@ export function createStudioStore(opts: StudioStoreOptions) {
|
||||
if (lexists(abs)) await unlink(abs);
|
||||
} else {
|
||||
const abs = await safeWritePath(dir, f.path);
|
||||
// never write into an inode shared with another path (imported media, an owner's original): replace the file
|
||||
if (lexists(abs) && lstatSync(abs).nlink > 1) await unlink(abs);
|
||||
// O_NOFOLLOW: a symlink swapped in after the check makes the open fail instead of writing through it
|
||||
const fh = await open(abs, constants.O_WRONLY | constants.O_CREAT | constants.O_TRUNC | constants.O_NOFOLLOW, 0o644);
|
||||
try { await fh.writeFile(f.content ?? ""); } finally { await fh.close(); }
|
||||
@@ -323,16 +540,182 @@ export function createStudioStore(opts: StudioStoreOptions) {
|
||||
});
|
||||
},
|
||||
|
||||
/** An importer folder `<importedDir>/<id>` holding an imported (not authored) course the owner may convert. */
|
||||
isConvertibleImport(courseId: string): boolean {
|
||||
if (!ID_RE.test(courseId) || isManaged(courseId)) return false;
|
||||
const p = paths(courseId);
|
||||
try {
|
||||
if (!lstatSync(p.imported).isDirectory()) return false;
|
||||
const m = parseLoose(readFileSync(join(p.imported, MANIFEST), "utf8"));
|
||||
return !!m?.source?.kind && m.source.kind !== "authored";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* A conversion that got past its commit but did not finish (the media move failed or was interrupted): the course
|
||||
* is served from the published worktree, and `convertImported` (which recovers first) completes it.
|
||||
*/
|
||||
conversionIncomplete(courseId: string): boolean {
|
||||
if (!ID_RE.test(courseId)) return false;
|
||||
if (lexists(convertingDir(courseId))) return true;
|
||||
return listDir(mediaDir).some((n) => n.startsWith(`.staging-${courseId}.`)) || listDir(importedDir).some((n) => n.startsWith(`.converted-${courseId}.`));
|
||||
},
|
||||
|
||||
/** Re-links referenced media into the published worktree (after an outside checkout removed them, say). */
|
||||
repairMedia: (courseId: string) => lock(courseId, async () => {
|
||||
const p = requireManaged(courseId);
|
||||
return materializeMedia(p.published, p.media, p.bare);
|
||||
}),
|
||||
|
||||
/**
|
||||
* Finishes or undoes every interrupted conversion (server start; `convertImported` also does it for its course).
|
||||
* Returns the ids it touched. Never throws: a course it cannot repair is logged and left for the owner.
|
||||
*/
|
||||
async recoverConversions(): Promise<string[]> {
|
||||
const ids = new Set<string>();
|
||||
const scan = (dir: string, re: RegExp) => { for (const n of listDir(dir)) { const m = re.exec(n); if (m && ID_RE.test(m[1]!)) ids.add(m[1]!); } };
|
||||
scan(mediaDir, /^\.staging-([^.]+)\./);
|
||||
scan(importedDir, /^\.converting-([^.]+)$/);
|
||||
scan(importedDir, /^\.converted-([^.]+)\./);
|
||||
const done: string[] = [];
|
||||
for (const id of [...ids].sort()) {
|
||||
try { await lock(id, () => recoverUnlocked(id)); done.push(id); } catch (e) { console.error(`[studio] could not recover the conversion of ${id}:`, e); }
|
||||
}
|
||||
return done;
|
||||
},
|
||||
|
||||
/**
|
||||
* Turns the imported course `<importedDir>/<id>` into a studio course with the same id (v1.1 S, ruling R9):
|
||||
* small files are committed as they are (manifest unchanged: visibility and source.kind stay), large media are
|
||||
* referenced from MEDIA_REFS and hard-linked into each worktree, and the import folder ends up as `<mediaDir>/<id>`.
|
||||
* Owner-only at the HTTP layer; holds the shared course lock, so importer jobs on the course wait or fail cleanly.
|
||||
*
|
||||
* Crash safety: the import folder is hidden (`<importedDir>/.converting-<id>`, never discovered) before the
|
||||
* published worktree appears, so the id is never loaded twice. Until the worktree is checked out, a failure undoes
|
||||
* everything; after that the conversion is committed and `recoverUnlocked` (run first by every attempt and at
|
||||
* server start) finishes the move. Across filesystems the media are copied into `<mediaDir>/.staging-<id>.<n>`,
|
||||
* fsynced, verified against MEDIA_REFS (size and sha256) and only then renamed into place.
|
||||
*/
|
||||
convertImported(courseId: string, who: Identity, opts: { onProgress?: ConvertProgress } = {}): Promise<ConvertResult> {
|
||||
if (!ID_RE.test(courseId)) return Promise.reject(new StudioError(`invalid course id: ${courseId}`, 404));
|
||||
const progress = opts.onProgress ?? (() => {});
|
||||
return lock(courseId, async () => {
|
||||
const p = paths(courseId);
|
||||
await recoverUnlocked(courseId);
|
||||
if (isManaged(courseId)) {
|
||||
const refsFile = await readFileIn(p.published, MEDIA_REFS);
|
||||
const media = refsFile ? (parseMediaRefs(refsFile.content) ?? []).length : 0;
|
||||
const head = async () => rev(p.published, "HEAD");
|
||||
// already converted (a job re-run after a restart, say): report success
|
||||
if (refsFile && lexists(p.media) && !lexists(p.imported)) return { commit: await head(), media, files: 0, skipped: [], resumed: true };
|
||||
// a conversion made before the import folder was hidden first: finish its move
|
||||
if (refsFile && lexists(p.imported)) {
|
||||
await finishMove(courseId, p.imported);
|
||||
await materializeMedia(p.published, p.media, p.bare);
|
||||
return { commit: await head(), media, files: 0, skipped: [], resumed: true };
|
||||
}
|
||||
throw new StudioError(`course "${courseId}" is already a studio course`, 409);
|
||||
}
|
||||
const srcManifest = join(p.imported, MANIFEST);
|
||||
if (!lexists(srcManifest)) throw new StudioError(`there is no imported course "${courseId}" in ${importedDir}`, 404);
|
||||
const top = lstatSync(p.imported);
|
||||
if (top.isSymbolicLink() || !top.isDirectory() || lstatSync(srcManifest).isSymbolicLink()) throw new StudioError("the import folder and its manifest must not be symlinks", 400);
|
||||
if (lexists(p.media)) throw new StudioError(`the media store ${p.media} already exists; move it away and retry`, 409);
|
||||
const parsed = parseManifest(await Bun.file(srcManifest).text());
|
||||
if (!parsed.ok) throw new StudioError(`the imported manifest is invalid: ${parsed.errors.join("; ")}`, 400);
|
||||
const m = parsed.manifest;
|
||||
if (m.id !== courseId) throw new StudioError(`the imported manifest has id "${m.id}", not "${courseId}"`, 400);
|
||||
if (m.root) throw new StudioError("an imported course with root: cannot become a studio course", 400);
|
||||
if (!m.source?.kind || m.source.kind === "authored") throw new StudioError("only imported courses can be made editable this way", 400);
|
||||
|
||||
await progress(0.02, "Reading the import folder");
|
||||
const scan = await scanImport(p.imported);
|
||||
const bad = unreferencable(scan.large);
|
||||
if (bad.length) throw new StudioError(`these large files have names the studio cannot keep outside git; rename them first: ${bad.map((b) => JSON.stringify(b)).join(", ")}`, 400);
|
||||
|
||||
// A previous attempt that never got as far as the published worktree: start again.
|
||||
rmSync(p.bare, { recursive: true, force: true });
|
||||
rmSync(p.root, { recursive: true, force: true });
|
||||
|
||||
const total = scan.large.reduce((n, f) => n + f.size, 0) || 1;
|
||||
let done = 0;
|
||||
const refs: MediaRef[] = [];
|
||||
for (const f of scan.large) {
|
||||
await progress(0.05 + 0.75 * (done / total), `Checksumming ${f.path}`);
|
||||
refs.push({ path: f.path, size: f.size, sha256: await sha256File(join(p.imported, f.path)) });
|
||||
done += f.size;
|
||||
}
|
||||
|
||||
const conv = convertingDir(courseId);
|
||||
const work = join(dataDir, "tmp", `convert-${courseId}-${Bun.randomUUIDv7()}`);
|
||||
const stage = join(work, "tree");
|
||||
const list = [...new Set([...scan.small, MEDIA_REFS])];
|
||||
let hidden = false;
|
||||
try {
|
||||
await progress(0.82, "Committing the course files");
|
||||
for (const f of scan.small) {
|
||||
await mkdir(dirname(join(stage, f)), { recursive: true });
|
||||
await copyFile(join(p.imported, f), join(stage, f));
|
||||
}
|
||||
await mkdir(join(stage, dirname(MEDIA_REFS)), { recursive: true });
|
||||
await writeFile(join(stage, MEDIA_REFS), stringifyMediaRefs(refs));
|
||||
await writeFile(join(work, "paths"), list.join("\0") + "\0");
|
||||
|
||||
mkdirSync(reposDir, { recursive: true });
|
||||
await gitOk(dataDir, ["init", "-q", "--bare", "-b", main, p.bare]);
|
||||
await writeExclude(p.bare, refs.map((r) => r.path));
|
||||
const gd = ["--literal-pathspecs", "--git-dir", p.bare, "--work-tree", stage];
|
||||
// -f: a nested .gitignore from the course's own code must not drop course files
|
||||
await gitOk(stage, [...gd, "add", "-f", `--pathspec-from-file=${join(work, "paths")}`, "--pathspec-file-nul"]);
|
||||
await gitOk(stage, [...gd, "commit", "-q", "-m", `Make ${courseId} editable in the studio\n\nConverted from the import in ${p.imported}. ${refs.length} large media files stay outside git (${MEDIA_REFS}).`], who);
|
||||
await crash("committed");
|
||||
|
||||
// Hide the import folder before the published worktree exists: the id is never discoverable twice.
|
||||
await rename(p.imported, conv);
|
||||
hidden = true;
|
||||
await crash("hidden");
|
||||
mkdirSync(p.root, { recursive: true });
|
||||
await gitOk(p.bare, ["worktree", "add", p.published, main]);
|
||||
await progress(0.9, "Linking media into the published course");
|
||||
const mat = await materializeMedia(p.published, conv, p.bare);
|
||||
if (mat.missing.length) throw new Error(`could not link media: ${mat.missing.join(", ")}`);
|
||||
const dirty = await dirtyFiles(p.published);
|
||||
if (dirty.length) throw new Error(`the converted course is not clean after checkout: ${dirty.join(", ")}`);
|
||||
await crash("worktree");
|
||||
} catch (e) {
|
||||
if (!(e instanceof SimulatedCrash)) {
|
||||
rmSync(p.bare, { recursive: true, force: true });
|
||||
rmSync(p.root, { recursive: true, force: true });
|
||||
if (hidden && !lexists(p.imported)) await rename(conv, p.imported);
|
||||
}
|
||||
throw e;
|
||||
} finally {
|
||||
await rm(work, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
// Committed: from here on a failure leaves a state that the next attempt (or a restart) finishes.
|
||||
await progress(0.95, "Moving the import folder to the media store");
|
||||
await finishMove(courseId, conv);
|
||||
await materializeMedia(p.published, p.media, p.bare);
|
||||
return { commit: await rev(p.published, "HEAD"), media: refs.length, files: list.length, skipped: scan.skipped, resumed: false };
|
||||
});
|
||||
},
|
||||
|
||||
ensureDraft: (courseId: string, authorId: string) => lock(courseId, () => ensureDraftUnlocked(courseId, authorId)),
|
||||
|
||||
async listFiles(courseId: string, authorId: string): Promise<string[]> {
|
||||
const dir = await api.ensureDraft(courseId, authorId);
|
||||
const tracked = splitZ(await gitOk(dir, ["ls-files", "-z", "--cached", "--others", "--exclude-standard"]));
|
||||
return [...new Set(tracked)].filter((f) => existsSync(join(dir, f))).sort();
|
||||
const media = (await mediaRefsIn(dir)).map((r) => r.path);
|
||||
return [...new Set([...tracked, ...media])].filter((f) => existsSync(join(dir, f))).sort();
|
||||
},
|
||||
|
||||
async readFile(courseId: string, authorId: string, rel: string) {
|
||||
const dir = await api.ensureDraft(courseId, authorId);
|
||||
// referenced media can be gigabytes and are not text: never read them whole (the raw route streams them)
|
||||
if ((await mediaRefsIn(dir)).some((r) => r.path === rel)) return null;
|
||||
return readFileIn(dir, rel);
|
||||
},
|
||||
|
||||
@@ -348,6 +731,7 @@ export function createStudioStore(opts: StudioStoreOptions) {
|
||||
return lock(courseId, async () => {
|
||||
const dir = await ensureDraftUnlocked(courseId, who.id);
|
||||
if (await isMerging(dir)) throw new StudioError("the draft has a merge in progress: resolve or abort it first", 409);
|
||||
await refuseMediaWrites(dir, files.map((f) => f.path));
|
||||
const conflicts: FileConflict[] = [];
|
||||
for (const f of files) {
|
||||
if (f.baseHash === undefined) continue;
|
||||
@@ -419,6 +803,8 @@ export function createStudioStore(opts: StudioStoreOptions) {
|
||||
const diags = (await api.validateDraftUnlocked(courseId, dir)).filter((d) => d.level === "error");
|
||||
if (diags.length) return { ok: false, reason: "invalid", diagnostics: diags };
|
||||
if ((await count(p.bare, `${main}..${branch}`)) === 0) return { ok: true, commit: await rev(p.published, "HEAD"), published: false };
|
||||
const clash = await mediaClash(p.bare, branch);
|
||||
if (clash.length) return { ok: false, reason: "invalid", diagnostics: clash.map((path) => ({ level: "error" as const, courseId, path, message: `${path} is imported media kept outside git; the draft must not commit a file there` })) };
|
||||
const baseManifest = await showText(p.published, `HEAD:${MANIFEST}`);
|
||||
// Merge without committing, so the merged result is checked before it becomes the published version.
|
||||
const r = await git(p.published, ["merge", "--no-ff", "--no-commit", branch], who);
|
||||
@@ -440,6 +826,7 @@ export function createStudioStore(opts: StudioStoreOptions) {
|
||||
return { ok: false, reason: "invalid", diagnostics: mergedErrors };
|
||||
}
|
||||
await gitOk(p.published, ["commit", "-q", "-m", `Publish ${branch}`], who);
|
||||
await materializeMedia(p.published, p.media, p.bare);
|
||||
// Bring the draft level with what was just published, so "behind" stays meaningful.
|
||||
await git(dir, ["merge", "-q", "--ff-only", main]);
|
||||
return { ok: true, commit: await rev(p.published, "HEAD"), published: true };
|
||||
@@ -501,6 +888,7 @@ export function createStudioStore(opts: StudioStoreOptions) {
|
||||
return lock(courseId, async () => {
|
||||
const dir = await ensureDraftUnlocked(courseId, who.id);
|
||||
if (!(await isMerging(dir))) throw new StudioError("no merge in progress", 409);
|
||||
await refuseMediaWrites(dir, files.map((f) => f.path));
|
||||
const m = files.find((f) => checkRelPath(f.path) === MANIFEST);
|
||||
if (m && !who.owner) {
|
||||
const needOwner = ownerRequiredFor(await showText(requireManaged(courseId).bare, `${main}:${MANIFEST}`), m.content);
|
||||
@@ -562,6 +950,8 @@ export function createStudioStore(opts: StudioStoreOptions) {
|
||||
const local = await rev(p.bare, main);
|
||||
const remote = await rev(p.bare, remoteRef);
|
||||
if (local !== remote) {
|
||||
const clash = await mediaClash(p.bare, remoteRef);
|
||||
if (clash.length) return { ok: false, reason: "forbidden", message: `the remote commits files over imported media kept outside git: ${clash.join(", ")}` };
|
||||
const localBehind = (await git(p.bare, ["merge-base", "--is-ancestor", local, remote])).code === 0;
|
||||
const remoteBehind = (await git(p.bare, ["merge-base", "--is-ancestor", remote, local])).code === 0;
|
||||
const baseManifest = await showText(p.bare, `${main}:${MANIFEST}`);
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { Pill, ProgressBar } from "@stc/ui";
|
||||
import { api, coursePath } from "./api";
|
||||
|
||||
export type ConvertPhase =
|
||||
| { kind: "idle" }
|
||||
| { kind: "running"; progress: number; message: string | null }
|
||||
| { kind: "failed"; error: string };
|
||||
|
||||
export interface ReadOnlyInfo { title: string; message: string; root: string; imported?: boolean; convertible?: boolean }
|
||||
|
||||
/** The owner's "Make editable in the studio" panel for an imported course (v1.1 S). Presentational. */
|
||||
export function ConvertPanelView({ phase, onConvert }: { phase: ConvertPhase; onConvert: () => void }) {
|
||||
const running = phase.kind === "running";
|
||||
return (
|
||||
<section className="studio-section" aria-labelledby="convert-head">
|
||||
<h2 id="convert-head">Make editable in the studio</h2>
|
||||
<p className="studio-lede" style={{ margin: "0 0 12px" }}>
|
||||
The course becomes a studio course with the same id, so learners keep their progress and notes. The manifest,
|
||||
lessons and small files go into a git repository you edit on a draft and publish. Videos and other large files
|
||||
stay outside git, on this server, and keep playing at the same paths. Visibility does not change.
|
||||
</p>
|
||||
{running ? (
|
||||
<div className="studio-convert-progress" aria-live="polite">
|
||||
<ProgressBar value={phase.progress} label="Conversion progress" />
|
||||
<span className="faint">{phase.message ?? "Waiting to start…"}</span>
|
||||
</div>
|
||||
) : null}
|
||||
{phase.kind === "failed" ? <p className="studio-errors" role="alert">{phase.error}</p> : null}
|
||||
<div className="studio-actions">
|
||||
<Pill variant="primary" disabled={running} onClick={onConvert}>{phase.kind === "failed" ? "Try again" : "Make editable in the studio"}</Pill>
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
/** Starts the conversion job (`POST …/convert`) and polls it; calls `onDone` once it has finished. */
|
||||
function useConvertJob(courseId: string, onDone: () => void) {
|
||||
const [phase, setPhase] = useState<ConvertPhase>({ kind: "idle" });
|
||||
const live = useRef(true);
|
||||
useEffect(() => () => { live.current = false; }, []);
|
||||
|
||||
const start = async () => {
|
||||
setPhase({ kind: "running", progress: 0, message: null });
|
||||
const { status, data } = await api("POST", coursePath(courseId, "/convert"));
|
||||
if (status !== 202) return setPhase({ kind: "failed", error: data?.error ?? `Could not start (${status})` });
|
||||
for (;;) {
|
||||
await new Promise((r) => setTimeout(r, 700));
|
||||
if (!live.current) return;
|
||||
const j = await api("GET", `/api/studio/jobs/${encodeURIComponent(data.jobId)}`);
|
||||
if (j.status !== 200) return setPhase({ kind: "failed", error: j.data?.error ?? `Could not follow the conversion (${j.status})` });
|
||||
if (j.data.status === "done") return onDone();
|
||||
if (j.data.status === "failed") return setPhase({ kind: "failed", error: j.data.error ?? "The conversion failed" });
|
||||
setPhase({ kind: "running", progress: j.data.progress ?? 0, message: j.data.message });
|
||||
}
|
||||
};
|
||||
return { phase, start };
|
||||
}
|
||||
|
||||
export function ConvertPanel({ courseId, onDone }: { courseId: string; onDone: () => void }) {
|
||||
const { phase, start } = useConvertJob(courseId, onDone);
|
||||
return <ConvertPanelView phase={phase} onConvert={start} />;
|
||||
}
|
||||
|
||||
/** A conversion that failed after its commit: the course is served, but its media move did not finish. */
|
||||
export function IncompleteConversionView({ phase, canRetry, onRetry }: { phase: ConvertPhase; canRetry: boolean; onRetry: () => void }) {
|
||||
return (
|
||||
<div className="studio-note studio-note--warn" role="status">
|
||||
<strong>Conversion incomplete.</strong> Learners see this course, but moving its imported media into place did not
|
||||
finish. {canRetry ? "Retry to complete it." : "Ask the owner to retry it."}
|
||||
{phase.kind === "running" ? (
|
||||
<div className="studio-convert-progress" aria-live="polite" style={{ marginTop: 8 }}>
|
||||
<ProgressBar value={phase.progress} label="Conversion progress" />
|
||||
<span className="faint">{phase.message ?? "Waiting to start…"}</span>
|
||||
</div>
|
||||
) : null}
|
||||
{phase.kind === "failed" ? <p className="studio-errors" role="alert">{phase.error}</p> : null}
|
||||
{canRetry ? <div className="studio-actions" style={{ marginTop: 8 }}><Pill disabled={phase.kind === "running"} onClick={onRetry}>Retry</Pill></div> : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function IncompleteConversion({ courseId, canRetry, onDone }: { courseId: string; canRetry: boolean; onDone: () => void }) {
|
||||
const { phase, start } = useConvertJob(courseId, onDone);
|
||||
return <IncompleteConversionView phase={phase} canRetry={canRetry} onRetry={start} />;
|
||||
}
|
||||
|
||||
/** What the studio shows for a course it does not manage. */
|
||||
export function ReadOnlyView({ courseId, info, onConverted }: { courseId: string; info: ReadOnlyInfo; onConverted: () => void }) {
|
||||
if (info.imported) {
|
||||
return (
|
||||
<>
|
||||
<div className="studio-note studio-note--warn" role="note">
|
||||
<strong>Imported course.</strong>{" "}
|
||||
{info.convertible ? "It is read-only until you make it editable in the studio." : "It is read-only here. Only the owner can make it editable in the studio."}
|
||||
</div>
|
||||
{info.convertible ? <ConvertPanel courseId={courseId} onDone={onConverted} /> : null}
|
||||
</>
|
||||
);
|
||||
}
|
||||
return <div className="studio-note studio-note--warn" role="note"><strong>{info.message}.</strong> It is read-only here; edit it in its own repository at <code>{info.root}</code>.</div>;
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { Link, useParams, useSearchParams } from "react-router-dom";
|
||||
import { CourseCover, Pill } from "@stc/ui";
|
||||
import { api, coursePath, uniqueId, type Diagnostic, type ManifestLesson } from "./api";
|
||||
import { AccessMessage } from "./StudioHome";
|
||||
import { IncompleteConversion, ReadOnlyView } from "./ConvertPanel";
|
||||
import { clone, useWorkspace, type Workspace } from "./workspace";
|
||||
import { CourseSettings } from "./editors/CourseSettings";
|
||||
import { LessonEditor } from "./editors/LessonEditor";
|
||||
@@ -15,7 +16,7 @@ type Notice = { kind: "ok" | "warn" | "bad"; text: string; files?: string[]; dia
|
||||
|
||||
export default function CourseStudio() {
|
||||
const { courseId = "" } = useParams();
|
||||
const { ws, readOnly, error } = useWorkspace(courseId);
|
||||
const { ws, readOnly, error, reload } = useWorkspace(courseId);
|
||||
|
||||
if (error) return <div className="studio studio-page"><p className="chap"><Link to="/studio">Studio</Link></p>{error.status === 404 ? <div className="studio-note studio-note--bad"><strong>No such course.</strong></div> : <AccessMessage {...error} />}</div>;
|
||||
if (readOnly) {
|
||||
@@ -23,7 +24,7 @@ export default function CourseStudio() {
|
||||
<div className="studio studio-page">
|
||||
<p className="chap"><Link to="/studio">Studio</Link></p>
|
||||
<h1>{readOnly.title}</h1>
|
||||
<div className="studio-note studio-note--warn" role="note"><strong>{readOnly.message}.</strong> It is read-only here; edit it in its own repository at <code>{readOnly.root}</code>.</div>
|
||||
<ReadOnlyView courseId={courseId} info={readOnly} onConverted={reload} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -106,6 +107,8 @@ function Workbench({ ws }: { ws: Workspace }) {
|
||||
<Pill variant="primary" disabled={busy || st.merging || !st.ahead || hasErrors} title={hasErrors ? "Fix the errors in the draft first" : undefined} onClick={publish}>Publish</Pill>
|
||||
</div>
|
||||
|
||||
{ws.state.conversionIncomplete ? <IncompleteConversion courseId={ws.courseId} canRetry={!!ws.state.canRetryConversion} onDone={ws.reload} /> : null}
|
||||
|
||||
{notice ? (
|
||||
<div className={`studio-note studio-note--${notice.kind}`} role="status">
|
||||
{notice.text}
|
||||
|
||||
@@ -6,7 +6,7 @@ import "./studio.css";
|
||||
|
||||
interface CourseRow {
|
||||
id: string; title: string; pitch: string | null; accent: string | null; managed: boolean; publishedStatus: "ok" | "error";
|
||||
ahead?: number; behind?: number; merging?: boolean; remote?: string | null; message?: string;
|
||||
ahead?: number; behind?: number; merging?: boolean; remote?: string | null; message?: string; imported?: boolean; convertible?: boolean;
|
||||
}
|
||||
|
||||
export function AccessMessage({ status, error }: { status: number; error?: string }) {
|
||||
@@ -99,7 +99,7 @@ export default function StudioHome() {
|
||||
{outside.length ? (
|
||||
<section className="studio-section" aria-labelledby="outside-courses">
|
||||
<h2 id="outside-courses">Managed outside STCourses</h2>
|
||||
<p className="faint" style={{ margin: 0 }}>These courses come from repositories the platform does not own. Edit them in their own repository.</p>
|
||||
<p className="faint" style={{ margin: 0 }}>These courses come from repositories the platform does not own, or from imports. Edit them in their own repository; the owner can make an imported course editable here.</p>
|
||||
<ul className="studio-rows">
|
||||
{outside.map((c) => (
|
||||
<li className="studio-row" key={c.id}>
|
||||
@@ -108,7 +108,7 @@ export default function StudioHome() {
|
||||
<Link to={`/studio/${encodeURIComponent(c.id)}`}>{c.title}</Link>
|
||||
<span className="s"><code>{c.id}</code> · {c.message}</span>
|
||||
</div>
|
||||
<span className="studio-tag">Read-only</span>
|
||||
{c.convertible ? <Link className="studio-tag studio-tag--accent" to={`/studio/${encodeURIComponent(c.id)}`}>Make editable</Link> : <span className="studio-tag">{c.imported ? "Imported" : "Read-only"}</span>}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
|
||||
@@ -42,10 +42,12 @@ export interface DraftStatus {
|
||||
export interface Diagnostic { level: "error" | "warning"; lessonId?: string; path?: string; message: string }
|
||||
|
||||
export interface CourseState {
|
||||
readOnly: false; course: { id: string; title: string }; status: DraftStatus; manifestText: string; manifestHash: string | null;
|
||||
readOnly: false; course: { id: string; title: string }; status: DraftStatus;
|
||||
/** A conversion from an import that did not finish its media move (v1.1 S); the owner may retry it. */
|
||||
conversionIncomplete?: boolean; canRetryConversion?: boolean; manifestText: string; manifestHash: string | null;
|
||||
manifestRaw: ManifestDoc | null; manifestErrors: string[]; diagnostics: Diagnostic[]; files: string[];
|
||||
}
|
||||
export interface ReadOnlyState { readOnly: true; message: string; course: { id: string; title: string; root: string }; diagnostics: Diagnostic[] }
|
||||
export interface ReadOnlyState { readOnly: true; message: string; imported?: boolean; convertible?: boolean; course: { id: string; title: string; root: string }; diagnostics: Diagnostic[] }
|
||||
|
||||
export interface FileWrite { path: string; content?: string; baseHash?: string | null; delete?: boolean }
|
||||
export interface FileConflict { path: string; current: string | null; currentHash: string | null }
|
||||
|
||||
@@ -96,3 +96,8 @@
|
||||
.studio-content { padding: 0 16px 48px; width: 100%; }
|
||||
.studio-bar { margin: 0 -16px 20px; padding: 10px 16px; }
|
||||
}
|
||||
|
||||
/* "Make editable in the studio" (imported courses) */
|
||||
.studio-convert-progress { display: flex; align-items: center; gap: 12px; margin: 0 0 12px; max-width: 760px; }
|
||||
.studio-convert-progress .stc-bar { flex: 1; }
|
||||
a.studio-tag { text-decoration: none; }
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import type { ReadOnlyInfo } from "./ConvertPanel";
|
||||
import { api, coursePath, rawUrl, type CourseState, type FileConflict, type FileWrite, type ManifestDoc } from "./api";
|
||||
|
||||
export type SaveOutcome = { ok: true } | { ok: false; conflicts?: FileConflict[]; error?: string };
|
||||
@@ -51,7 +52,7 @@ export type OpenFile = ReturnType<typeof useFile>;
|
||||
|
||||
export function useWorkspace(courseId: string) {
|
||||
const [state, setState] = useState<CourseState | null>(null);
|
||||
const [readOnly, setReadOnly] = useState<{ message: string; title: string; root: string } | null>(null);
|
||||
const [readOnly, setReadOnly] = useState<ReadOnlyInfo | null>(null);
|
||||
const [error, setError] = useState<{ status: number; error?: string } | null>(null);
|
||||
const [doc, setDocState] = useState<ManifestDoc | null>(null);
|
||||
const [docDirty, setDocDirty] = useState(false);
|
||||
@@ -61,7 +62,8 @@ export function useWorkspace(courseId: string) {
|
||||
const { status, data } = await api("GET", coursePath(courseId));
|
||||
if (status !== 200) { setError({ status, error: data?.error }); return; }
|
||||
setError(null);
|
||||
if (data.readOnly) { setReadOnly({ message: data.message, title: data.course.title, root: data.course.root }); return; }
|
||||
if (data.readOnly) { setReadOnly({ message: data.message, title: data.course.title, root: data.course.root, imported: !!data.imported, convertible: !!data.convertible }); return; }
|
||||
setReadOnly(null);
|
||||
setState(data);
|
||||
setDocState(data.manifestRaw ? clone(data.manifestRaw) : null);
|
||||
setDocDirty(false);
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import { resolve } from "node:path";
|
||||
import type { AppConfig, AuthUser, ServerContext } from "@stc/contracts";
|
||||
import { createApp, resolveConfig } from "@stc/core";
|
||||
import type { AppConfig, AuthUser, JobQueue, ServerContext } from "@stc/contracts";
|
||||
import { createApp, createJobQueue, resolveConfig } from "@stc/core";
|
||||
import { createCourseRepository } from "@stc/course-format";
|
||||
import { connect, migrate } from "@stc/db";
|
||||
import { studioFeature } from "../src/server/feature";
|
||||
import { studioFeature, type StudioFeatureOptions } from "../src/server/feature";
|
||||
import { studioCourseDirs } from "../src/server/paths";
|
||||
import { tempDir } from "./helpers";
|
||||
|
||||
@@ -15,7 +15,7 @@ export const users: Record<string, AuthUser> = {
|
||||
};
|
||||
|
||||
/** A server context like main.ts builds, with header-selected users (`x-user: ada`). */
|
||||
export async function setupApp(extraCourseDirs: string[] = [], dbUrl = "sqlite://:memory:", prepare?: (dataDir: string) => void, configOverrides: Partial<AppConfig> = {}) {
|
||||
export async function setupApp(extraCourseDirs: string[] = [], dbUrl = "sqlite://:memory:", prepare?: (dataDir: string) => void, configOverrides: Partial<AppConfig> = {}, featureOpts: StudioFeatureOptions = {}) {
|
||||
const dataDir = tempDir();
|
||||
prepare?.(dataDir);
|
||||
let dropSchema = async () => {};
|
||||
@@ -34,10 +34,14 @@ export async function setupApp(extraCourseDirs: string[] = [], dbUrl = "sqlite:/
|
||||
const config = resolveConfig({ port: 0, databaseUrl: "sqlite://:memory:", dataDir, courseDirs: extraCourseDirs, secret: "t", publicUrl: "http://x", ...configOverrides });
|
||||
const courses = await createCourseRepository(studioCourseDirs(config));
|
||||
const ctx = {
|
||||
config, db, courses, jobs: null as never,
|
||||
config, db, courses, jobs: null as unknown as JobQueue,
|
||||
auth: { userFromRequest: async (req: Request) => users[req.headers.get("x-user") ?? ""] ?? null },
|
||||
} satisfies ServerContext;
|
||||
const feature = studioFeature(ctx, { watch: false });
|
||||
const feature = studioFeature(ctx, { watch: false, ...featureOpts });
|
||||
// the real queue, as composeServer builds it from the features' jobs
|
||||
const queue = createJobQueue(db, feature.jobs ?? {}, { pollMs: 20 });
|
||||
ctx.jobs = queue;
|
||||
queue.start();
|
||||
const app = createApp(ctx, [feature]);
|
||||
const call = async (as: string | null, method: string, path: string, body?: unknown) => {
|
||||
const headers: Record<string, string> = { origin: "http://x" };
|
||||
@@ -51,5 +55,14 @@ export async function setupApp(extraCourseDirs: string[] = [], dbUrl = "sqlite:/
|
||||
try { json = JSON.parse(text); } catch { json = text; }
|
||||
return { status: r.status, json };
|
||||
};
|
||||
return { ctx, db, dataDir, feature, app, call, stop: async () => { feature.stop(); await db.close(); await dropSchema(); } };
|
||||
/** Polls a job until it is done or failed. */
|
||||
const waitJob = async (id: string) => {
|
||||
for (let i = 0; i < 500; i++) {
|
||||
const j = await queue.get(id);
|
||||
if (j && (j.status === "done" || j.status === "failed")) return j;
|
||||
await Bun.sleep(20);
|
||||
}
|
||||
throw new Error(`job ${id} did not finish`);
|
||||
};
|
||||
return { ctx, db, dataDir, feature, app, call, waitJob, stop: async () => { feature.stop(); await queue.stop(); await db.close(); await dropSchema(); } };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
import { afterEach, describe, expect, test } from "bun:test";
|
||||
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { studioFeature } from "../src/server/feature";
|
||||
import { SimulatedCrash } from "../src/server/convert";
|
||||
import { createStudioStore } from "../src/server/store";
|
||||
import { REPO, setupApp } from "./app-helpers";
|
||||
import { makeImportedCourse, VIDEO, videoBytes } from "./import-fixture";
|
||||
|
||||
let env: Awaited<ReturnType<typeof setupApp>>;
|
||||
afterEach(() => env?.stop());
|
||||
|
||||
const PG = process.env.STC_TEST_PG_URL;
|
||||
const dbs: [string, string][] = [["sqlite", "sqlite://:memory:"], ...(PG ? [["postgres", PG] as [string, string]] : [])];
|
||||
|
||||
async function convert(as = "owner", id = "imp") {
|
||||
const r = await env.call(as, "POST", `/api/studio/courses/${id}/convert`);
|
||||
expect(r.status).toBe(202);
|
||||
const job = await env.waitJob(r.json.jobId);
|
||||
expect(job.error).toBeNull();
|
||||
expect(job.status).toBe("done");
|
||||
return job;
|
||||
}
|
||||
|
||||
describe.each(dbs)("convert routes [%s]", (_name, dbUrl) => {
|
||||
test("the owner converts; the course is then served from the published worktree, still restricted, ACL kept", async () => {
|
||||
env = await setupApp([], dbUrl, (d) => makeImportedCourse(d));
|
||||
const { call, ctx, db } = env;
|
||||
await db.sql`INSERT INTO course_acl (course_id, principal_type, principal_id) VALUES ('imp', 'user', 'u-lea')`;
|
||||
expect(ctx.courses.get("imp")?.root).toBe(join(env.dataDir, "courses/imp"));
|
||||
|
||||
const ro = (await call("owner", "GET", "/api/studio/courses/imp")).json;
|
||||
expect(ro).toMatchObject({ readOnly: true, convertible: true });
|
||||
expect((await call("ada", "GET", "/api/studio/courses/imp")).json).toMatchObject({ readOnly: true, convertible: false });
|
||||
expect((await call("owner", "GET", "/api/studio/courses")).json.courses).toContainEqual(expect.objectContaining({ id: "imp", managed: false, imported: true, convertible: true }));
|
||||
|
||||
const job = await convert();
|
||||
expect(job.result).toMatchObject({ courseId: "imp", media: 3, resumed: false });
|
||||
const c = ctx.courses.get("imp")!;
|
||||
expect(c.root).toBe(join(env.dataDir, "worktrees/imp/published"));
|
||||
expect(c.status).toBe("ok");
|
||||
expect(c.manifest?.visibility).toBe("restricted");
|
||||
expect(ctx.courses.list().filter((x) => x.id === "imp")).toHaveLength(1);
|
||||
expect(await db.sql`SELECT principal_id FROM course_acl WHERE course_id = 'imp'`).toEqual([{ principal_id: "u-lea" }]);
|
||||
|
||||
// media through core's route, Range included
|
||||
const r = await env.app.fetch(new Request(`http://x/api/media/imp/${VIDEO.split("/").map(encodeURIComponent).join("/")}`, { headers: { "x-user": "lea", range: "bytes=10-19" } }));
|
||||
expect(r.status).toBe(206);
|
||||
expect(new Uint8Array(await r.arrayBuffer())).toEqual(videoBytes(4096, 7).slice(10, 20));
|
||||
// anonymous users still see nothing
|
||||
expect((await env.app.fetch(new Request(`http://x/api/media/imp/${encodeURI(VIDEO)}`))).status).toBe(404);
|
||||
|
||||
// the studio now edits it
|
||||
const st = (await call("owner", "GET", "/api/studio/courses/imp")).json;
|
||||
expect(st.readOnly).toBe(false);
|
||||
expect(st.files).toContain(VIDEO);
|
||||
expect((await call("owner", "GET", "/api/studio/courses")).json.courses).toContainEqual(expect.objectContaining({ id: "imp", managed: true }));
|
||||
});
|
||||
|
||||
test("an author edits and publishes a lesson title but cannot make the course public; the owner can", async () => {
|
||||
env = await setupApp([], dbUrl, (d) => makeImportedCourse(d));
|
||||
const { call, ctx } = env;
|
||||
await convert();
|
||||
let st = (await call("ada", "GET", "/api/studio/courses/imp")).json;
|
||||
const m = structuredClone(st.manifestRaw);
|
||||
m.modules[0].lessons[0].title = "Welcome aboard";
|
||||
expect((await call("ada", "PUT", "/api/studio/courses/imp/files", { message: "Rename", manifest: m, manifestBaseHash: st.manifestHash })).status).toBe(200);
|
||||
expect((await call("ada", "POST", "/api/studio/courses/imp/publish")).json).toMatchObject({ ok: true, published: true });
|
||||
expect(ctx.courses.get("imp")?.lessonIndex.get("l1")?.lesson.title).toBe("Welcome aboard");
|
||||
|
||||
st = (await call("ada", "GET", "/api/studio/courses/imp")).json;
|
||||
const pub = structuredClone(st.manifestRaw);
|
||||
pub.visibility = "public";
|
||||
expect((await call("ada", "PUT", "/api/studio/courses/imp/files", { message: "Go public", manifest: pub, manifestBaseHash: st.manifestHash })).status).toBe(403);
|
||||
st = (await call("owner", "GET", "/api/studio/courses/imp")).json;
|
||||
expect((await call("owner", "PUT", "/api/studio/courses/imp/files", { message: "Go public", manifest: { ...st.manifestRaw, visibility: "public" }, manifestBaseHash: st.manifestHash })).status).toBe(200);
|
||||
expect((await call("owner", "POST", "/api/studio/courses/imp/publish")).status).toBe(200);
|
||||
expect(ctx.courses.get("imp")?.manifest?.visibility).toBe("public");
|
||||
});
|
||||
});
|
||||
|
||||
describe.each(dbs)("conversion and importer jobs [%s]", (_name, dbUrl) => {
|
||||
test("409 while an import job for the course is queued; the job itself re-checks", async () => {
|
||||
env = await setupApp([], dbUrl, (d) => makeImportedCourse(d));
|
||||
// no importer feature in this app: its jobs stay queued
|
||||
await env.ctx.jobs.enqueue("import.code-repo", { kind: "code-repo", path: "/x", options: { courseId: "imp" } });
|
||||
const blocker = await env.ctx.jobs.list("import.code-repo");
|
||||
const r = await env.call("owner", "POST", "/api/studio/courses/imp/convert");
|
||||
expect(r.status).toBe(409);
|
||||
expect(r.json.error).toContain("an import job for this course is still running");
|
||||
expect(r.json.error).toContain(`import.code-repo ${blocker[0]!.id}`);
|
||||
// a convert job that was queued anyway (before the import was) fails cleanly and changes nothing
|
||||
const id = await env.ctx.jobs.enqueue("studio.convert-import", { courseId: "imp", actor: { id: "u-own", name: "O", email: "o@x" } });
|
||||
const j = await env.waitJob(id);
|
||||
expect(j.status).toBe("failed");
|
||||
expect(j.error).toContain("an import job for this course is still running");
|
||||
expect(existsSync(join(env.dataDir, "courses/imp/stcourse.yaml"))).toBe(true);
|
||||
expect(existsSync(join(env.dataDir, "studio/repos/imp.git"))).toBe(false);
|
||||
// an import job for another course does not block
|
||||
await env.db.sql`DELETE FROM jobs WHERE kind = 'import.code-repo'`;
|
||||
await env.ctx.jobs.enqueue("import.transcribe", { courseId: "other" });
|
||||
expect((await env.call("owner", "POST", "/api/studio/courses/imp/convert")).status).toBe(202);
|
||||
});
|
||||
|
||||
test("server start recovers a conversion that crashed mid-move; learners see the course once", async () => {
|
||||
env = await setupApp([], dbUrl, (d) => makeImportedCourse(d));
|
||||
const crashing = createStudioStore({ dataDir: env.dataDir, testHooks: { crashAt: "staging-copied", forceExdev: true } });
|
||||
await expect(crashing.convertImported("imp", { name: "O", email: "o@x" })).rejects.toBeInstanceOf(SimulatedCrash);
|
||||
await env.ctx.courses.reload();
|
||||
expect(env.ctx.courses.list().filter((c) => c.id === "imp").length).toBeLessThanOrEqual(1);
|
||||
// a restart: a new studio feature recovers before serving
|
||||
const restarted = studioFeature(env.ctx, { watch: false });
|
||||
await restarted.ready;
|
||||
restarted.stop();
|
||||
const loaded = env.ctx.courses.list().filter((c) => c.id === "imp");
|
||||
expect(loaded).toHaveLength(1);
|
||||
expect(loaded[0]!.root).toBe(join(env.dataDir, "worktrees/imp/published"));
|
||||
expect(existsSync(join(env.dataDir, "courses/imp"))).toBe(false);
|
||||
const r = await env.app.fetch(new Request(`http://x/api/media/imp/${VIDEO.split("/").map(encodeURIComponent).join("/")}`, { headers: { "x-user": "owner", range: "bytes=0-9" } }));
|
||||
expect(r.status).toBe(206);
|
||||
// the requeued job (or the owner) retrying reports success
|
||||
expect((await env.call("owner", "POST", "/api/studio/courses/imp/convert")).status).toBe(409);
|
||||
expect(await env.feature.store.convertImported("imp", { name: "O", email: "o@x" })).toMatchObject({ resumed: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe.each(dbs)("a conversion that fails without a restart [%s]", (_name, dbUrl) => {
|
||||
const media = (as: string) => env.app.fetch(new Request(`http://x/api/media/imp/${VIDEO.split("/").map(encodeURIComponent).join("/")}`, { headers: { "x-user": as, range: "bytes=10-19" } }));
|
||||
const learnerView = async (where: "import" | "studio") => {
|
||||
const all = env.ctx.courses.list().filter((c) => c.id === "imp");
|
||||
expect(all).toHaveLength(1);
|
||||
expect(all[0]!.status).toBe("ok");
|
||||
expect(all[0]!.root).toBe(join(env.dataDir, where === "import" ? "courses/imp" : "worktrees/imp/published"));
|
||||
const r = await media("lea");
|
||||
expect(r.status).toBe(206);
|
||||
expect(new Uint8Array(await r.arrayBuffer())).toEqual(videoBytes(4096, 7).slice(10, 20));
|
||||
};
|
||||
|
||||
const cases: [string, { forceExdev?: boolean; failAt?: string; corruptStaging?: boolean }, "import" | "studio", RegExp][] = [
|
||||
["the cross-filesystem copy fails (disk full)", { forceExdev: true, failAt: "staging-copied" }, "studio", /injected failure at staging-copied/],
|
||||
["the copied media fail their checksum", { forceExdev: true, corruptStaging: true }, "studio", /do not match/],
|
||||
["the checkout fails before the commit point", { failAt: "worktree" }, "import", /injected failure at worktree/],
|
||||
];
|
||||
test.each(cases)("%s: learners keep a working course and the owner's retry completes", async (_c, hooks, after, error) => {
|
||||
env = await setupApp([], dbUrl, (d) => makeImportedCourse(d), {}, { storeTestHooks: hooks });
|
||||
await env.db.sql`INSERT INTO course_acl (course_id, principal_type, principal_id) VALUES ('imp', 'user', 'u-lea')`;
|
||||
await learnerView("import");
|
||||
|
||||
const r = await env.call("owner", "POST", "/api/studio/courses/imp/convert");
|
||||
expect(r.status).toBe(202);
|
||||
const j = await env.waitJob(r.json.jobId);
|
||||
expect(j.status).toBe("failed");
|
||||
expect(j.error).toMatch(error);
|
||||
// no restart: learners already get the intact import or the finished studio course
|
||||
await learnerView(after);
|
||||
|
||||
if (after === "studio") {
|
||||
const st = (await env.call("owner", "GET", "/api/studio/courses/imp")).json;
|
||||
expect(st).toMatchObject({ readOnly: false, conversionIncomplete: true, canRetryConversion: true });
|
||||
expect((await env.call("ada", "GET", "/api/studio/courses/imp")).json).toMatchObject({ conversionIncomplete: true, canRetryConversion: false });
|
||||
} else {
|
||||
expect((await env.call("owner", "GET", "/api/studio/courses/imp")).json).toMatchObject({ readOnly: true, convertible: true });
|
||||
}
|
||||
|
||||
// the cause goes away (space freed, disk fixed); the owner retries through the same route
|
||||
delete hooks.failAt;
|
||||
delete hooks.corruptStaging;
|
||||
expect((await env.call("ada", "POST", "/api/studio/courses/imp/convert")).status).toBe(403);
|
||||
const again = await env.call("owner", "POST", "/api/studio/courses/imp/convert");
|
||||
expect(again.status).toBe(202);
|
||||
const j2 = await env.waitJob(again.json.jobId);
|
||||
expect(j2.error).toBeNull();
|
||||
expect(j2.status).toBe("done");
|
||||
await learnerView("studio");
|
||||
expect((await env.call("owner", "GET", "/api/studio/courses/imp")).json.conversionIncomplete).toBe(false);
|
||||
expect(readdirSync(join(env.dataDir, "courses"))).toEqual([]);
|
||||
expect(readdirSync(join(env.dataDir, "media"))).toEqual(["imp"]);
|
||||
expect((await env.call("owner", "POST", "/api/studio/courses/imp/convert")).status).toBe(409);
|
||||
});
|
||||
});
|
||||
|
||||
describe("convert route refusals", () => {
|
||||
test("only the owner; cross-site requests are refused; nothing changes", async () => {
|
||||
env = await setupApp([], undefined, (d) => makeImportedCourse(d));
|
||||
for (const who of ["ada", "lea"]) expect((await env.call(who, "POST", "/api/studio/courses/imp/convert")).status).toBe(403);
|
||||
expect((await env.call(null, "POST", "/api/studio/courses/imp/convert")).status).toBe(401);
|
||||
const forged = await env.app.fetch(new Request("http://x/api/studio/courses/imp/convert", { method: "POST", headers: { "x-user": "owner", origin: "http://evil.example" } }));
|
||||
expect(forged.status).toBe(403);
|
||||
const noOrigin = await env.app.fetch(new Request("http://x/api/studio/courses/imp/convert", { method: "POST", headers: { "x-user": "owner" } }));
|
||||
expect(noOrigin.status).toBe(403);
|
||||
expect(existsSync(join(env.dataDir, "courses/imp/stcourse.yaml"))).toBe(true);
|
||||
expect(existsSync(join(env.dataDir, "studio/repos/imp.git"))).toBe(false);
|
||||
expect(await env.ctx.jobs.list("studio.convert-import")).toEqual([]);
|
||||
});
|
||||
|
||||
test("overlay, studio and unknown courses; a second request while one is pending", async () => {
|
||||
env = await setupApp([resolve(REPO, "fixtures/courses")], undefined, (d) => makeImportedCourse(d));
|
||||
expect((await env.call("owner", "POST", "/api/studio/courses/sample/convert")).status).toBe(409);
|
||||
expect((await env.call("owner", "POST", "/api/studio/courses/nope/convert")).status).toBe(404);
|
||||
await env.call("owner", "POST", "/api/studio/courses", { id: "c1", title: "C" });
|
||||
expect((await env.call("owner", "POST", "/api/studio/courses/c1/convert")).status).toBe(409);
|
||||
|
||||
await env.ctx.jobs.enqueue("studio.convert-import", { courseId: "imp", actor: { id: "u-own", name: "O", email: "o@x" } });
|
||||
// whether the first job is still queued, running or already done, a second request is refused
|
||||
const r = await env.call("owner", "POST", "/api/studio/courses/imp/convert");
|
||||
expect(r.status).toBe(409);
|
||||
});
|
||||
|
||||
test("job status is owner-only and limited to studio jobs", async () => {
|
||||
env = await setupApp([], undefined, (d) => makeImportedCourse(d));
|
||||
const { jobId } = (await env.call("owner", "POST", "/api/studio/courses/imp/convert")).json;
|
||||
expect((await env.call("ada", "GET", `/api/studio/jobs/${jobId}`)).status).toBe(403);
|
||||
await env.waitJob(jobId);
|
||||
const j = (await env.call("owner", "GET", `/api/studio/jobs/${jobId}`)).json;
|
||||
expect(j).toMatchObject({ id: jobId, status: "done", progress: 1, error: null });
|
||||
const other = await env.ctx.jobs.enqueue("import.transcribe", { courseId: "x" });
|
||||
expect((await env.call("owner", "GET", `/api/studio/jobs/${other}`)).status).toBe(404);
|
||||
});
|
||||
|
||||
test("a failed conversion reports the error and leaves the import as it was", async () => {
|
||||
env = await setupApp([], undefined, (d) => makeImportedCourse(d));
|
||||
const manifest = join(env.dataDir, "courses/imp/stcourse.yaml");
|
||||
// the route only checks that this is an imported course; the job validates the manifest fully
|
||||
await Bun.write(manifest, readFileSync(manifest, "utf8").replace("id: imp", "id: other"));
|
||||
const { jobId } = (await env.call("owner", "POST", "/api/studio/courses/imp/convert")).json;
|
||||
const j = await env.waitJob(jobId);
|
||||
expect(j.status).toBe("failed");
|
||||
expect(j.error).toContain('"other"');
|
||||
expect(existsSync(join(env.dataDir, "studio/repos/imp.git"))).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import { renderToString } from "react-dom/server";
|
||||
import { ConvertPanelView, IncompleteConversionView, ReadOnlyView } from "../src/ui/ConvertPanel";
|
||||
|
||||
const noop = () => {};
|
||||
|
||||
test("the owner sees the action and what it does", () => {
|
||||
const html = renderToString(<ConvertPanelView phase={{ kind: "idle" }} onConvert={noop} />);
|
||||
expect(html).toContain("Make editable in the studio");
|
||||
expect(html).toContain("same id");
|
||||
expect(html).toContain("progress and notes");
|
||||
expect(html).toContain("outside git");
|
||||
expect(html).toContain("<button");
|
||||
});
|
||||
|
||||
test("progress and failure states", () => {
|
||||
const running = renderToString(<ConvertPanelView phase={{ kind: "running", progress: 0.4, message: "Checksumming media/a.mp4" }} onConvert={noop} />);
|
||||
expect(running).toContain('role="progressbar"');
|
||||
expect(running).toContain('aria-valuenow="40"');
|
||||
expect(running).toContain("Checksumming media/a.mp4");
|
||||
expect(running).toContain("disabled");
|
||||
const failed = renderToString(<ConvertPanelView phase={{ kind: "failed", error: "the imported manifest is invalid" }} onConvert={noop} />);
|
||||
expect(failed).toContain('role="alert"');
|
||||
expect(failed).toContain("the imported manifest is invalid");
|
||||
expect(failed).toContain("Try again");
|
||||
});
|
||||
|
||||
test("read-only view: imported for the owner offers the action; for others explains who can", () => {
|
||||
const owner = renderToString(<ReadOnlyView courseId="imp" info={{ title: "Imp", message: "Imported course", root: "/d/courses/imp", imported: true, convertible: true }} onConverted={noop} />);
|
||||
expect(owner).toContain("Make editable in the studio");
|
||||
expect(owner).not.toContain("/d/courses/imp");
|
||||
const author = renderToString(<ReadOnlyView courseId="imp" info={{ title: "Imp", message: "Imported course", root: "/d/courses/imp", imported: true, convertible: false }} onConverted={noop} />);
|
||||
expect(author).not.toContain("<button");
|
||||
expect(author).toContain("Only the owner");
|
||||
const outside = renderToString(<ReadOnlyView courseId="o" info={{ title: "O", message: "This course is managed outside STCourses", root: "/repos/o", imported: false, convertible: false }} onConverted={noop} />);
|
||||
expect(outside).toContain("/repos/o");
|
||||
expect(outside).not.toContain("<button");
|
||||
});
|
||||
|
||||
test("an incomplete conversion: the owner can retry, others are told who can", () => {
|
||||
const owner = renderToString(<IncompleteConversionView phase={{ kind: "idle" }} canRetry onRetry={noop} />);
|
||||
expect(owner).toContain("Conversion incomplete");
|
||||
expect(owner).toContain(">Retry</button>");
|
||||
const author = renderToString(<IncompleteConversionView phase={{ kind: "idle" }} canRetry={false} onRetry={noop} />);
|
||||
expect(author).toContain("Conversion incomplete");
|
||||
expect(author).not.toContain("<button");
|
||||
expect(author).toContain("Ask the owner");
|
||||
const failed = renderToString(<IncompleteConversionView phase={{ kind: "failed", error: "disk full" }} canRetry onRetry={noop} />);
|
||||
expect(failed).toContain('role="alert"');
|
||||
expect(failed).toContain("disk full");
|
||||
});
|
||||
@@ -0,0 +1,299 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { loadAll, loadCourse } from "@stc/course-format";
|
||||
import { createStudioStore } from "../src/server/store";
|
||||
import { SimulatedCrash } from "../src/server/convert";
|
||||
import { gitignoreLine, MEDIA_REFS, parseMediaRefs } from "../src/server/media-refs";
|
||||
import { BIG_PDF, makeImportedCourse, VIDEO, VIDEO2, videoBytes } from "./import-fixture";
|
||||
import { ada, extGit, newStore, tempDir } from "./helpers";
|
||||
|
||||
const ino = (p: string) => statSync(p).ino;
|
||||
const tracked = async (dir: string) => (await extGit(dir, "-c", "core.quotepath=false", "ls-files")).split("\n").filter(Boolean).sort();
|
||||
|
||||
describe("convertImported", () => {
|
||||
test("builds the repo and published worktree; large media stay out of git as hard links", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
const src = makeImportedCourse(dataDir, { hostile: true });
|
||||
const srcVideoIno = ino(join(src, VIDEO));
|
||||
const r = await store.convertImported("imp", ada);
|
||||
expect(r.media).toBe(3);
|
||||
expect(store.isManaged("imp")).toBe(true);
|
||||
expect(existsSync(join(dataDir, "studio/repos/imp.git/HEAD"))).toBe(true);
|
||||
expect(existsSync(src)).toBe(false);
|
||||
const mediaRoot = join(dataDir, "media/imp");
|
||||
expect(existsSync(join(mediaRoot, "stcourse.yaml"))).toBe(true);
|
||||
|
||||
const pub = store.paths("imp").published;
|
||||
expect(await tracked(pub)).toEqual([
|
||||
MEDIA_REFS, "code/S01 - Basics/.gitignore", "code/S01 - Basics/README.md", "media/cover.jpg", "stcourse.yaml", "transcripts/l1.md",
|
||||
]);
|
||||
// the symlink was skipped, never followed; .git contents never copied
|
||||
expect(existsSync(join(pub, "code/leak.txt"))).toBe(false);
|
||||
expect(existsSync(join(pub, "code/.git"))).toBe(false);
|
||||
expect(await extGit(pub, "status", "--porcelain", "--untracked-files=all")).toBe("");
|
||||
// the exclusions live in the repo's info/exclude, which authors and remotes cannot change
|
||||
const exclude = readFileSync(join(dataDir, "studio/repos/imp.git/info/exclude"), "utf8");
|
||||
for (const p of [VIDEO, VIDEO2, BIG_PDF]) expect(exclude).toContain(gitignoreLine(p));
|
||||
|
||||
const refs = parseMediaRefs(readFileSync(join(pub, MEDIA_REFS), "utf8"))!;
|
||||
expect(refs.map((x) => x.path).sort()).toEqual([BIG_PDF, VIDEO2, VIDEO].sort());
|
||||
const v = refs.find((x) => x.path === VIDEO)!;
|
||||
expect(v.size).toBe(4096);
|
||||
expect(v.sha256).toBe(new Bun.CryptoHasher("sha256").update(videoBytes(4096, 7)).digest("hex"));
|
||||
for (const p of [VIDEO, VIDEO2, BIG_PDF]) expect(ino(join(pub, p))).toBe(ino(join(mediaRoot, p)));
|
||||
expect(ino(join(mediaRoot, VIDEO))).toBe(srcVideoIno);
|
||||
|
||||
// the course loads from the published worktree exactly as before
|
||||
const c = await loadCourse(join(pub, "stcourse.yaml"));
|
||||
expect(c.status).toBe("ok");
|
||||
expect(c.manifest?.visibility).toBe("restricted");
|
||||
expect(c.manifest?.source?.kind).toBe("video-folder");
|
||||
// the commit is attributed to the converting owner
|
||||
expect(await extGit(pub, "log", "-1", "--format=%an <%ae>")).toBe(`${ada.name} <${ada.email}>`);
|
||||
});
|
||||
|
||||
test("a draft gets the media; listFiles shows them; the media and reference file are read-only", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
makeImportedCourse(dataDir);
|
||||
await store.convertImported("imp", ada);
|
||||
const draft = await store.ensureDraft("imp", ada.id);
|
||||
expect(ino(join(draft, VIDEO))).toBe(ino(join(dataDir, "media/imp", VIDEO)));
|
||||
const files = await store.listFiles("imp", ada.id);
|
||||
expect(files).toContain(VIDEO);
|
||||
expect(files).toContain(BIG_PDF);
|
||||
for (const path of [VIDEO, MEDIA_REFS]) {
|
||||
await expect(store.saveFiles("imp", ada, [{ path, content: "x" }], "x")).rejects.toMatchObject({ status: 409 });
|
||||
await expect(store.saveFiles("imp", ada, [{ path, delete: true }], "x")).rejects.toMatchObject({ status: 409 });
|
||||
}
|
||||
expect(readFileSync(join(dataDir, "media/imp", VIDEO))).toEqual(Buffer.from(videoBytes(4096, 7)));
|
||||
});
|
||||
|
||||
test("a studio write to a hard-linked file breaks the link instead of writing through it", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
makeImportedCourse(dataDir);
|
||||
await store.convertImported("imp", ada);
|
||||
const draft = await store.ensureDraft("imp", ada.id);
|
||||
// a planted hard link at an ordinary (non-referenced) path
|
||||
const outside = join(dataDir, "media/imp/transcripts/l1.md");
|
||||
rmSync(join(draft, "transcripts/l1.md"));
|
||||
const { linkSync } = await import("node:fs");
|
||||
linkSync(outside, join(draft, "transcripts/l1.md"));
|
||||
expect((await store.saveFiles("imp", ada, [{ path: "transcripts/l1.md", content: "# changed\n" }], "edit")).ok).toBe(true);
|
||||
expect(readFileSync(outside, "utf8")).toBe("# Intro\n\n[0:00] Hello.\n");
|
||||
expect(readFileSync(join(draft, "transcripts/l1.md"), "utf8")).toBe("# changed\n");
|
||||
});
|
||||
|
||||
test("edit and publish: the published worktree keeps its media and stays clean", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
makeImportedCourse(dataDir);
|
||||
await store.convertImported("imp", ada);
|
||||
const m = (await store.readFile("imp", ada.id, "stcourse.yaml"))!;
|
||||
const next = m.content.replace("title: Intro", "title: Welcome aboard");
|
||||
expect((await store.saveFiles("imp", ada, [{ path: "stcourse.yaml", content: next, baseHash: m.hash }], "Rename lesson")).ok).toBe(true);
|
||||
const r = await store.publish("imp", ada);
|
||||
expect(r).toMatchObject({ ok: true, published: true });
|
||||
const pub = store.paths("imp").published;
|
||||
expect(readFileSync(join(pub, "stcourse.yaml"), "utf8")).toContain("title: Welcome aboard");
|
||||
expect(ino(join(pub, VIDEO))).toBe(ino(join(dataDir, "media/imp", VIDEO)));
|
||||
expect(await extGit(pub, "status", "--porcelain", "--untracked-files=all")).toBe("");
|
||||
});
|
||||
|
||||
test("a non-owner cannot publish the converted course as public; the owner can", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
makeImportedCourse(dataDir);
|
||||
await store.convertImported("imp", ada);
|
||||
const m = (await store.readFile("imp", ada.id, "stcourse.yaml"))!;
|
||||
await store.saveFiles("imp", ada, [{ path: "stcourse.yaml", content: m.content.replace("visibility: restricted", "visibility: public"), baseHash: m.hash }], "go public");
|
||||
expect(await store.publish("imp", ada)).toMatchObject({ ok: false, reason: "forbidden" });
|
||||
expect(await store.publish("imp", { ...ada, owner: true })).toMatchObject({ ok: true, published: true });
|
||||
});
|
||||
|
||||
describe("refusals leave the import untouched and no repo behind", () => {
|
||||
const cases: [string, (dataDir: string) => void, number][] = [
|
||||
["no such import", () => {}, 404],
|
||||
["an authored course", (d) => makeImportedCourse(d, { kind: "authored" }), 400],
|
||||
["a manifest id mismatch", (d) => makeImportedCourse(d, { manifestId: "other" }), 400],
|
||||
["a root: field", (d) => makeImportedCourse(d, { root: "." }), 400],
|
||||
["a media store already there", (d) => { makeImportedCourse(d); mkdirSync(join(d, "media/imp"), { recursive: true }); }, 409],
|
||||
];
|
||||
test.each(cases)("%s", async (_name, prepare, status) => {
|
||||
const { dataDir, store } = await newStore();
|
||||
prepare(dataDir);
|
||||
const before = existsSync(join(dataDir, "courses/imp/stcourse.yaml")) ? readFileSync(join(dataDir, "courses/imp/stcourse.yaml"), "utf8") : null;
|
||||
await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status });
|
||||
expect(existsSync(join(dataDir, "studio/repos/imp.git"))).toBe(false);
|
||||
expect(existsSync(join(dataDir, "worktrees/imp"))).toBe(false);
|
||||
if (before !== null) expect(readFileSync(join(dataDir, "courses/imp/stcourse.yaml"), "utf8")).toBe(before);
|
||||
});
|
||||
|
||||
test("a studio course with the same id", async () => {
|
||||
const { store } = await newStore();
|
||||
await store.createCourse({ id: "imp", title: "Studio" }, ada);
|
||||
await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status: 409 });
|
||||
expect(store.isManaged("imp")).toBe(true);
|
||||
});
|
||||
|
||||
test("a large file whose name has a line break", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
const src = makeImportedCourse(dataDir);
|
||||
writeFileSync(join(src, "media/bad\nname.mp4"), "x");
|
||||
await expect(store.convertImported("imp", ada)).rejects.toMatchObject({ status: 400 });
|
||||
expect(existsSync(join(dataDir, "studio/repos/imp.git"))).toBe(false);
|
||||
expect(existsSync(join(src, "stcourse.yaml"))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("restart safety", () => {
|
||||
test("a leftover partial repo (no published worktree) is rebuilt", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
makeImportedCourse(dataDir);
|
||||
mkdirSync(join(dataDir, "studio/repos/imp.git"), { recursive: true });
|
||||
writeFileSync(join(dataDir, "studio/repos/imp.git/junk"), "x");
|
||||
await store.convertImported("imp", ada);
|
||||
expect(store.isManaged("imp")).toBe(true);
|
||||
expect(existsSync(join(dataDir, "studio/repos/imp.git/junk"))).toBe(false);
|
||||
});
|
||||
|
||||
test("a managed course whose import folder is still there only finishes the move", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
makeImportedCourse(dataDir);
|
||||
await store.convertImported("imp", ada);
|
||||
const head = await store.publishedHead("imp");
|
||||
// simulate a crash between the checkout and the move
|
||||
const { renameSync } = await import("node:fs");
|
||||
renameSync(join(dataDir, "media/imp"), join(dataDir, "courses/imp"));
|
||||
const r = await store.convertImported("imp", ada);
|
||||
expect(r.resumed).toBe(true);
|
||||
expect(await store.publishedHead("imp")).toBe(head);
|
||||
expect(existsSync(join(dataDir, "courses/imp"))).toBe(false);
|
||||
expect(existsSync(join(dataDir, "media/imp", VIDEO))).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("an author's or a remote's .gitignore never blocks the course", () => {
|
||||
const editManifest = async (store: Awaited<ReturnType<typeof newStore>>["store"], title: string) => {
|
||||
const m = (await store.readFile("imp", ada.id, "stcourse.yaml"))!;
|
||||
const text = m.content.replace(/title: (Intro|Welcome \d+)\n/, `title: ${title}\n`);
|
||||
expect((await store.saveFiles("imp", ada, [{ path: "stcourse.yaml", content: text, baseHash: m.hash }], title)).ok).toBe(true);
|
||||
};
|
||||
|
||||
test("un-ignoring the media in .gitignore: publish keeps working, again and again", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
makeImportedCourse(dataDir);
|
||||
await store.convertImported("imp", ada);
|
||||
expect((await store.saveFiles("imp", ada, [{ path: ".gitignore", content: "!*.mp4\n!*.pdf\n!/media/**\n" }], "tidy .gitignore")).ok).toBe(true);
|
||||
expect(await store.publish("imp", ada)).toMatchObject({ ok: true, published: true });
|
||||
const pub = store.paths("imp").published;
|
||||
// git itself now sees the media as untracked; the studio does not count them
|
||||
expect(await extGit(pub, "-c", "core.quotepath=false", "status", "--porcelain")).toContain("media/");
|
||||
expect((await store.status("imp", ada.id)).publishedDirty).toEqual([]);
|
||||
await editManifest(store, "Welcome 2");
|
||||
expect(await store.publish("imp", ada)).toMatchObject({ ok: true, published: true });
|
||||
await editManifest(store, "Welcome 3");
|
||||
expect(await store.publish("imp", ada)).toMatchObject({ ok: true, published: true });
|
||||
expect(readFileSync(join(pub, "stcourse.yaml"), "utf8")).toContain("title: Welcome 3");
|
||||
});
|
||||
|
||||
test("a remote commit that rewrites .gitignore: Sync and publish keep working", async () => {
|
||||
const { dataDir, store } = await newStore({ allowLocalRemotes: true });
|
||||
makeImportedCourse(dataDir);
|
||||
await store.convertImported("imp", ada);
|
||||
const remote = join(tempDir(), "remote.git");
|
||||
await extGit(tempDir(), "init", "-q", "--bare", "-b", "main", remote);
|
||||
await store.setRemote("imp", remote);
|
||||
expect(await store.sync("imp", ada)).toMatchObject({ ok: true, pushed: true });
|
||||
const other = join(tempDir(), "other");
|
||||
await extGit(tempDir(), "clone", "-q", remote, other);
|
||||
writeFileSync(join(other, ".gitignore"), "!*\n");
|
||||
await extGit(other, "add", ".gitignore");
|
||||
await extGit(other, "commit", "-qm", "unignore everything");
|
||||
await extGit(other, "push", "-q", "origin", "main");
|
||||
expect(await store.sync("imp", ada)).toMatchObject({ ok: true, pulled: true });
|
||||
expect(await store.updateDraft("imp", ada)).toMatchObject({ ok: true });
|
||||
await editManifest(store, "Welcome 2");
|
||||
expect(await store.publish("imp", ada)).toMatchObject({ ok: true, published: true });
|
||||
expect(await store.sync("imp", ada)).toMatchObject({ ok: true, pushed: true });
|
||||
});
|
||||
|
||||
test("a remote commit that puts a file over referenced media is refused at Sync; the media stay", async () => {
|
||||
const { dataDir, store } = await newStore({ allowLocalRemotes: true });
|
||||
makeImportedCourse(dataDir);
|
||||
await store.convertImported("imp", ada);
|
||||
const remote = join(tempDir(), "remote.git");
|
||||
await extGit(tempDir(), "init", "-q", "--bare", "-b", "main", remote);
|
||||
await store.setRemote("imp", remote);
|
||||
await store.sync("imp", ada);
|
||||
const other = join(tempDir(), "other");
|
||||
await extGit(tempDir(), "clone", "-q", remote, other);
|
||||
mkdirSync(join(other, "media"), { recursive: true });
|
||||
writeFileSync(join(other, VIDEO), "not the video");
|
||||
await extGit(other, "add", "-f", VIDEO);
|
||||
await extGit(other, "commit", "-qm", "replace media");
|
||||
await extGit(other, "push", "-q", "origin", "main");
|
||||
expect(await store.sync("imp", ada)).toMatchObject({ ok: false, reason: "forbidden" });
|
||||
expect(ino(join(store.paths("imp").published, VIDEO))).toBe(ino(join(dataDir, "media/imp", VIDEO)));
|
||||
});
|
||||
|
||||
test("referenced media are not read whole into the editor", async () => {
|
||||
const { dataDir, store } = await newStore();
|
||||
makeImportedCourse(dataDir);
|
||||
await store.convertImported("imp", ada);
|
||||
expect(await store.readFile("imp", ada.id, VIDEO)).toBeNull();
|
||||
expect(await store.readFile("imp", ada.id, "transcripts/l1.md")).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("crash safety: a retry after any crash point succeeds and the id is never loaded twice", () => {
|
||||
const loaded = async (dataDir: string) => (await loadAll([join(dataDir, "courses"), join(dataDir, "worktrees")])).filter((c) => c.id === "imp");
|
||||
const sameFs = ["committed", "hidden", "worktree", "before-move"];
|
||||
const crossFs = [...sameFs, "staging-copied", "staging-placed", "source-retired"];
|
||||
const cases: [string, boolean][] = [...sameFs.map((s) => [s, false] as [string, boolean]), ...crossFs.map((s) => [s, true] as [string, boolean])];
|
||||
|
||||
test.each(cases)("crash at %s (cross-filesystem: %p)", async (step, exdev) => {
|
||||
const dataDir = tempDir();
|
||||
makeImportedCourse(dataDir);
|
||||
const crashing = createStudioStore({ dataDir, testHooks: { crashAt: step, forceExdev: exdev } });
|
||||
await expect(crashing.convertImported("imp", ada)).rejects.toBeInstanceOf(SimulatedCrash);
|
||||
expect((await loaded(dataDir)).length).toBeLessThanOrEqual(1);
|
||||
|
||||
// the process restarts: recovery first, then the owner (or the requeued job) retries
|
||||
const fresh = createStudioStore({ dataDir, testHooks: { forceExdev: exdev } });
|
||||
await fresh.recoverConversions();
|
||||
expect((await loaded(dataDir)).length).toBeLessThanOrEqual(1);
|
||||
await fresh.convertImported("imp", ada);
|
||||
// and once more: a re-run of a finished conversion reports success
|
||||
expect(await fresh.convertImported("imp", ada)).toMatchObject({ resumed: true });
|
||||
|
||||
expect(fresh.isManaged("imp")).toBe(true);
|
||||
const all = await loaded(dataDir);
|
||||
expect(all).toHaveLength(1);
|
||||
expect(all[0]!.status).toBe("ok");
|
||||
expect(all[0]!.root).toBe(fresh.paths("imp").published);
|
||||
expect(readdirSync(join(dataDir, "courses"))).toEqual([]);
|
||||
expect(readdirSync(join(dataDir, "media"))).toEqual(["imp"]);
|
||||
const pub = fresh.paths("imp").published;
|
||||
for (const p of [VIDEO, VIDEO2, BIG_PDF]) {
|
||||
expect(statSync(join(pub, p)).size).toBe(statSync(join(dataDir, "media/imp", p)).size);
|
||||
if (!exdev) expect(ino(join(pub, p))).toBe(ino(join(dataDir, "media/imp", p)));
|
||||
}
|
||||
expect(readFileSync(join(dataDir, "media/imp", VIDEO))).toEqual(Buffer.from(videoBytes(4096, 7)));
|
||||
expect(await extGit(pub, "status", "--porcelain", "--untracked-files=all")).toBe("");
|
||||
});
|
||||
|
||||
test("a failed cross-filesystem copy leaves no staging dir and a retry finishes", async () => {
|
||||
const dataDir = tempDir();
|
||||
makeImportedCourse(dataDir);
|
||||
const store = createStudioStore({ dataDir, testHooks: { forceExdev: true } });
|
||||
// the media store's parent is a file: the copy fails in-process
|
||||
writeFileSync(join(dataDir, "media"), "not a dir");
|
||||
await expect(store.convertImported("imp", ada)).rejects.toThrow();
|
||||
expect((await loaded(dataDir)).length).toBeLessThanOrEqual(1);
|
||||
rmSync(join(dataDir, "media"));
|
||||
await store.convertImported("imp", ada);
|
||||
expect(readdirSync(join(dataDir, "media"))).toEqual(["imp"]);
|
||||
expect(readdirSync(join(dataDir, "courses"))).toEqual([]);
|
||||
expect(await loaded(dataDir)).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,86 @@
|
||||
// A synthetic imported course, laid out as the importer leaves it in <dataDir>/courses/<id>. Nothing here is real material.
|
||||
import { mkdirSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { LARGE_FILE_BYTES } from "../src/server/media-refs";
|
||||
import { tempDir } from "./helpers";
|
||||
|
||||
export const VIDEO = "media/01 Intro – Übersicht.mp4";
|
||||
export const VIDEO2 = "media/[2] Next step.mp4";
|
||||
export const BIG_PDF = "decks/big deck.pdf";
|
||||
/** Fake MP4 bytes: the server only streams them, nothing decodes them. */
|
||||
export const videoBytes = (n = 4096, seed = 7) => new Uint8Array(n).map((_, i) => (i * seed) % 256);
|
||||
|
||||
export interface ImportFixtureOpts {
|
||||
id?: string;
|
||||
kind?: string;
|
||||
manifestId?: string;
|
||||
root?: string;
|
||||
visibility?: string;
|
||||
/** Extra hostile bits: a symlink to outside, a .git dir, a nested .gitignore that hides a lesson. */
|
||||
hostile?: boolean;
|
||||
}
|
||||
|
||||
function put(dir: string, rel: string, content: string | Uint8Array) {
|
||||
mkdirSync(dirname(join(dir, rel)), { recursive: true });
|
||||
writeFileSync(join(dir, rel), content);
|
||||
}
|
||||
|
||||
export function manifestYaml(o: ImportFixtureOpts = {}): string {
|
||||
const id = o.manifestId ?? o.id ?? "imp";
|
||||
return [
|
||||
"schema: 1",
|
||||
`id: ${id}`,
|
||||
"title: Imported Course",
|
||||
`visibility: ${o.visibility ?? "restricted"}`,
|
||||
...(o.root ? [`root: ${o.root}`] : []),
|
||||
"cover: media/cover.jpg",
|
||||
"source:",
|
||||
` kind: ${o.kind ?? "video-folder"}`,
|
||||
" origin: /import/synthetic",
|
||||
"decks:",
|
||||
" - id: d1",
|
||||
` file: ${BIG_PDF}`,
|
||||
" attachTo: l1",
|
||||
"modules:",
|
||||
" - id: m1",
|
||||
" title: Videos",
|
||||
" lessons:",
|
||||
" - id: l1",
|
||||
" title: Intro",
|
||||
" type: video",
|
||||
` file: "${VIDEO}"`,
|
||||
" transcript: transcripts/l1.md",
|
||||
" - id: l2",
|
||||
" title: Next step",
|
||||
" type: video",
|
||||
` file: "${VIDEO2}"`,
|
||||
" - id: notes",
|
||||
" title: Reading",
|
||||
" type: text",
|
||||
" file: code/S01 - Basics/README.md",
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
/** Writes the course into `<dataDir>/courses/<id>` and returns that folder. */
|
||||
export function makeImportedCourse(dataDir: string, o: ImportFixtureOpts = {}): string {
|
||||
const id = o.id ?? "imp";
|
||||
const dir = join(dataDir, "courses", id);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
put(dir, "stcourse.yaml", manifestYaml(o));
|
||||
put(dir, VIDEO, videoBytes(4096, 7));
|
||||
put(dir, VIDEO2, videoBytes(2048, 11));
|
||||
put(dir, "media/cover.jpg", new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 0, 0x10, 0x4a, 0x46, 0x49, 0x46, 0, 1, 0xff, 0xd9]));
|
||||
put(dir, BIG_PDF, new Uint8Array(LARGE_FILE_BYTES + 100).fill(0x25));
|
||||
put(dir, "transcripts/l1.md", "# Intro\n\n[0:00] Hello.\n");
|
||||
put(dir, "code/S01 - Basics/README.md", "# Basics\n\nRead me.\n");
|
||||
if (o.hostile) {
|
||||
// the companion code repo's own .gitignore hides Markdown: the lesson must still be committed
|
||||
put(dir, "code/S01 - Basics/.gitignore", "*.md\n");
|
||||
put(dir, "code/.git/config", "[core]\n");
|
||||
const outside = join(tempDir(), "secret.txt");
|
||||
writeFileSync(outside, "outside\n");
|
||||
symlinkSync(outside, join(dir, "code/leak.txt"));
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { $ } from "bun";
|
||||
import { gitignoreLine, isLargeMedia, LARGE_FILE_BYTES, mediaGitignore, parseMediaRefs, sha256File } from "../src/server/media-refs";
|
||||
import { tempDir } from "./helpers";
|
||||
|
||||
describe("isLargeMedia", () => {
|
||||
test("video and audio by extension, whatever the size", () => {
|
||||
for (const p of ["a.mp4", "x/B.MOV", "c.webm", "d.mkv", "e.mp3", "f.m4a", "g.wav", "h.flac", "i.avi", "j.m4v"]) expect(isLargeMedia(p, 10)).toBe(true);
|
||||
});
|
||||
test("other files only above the size limit", () => {
|
||||
expect(isLargeMedia("deck.pdf", 1000)).toBe(false);
|
||||
expect(isLargeMedia("deck.pdf", LARGE_FILE_BYTES + 1)).toBe(true);
|
||||
expect(isLargeMedia("lesson.md", LARGE_FILE_BYTES)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("gitignoreLine", () => {
|
||||
test("anchors and escapes", () => {
|
||||
expect(gitignoreLine("media/a.mp4")).toBe("/media/a.mp4");
|
||||
expect(gitignoreLine("media/[1] *?.mp4")).toBe("/media/\\[1\\] \\*\\?.mp4");
|
||||
expect(gitignoreLine("a\\b.mp4")).toBe("/a\\\\b.mp4");
|
||||
expect(gitignoreLine("trail.mp4 ")).toBe("/trail.mp4\\ \\ ");
|
||||
});
|
||||
test("refuses names a .gitignore cannot express", () => {
|
||||
expect(() => gitignoreLine("a\nb.mp4")).toThrow();
|
||||
expect(() => gitignoreLine("a\rb.mp4")).toThrow();
|
||||
});
|
||||
|
||||
test("git ignores exactly the listed paths", async () => {
|
||||
const dir = tempDir();
|
||||
await $`git init -q ${dir}`.quiet();
|
||||
const listed = ["media/01 Intro – Übersicht.mp4", "media/[draft] *star?.mp4", "#hash.mp4", "!bang.mp4", "media/trailing.mp4 ", "deep/a b/c.mkv"];
|
||||
const siblings = ["media/01 Intro – Übersicht.mp4.txt", "media/d.mp4", "hash.mp4", "bang.mp4", "media/trailing.mp4", "deep/a b/c.mkv2", "sub/#hash.mp4"];
|
||||
for (const p of [...listed, ...siblings]) {
|
||||
mkdirSync(join(dir, p, ".."), { recursive: true });
|
||||
writeFileSync(join(dir, p), "x");
|
||||
}
|
||||
writeFileSync(join(dir, ".gitignore"), mediaGitignore(listed, null));
|
||||
const input = Buffer.from([...listed, ...siblings].join("\0") + "\0");
|
||||
const r = await $`git -c core.quotepath=false check-ignore -z --no-index --stdin < ${input}`.cwd(dir).quiet().nothrow();
|
||||
const ignored = r.stdout.toString().split("\0").filter(Boolean).sort();
|
||||
expect(ignored).toEqual([...listed].sort());
|
||||
});
|
||||
|
||||
test("an existing .gitignore is kept above the media block", () => {
|
||||
const out = mediaGitignore(["a.mp4"], "node_modules/\n");
|
||||
expect(out.startsWith("node_modules/\n")).toBe(true);
|
||||
expect(out).toContain("/a.mp4\n");
|
||||
expect(mediaGitignore(["a.mp4"], "dist")).toStartWith("dist\n");
|
||||
});
|
||||
});
|
||||
|
||||
test("sha256File streams the file", async () => {
|
||||
const dir = tempDir();
|
||||
const bytes = new Uint8Array(3 * 1024 * 1024 + 7).map((_, i) => i % 251);
|
||||
writeFileSync(join(dir, "f.bin"), bytes);
|
||||
expect(await sha256File(join(dir, "f.bin"))).toBe(new Bun.CryptoHasher("sha256").update(bytes).digest("hex"));
|
||||
});
|
||||
|
||||
describe("parseMediaRefs", () => {
|
||||
const sha = "a".repeat(64);
|
||||
test("accepts the schema", () => {
|
||||
expect(parseMediaRefs(JSON.stringify({ schema: 1, files: [{ path: "media/a.mp4", size: 3, sha256: sha }] }))).toEqual([{ path: "media/a.mp4", size: 3, sha256: sha }]);
|
||||
});
|
||||
test.each([
|
||||
"not json", "null", "{}", JSON.stringify({ schema: 2, files: [] }),
|
||||
JSON.stringify({ schema: 1, files: [{ path: "../x.mp4", size: 1, sha256: sha }] }),
|
||||
JSON.stringify({ schema: 1, files: [{ path: "/abs.mp4", size: 1, sha256: sha }] }),
|
||||
JSON.stringify({ schema: 1, files: [{ path: ".git/config", size: 1, sha256: sha }] }),
|
||||
JSON.stringify({ schema: 1, files: [{ path: "a.mp4", size: -1, sha256: sha }] }),
|
||||
JSON.stringify({ schema: 1, files: [{ path: "a.mp4", size: 1, sha256: "zz" }] }),
|
||||
])("rejects %p", (text) => {
|
||||
expect(parseMediaRefs(text)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,7 @@
|
||||
import { afterEach, describe, expect, test } from "bun:test";
|
||||
import { existsSync, mkdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { IMPORTED_MESSAGE } from "../src/server/feature";
|
||||
import { isAllowedRemote, ownerRequiredFor } from "../src/server/store";
|
||||
import { ada, extGit, newStore, tempDir } from "./helpers";
|
||||
import { setupApp } from "./app-helpers";
|
||||
@@ -229,11 +230,11 @@ describe("I4: layout and imported courses", () => {
|
||||
});
|
||||
expect(env.ctx.courses.get("imp")?.status).toBe("ok");
|
||||
const list = (await env.call("ada", "GET", "/api/studio/courses")).json;
|
||||
expect(list.courses).toContainEqual(expect.objectContaining({ id: "imp", managed: false, message: "Imported course: read-only in the studio for now" }));
|
||||
expect((await env.call("ada", "GET", "/api/studio/courses/imp")).json.message).toBe("Imported course: read-only in the studio for now");
|
||||
expect(list.courses).toContainEqual(expect.objectContaining({ id: "imp", managed: false, message: IMPORTED_MESSAGE }));
|
||||
expect((await env.call("ada", "GET", "/api/studio/courses/imp")).json.message).toBe(IMPORTED_MESSAGE);
|
||||
const w = await env.call("ada", "PUT", "/api/studio/courses/imp/files", { message: "x", files: [{ path: "a.md", content: "x" }] });
|
||||
expect(w.status).toBe(403);
|
||||
expect(w.json.error).toBe("Imported course: read-only in the studio for now");
|
||||
expect(w.json.error).toBe(IMPORTED_MESSAGE);
|
||||
expect((await env.call("ada", "POST", "/api/studio/courses", { id: "imp", title: "Dup" })).status).toBe(409);
|
||||
// studio repos are not in the importer's folder
|
||||
expect((await env.call("ada", "POST", "/api/studio/courses", { id: "c1", title: "C" })).status).toBe(201);
|
||||
|
||||
Reference in New Issue
Block a user