feat(import): owner-only API routes and importFeature
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Lucas Winther
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
7c3f613feb
commit
596099a13f
@@ -0,0 +1,40 @@
|
||||
import type { Feature, JobHandler, ServerContext, WebFeature } from "@stc/contracts";
|
||||
import type { Deps, ImportOptions } from "./context";
|
||||
import { importCodeRepo } from "./jobs/code-repo";
|
||||
import { importPacktVideo } from "./jobs/packt-video";
|
||||
import { importPdfDeck } from "./jobs/pdf-deck";
|
||||
import { transcribeCourse } from "./jobs/transcribe";
|
||||
import { importVideoFolder } from "./jobs/video-folder";
|
||||
import { importRoutes } from "./routes";
|
||||
|
||||
export type { ImportOptions } from "./context";
|
||||
export { deckCacheDir, coverCacheDir, transcriptCacheDir } from "./context";
|
||||
|
||||
/**
|
||||
* `ctx` is only read when a job runs or a request arrives, never while this function runs, so the server can pass
|
||||
* its context object before the job queue (which needs these handlers) exists.
|
||||
* Environment: STC_PYTHON (Python with PyMuPDF, default python3), STC_FFMPEG / STC_FFPROBE, STC_TRANSCRIBE_URL (+ _KEY, _MODEL).
|
||||
*/
|
||||
export function importFeature(ctx: ServerContext, opts: ImportOptions = {}): Feature {
|
||||
const deps = (): Deps => {
|
||||
const env = opts.env ?? process.env;
|
||||
return { ctx, env, tools: { ffmpeg: env.STC_FFMPEG || "ffmpeg", ffprobe: env.STC_FFPROBE || "ffprobe" }, python: env.STC_PYTHON || "python3" };
|
||||
};
|
||||
const job = (fn: (d: Deps, p: unknown, jc: Parameters<JobHandler>[1]) => Promise<unknown>): JobHandler => (p, jc) => fn(deps(), p, jc);
|
||||
return {
|
||||
name: "import",
|
||||
routes: importRoutes(deps),
|
||||
jobs: {
|
||||
"import.packt-video": job(importPacktVideo),
|
||||
"import.code-repo": job(importCodeRepo),
|
||||
"import.pdf-deck": job(importPdfDeck),
|
||||
"import.video-folder": job(importVideoFolder),
|
||||
"import.transcribe": job(transcribeCourse),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const importWeb: WebFeature = {
|
||||
name: "import",
|
||||
routes: [{ path: "/admin/import", element: () => import("./web/ImportPage"), nav: { label: "Import", area: "admin" } }],
|
||||
};
|
||||
@@ -0,0 +1,109 @@
|
||||
import { copyFile, mkdir, readdir } from "node:fs/promises";
|
||||
import { existsSync } from "node:fs";
|
||||
import { join, relative, resolve } from "node:path";
|
||||
import type { Feature, RouteHandler } from "@stc/contracts";
|
||||
import { serveMedia } from "@stc/core";
|
||||
import { coverCacheDir, coursesDir, exists, managedCourse, saveManifest, type Deps } from "./context";
|
||||
|
||||
export const KINDS = ["packt-video", "code-repo", "pdf-deck", "video-folder"] as const;
|
||||
const OPTION_KEYS = ["title", "courseId", "attachTo", "media", "transcribe"] as const;
|
||||
const COVER_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*\.jpg$/;
|
||||
const GIT_URL = /^(https?:\/\/|ssh:\/\/|git@)\S+$/;
|
||||
|
||||
const json = (body: unknown, status = 200) => Response.json(body, { status });
|
||||
|
||||
export function importRoutes(deps: () => Deps): Feature["routes"] {
|
||||
/** Imports read arbitrary server paths, so every route needs the owner role. */
|
||||
const owner = (h: RouteHandler): RouteHandler => (req, rc) => {
|
||||
if (!rc.user) return json({ error: "Sign in required" }, 401);
|
||||
if (!rc.user.roles.includes("owner")) return json({ error: "Only the owner can import" }, 403);
|
||||
return h(req, rc);
|
||||
};
|
||||
|
||||
return {
|
||||
"/api/import/info": {
|
||||
GET: owner(() => {
|
||||
const d = deps();
|
||||
const dir = coursesDir(d);
|
||||
const served = d.ctx.config.courseDirs.some((c) => resolve(c) === dir);
|
||||
return json({ transcribe: Boolean(d.env.STC_TRANSCRIBE_URL), coursesDir: dir, coursesDirServed: served });
|
||||
}),
|
||||
},
|
||||
"/api/import/jobs": {
|
||||
GET: owner(async () => {
|
||||
const jobs = (await deps().ctx.jobs.list()).filter((j) => j.kind.startsWith("import."));
|
||||
return json(jobs);
|
||||
}),
|
||||
POST: owner(async (req) => {
|
||||
const d = deps();
|
||||
let body: any;
|
||||
try { body = await req.json(); } catch { return json({ error: "Body must be JSON" }, 400); }
|
||||
const kind = String(body?.kind ?? "");
|
||||
const options: Record<string, unknown> = {};
|
||||
for (const k of OPTION_KEYS) if (body?.options?.[k] !== undefined && body.options[k] !== "") options[k] = body.options[k];
|
||||
if (kind === "transcribe") {
|
||||
if (!d.env.STC_TRANSCRIBE_URL) return json({ error: "STC_TRANSCRIBE_URL is not set" }, 409);
|
||||
if (typeof options.courseId !== "string") return json({ error: "courseId is required" }, 400);
|
||||
return json({ id: await d.ctx.jobs.enqueue("import.transcribe", { courseId: options.courseId }) }, 202);
|
||||
}
|
||||
if (!(KINDS as readonly string[]).includes(kind)) return json({ error: `kind must be one of ${KINDS.join(", ")}` }, 400);
|
||||
const path = typeof body?.path === "string" ? body.path.trim() : "";
|
||||
if (!path) return json({ error: "path is required" }, 400);
|
||||
const isGit = kind === "code-repo" && GIT_URL.test(path);
|
||||
if (!isGit && !existsSync(path)) return json({ error: `Path not found on the server: ${path}` }, 400);
|
||||
if (!isGit && !path.startsWith("/")) return json({ error: "Use an absolute path" }, 400);
|
||||
if ((kind === "code-repo" || kind === "pdf-deck") && typeof options.courseId !== "string") return json({ error: "courseId is required" }, 400);
|
||||
if (kind === "pdf-deck" && typeof options.attachTo !== "string") return json({ error: "attachTo is required" }, 400);
|
||||
return json({ id: await d.ctx.jobs.enqueue(`import.${kind}`, { kind, path, options }) }, 202);
|
||||
}),
|
||||
},
|
||||
"/api/import/courses": {
|
||||
GET: owner(() => {
|
||||
const d = deps();
|
||||
const base = coursesDir(d);
|
||||
const out = d.ctx.courses.list()
|
||||
.filter((c) => c.manifest && !relative(base, c.root).startsWith(".."))
|
||||
.map((c) => ({
|
||||
id: c.id, title: c.manifest!.title,
|
||||
modules: c.manifest!.modules.map((m) => ({ id: m.id, title: m.title, lessons: m.lessons.map((l) => ({ id: l.id, title: l.title })) })),
|
||||
}));
|
||||
return json(out);
|
||||
}),
|
||||
},
|
||||
"/api/import/covers/:courseId": {
|
||||
GET: owner(async (_req, rc) => {
|
||||
const d = deps();
|
||||
const id = rc.params.courseId!;
|
||||
const c = d.ctx.courses.get(id);
|
||||
const names = (await readdir(coverCacheDir(d, id)).catch(() => [] as string[])).filter((n) => COVER_NAME.test(n)).sort();
|
||||
return json({ current: c?.manifest?.cover ?? null, candidates: names });
|
||||
}),
|
||||
POST: owner(async (req, rc) => {
|
||||
const d = deps();
|
||||
let body: any;
|
||||
try { body = await req.json(); } catch { return json({ error: "Body must be JSON" }, 400); }
|
||||
const name = String(body?.name ?? "");
|
||||
if (!COVER_NAME.test(name)) return json({ error: "Invalid candidate name" }, 400);
|
||||
const src = join(coverCacheDir(d, rc.params.courseId!), name);
|
||||
if (!(await exists(src))) return json({ error: "No such candidate" }, 404);
|
||||
let course;
|
||||
try { course = await managedCourse(d, rc.params.courseId); } catch (e) { return json({ error: (e as Error).message }, 404); }
|
||||
await mkdir(join(course.dir, "media"), { recursive: true });
|
||||
await copyFile(src, join(course.dir, "media", "cover.jpg"));
|
||||
course.manifest.cover = "media/cover.jpg";
|
||||
await saveManifest(course.dir, course.manifest);
|
||||
await d.ctx.courses.reload(course.id);
|
||||
return json({ cover: "media/cover.jpg" });
|
||||
}),
|
||||
},
|
||||
"/api/import/covers/:courseId/:name": {
|
||||
GET: owner(async (req, rc) => {
|
||||
const name = rc.params.name!;
|
||||
if (!COVER_NAME.test(name)) return new Response("Not found", { status: 404 });
|
||||
const file = join(coverCacheDir(deps(), rc.params.courseId!), name);
|
||||
if (!(await exists(file))) return new Response("Not found", { status: 404 });
|
||||
return serveMedia(req, file);
|
||||
}),
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export default function ImportPage() { return <p>Import</p>; }
|
||||
@@ -0,0 +1,88 @@
|
||||
import { expect } from "bun:test";
|
||||
import { join } from "node:path";
|
||||
import type { AuthUser, ServerContext } from "@stc/contracts";
|
||||
import { createApp, createJobQueue } from "@stc/core";
|
||||
import { createCourseRepository } from "@stc/course-format";
|
||||
import { forEachDb } from "../../db/test/helpers";
|
||||
import { importFeature } from "../src";
|
||||
import { makeVideo, tmp } from "./fixtures";
|
||||
|
||||
const owner: AuthUser = { id: "o", email: "o@x", name: "O", roles: ["owner"], groups: [] };
|
||||
const learner: AuthUser = { id: "l", email: "l@x", name: "L", roles: ["learner"], groups: [] };
|
||||
|
||||
async function setup(db: any, env: Record<string, string | undefined> = {}) {
|
||||
const dataDir = await tmp("stc-data-");
|
||||
const courses = await createCourseRepository([join(dataDir, "courses")]);
|
||||
const ctx = {
|
||||
config: { port: 0, databaseUrl: "", dataDir, courseDirs: [join(dataDir, "courses")], secret: "t", publicUrl: "" },
|
||||
db, courses,
|
||||
auth: { userFromRequest: async (req: Request) => ({ owner, learner }[req.headers.get("x-user") ?? ""] ?? null) },
|
||||
} as unknown as ServerContext;
|
||||
const feature = importFeature(ctx, { env: { STC_PYTHON: process.env.STC_PYTHON, ...env } });
|
||||
const queue = createJobQueue(db, feature.jobs!, { pollMs: 10 });
|
||||
ctx.jobs = queue;
|
||||
const app = createApp(ctx, [feature]);
|
||||
const call = (path: string, init: RequestInit & { as?: string; json?: unknown } = {}) =>
|
||||
app.fetch(new Request(`http://x${path}`, {
|
||||
method: init.method, headers: { ...(init.as ? { "x-user": init.as } : {}), ...(init.json !== undefined ? { "content-type": "application/json" } : {}) },
|
||||
body: init.json !== undefined ? JSON.stringify(init.json) : undefined,
|
||||
}));
|
||||
return { ctx, queue, call, dataDir };
|
||||
}
|
||||
const until = async (f: () => Promise<boolean>) => { for (let i = 0; i < 600; i++) { if (await f()) return; await Bun.sleep(20); } throw new Error("timeout"); };
|
||||
|
||||
forEachDb("only the owner reaches the import API", async (db) => {
|
||||
const { call } = await setup(db);
|
||||
expect((await call("/api/import/info")).status).toBe(401);
|
||||
expect((await call("/api/import/info", { as: "learner" })).status).toBe(403);
|
||||
expect((await call("/api/import/jobs", { as: "learner", method: "POST", json: {} })).status).toBe(403);
|
||||
const r = await call("/api/import/info", { as: "owner" });
|
||||
expect(await r.json()).toMatchObject({ transcribe: false, coursesDirServed: true });
|
||||
});
|
||||
|
||||
forEachDb("job creation validates input", async (db) => {
|
||||
const { call } = await setup(db);
|
||||
const post = async (json: unknown) => call("/api/import/jobs", { as: "owner", method: "POST", json });
|
||||
expect((await post({ kind: "nope", path: "/tmp" })).status).toBe(400);
|
||||
expect((await post({ kind: "video-folder" })).status).toBe(400);
|
||||
expect((await post({ kind: "video-folder", path: "/definitely/not/here" })).status).toBe(400);
|
||||
expect((await post({ kind: "pdf-deck", path: "/tmp", options: { courseId: "x" } })).status).toBe(400);
|
||||
expect((await post({ kind: "code-repo", path: "/tmp" })).status).toBe(400);
|
||||
expect((await post({ kind: "transcribe", options: { courseId: "x" } })).status).toBe(409);
|
||||
});
|
||||
|
||||
forEachDb("a folder import runs through the real queue, then cover candidates can be chosen", async (db) => {
|
||||
const { call, queue, dataDir } = await setup(db);
|
||||
queue.start();
|
||||
try {
|
||||
const src = join(await tmp(), "Queue Videos");
|
||||
await makeVideo(join(src, "1 - a.mp4"), 1);
|
||||
await makeVideo(join(src, "2 - b.mp4"), 1);
|
||||
const r = await call("/api/import/jobs", { as: "owner", method: "POST", json: { kind: "video-folder", path: src, options: { title: "Queued", bogus: 1 } } });
|
||||
expect(r.status).toBe(202);
|
||||
const { id } = (await r.json()) as { id: string };
|
||||
await until(async () => ["done", "failed"].includes((await queue.get(id))!.status));
|
||||
const job = (await queue.get(id))!;
|
||||
expect(job.error).toBeNull();
|
||||
expect(job.status).toBe("done");
|
||||
expect(job.result).toMatchObject({ courseId: "queued", warnings: [] });
|
||||
expect(job.payload).toEqual({ kind: "video-folder", path: src, options: { title: "Queued" } });
|
||||
|
||||
const list = (await (await call("/api/import/jobs", { as: "owner" })).json()) as any[];
|
||||
expect(list.map((j) => j.id)).toContain(id);
|
||||
const courses = (await (await call("/api/import/courses", { as: "owner" })).json()) as any[];
|
||||
expect(courses).toEqual([{ id: "queued", title: "Queued", modules: [{ id: "m1", title: "Videos", lessons: [{ id: "l1", title: "a" }, { id: "l2", title: "b" }] }] }]);
|
||||
|
||||
const covers = (await (await call("/api/import/covers/queued", { as: "owner" })).json()) as any;
|
||||
expect(covers.current).toBe("media/cover.jpg");
|
||||
expect(covers.candidates).toEqual(["frame-1.jpg", "frame-2.jpg"]);
|
||||
const img = await call("/api/import/covers/queued/frame-2.jpg", { as: "owner" });
|
||||
expect(img.headers.get("content-type")).toBe("image/jpeg");
|
||||
expect((await call("/api/import/covers/queued/frame-2.jpg")).status).toBe(401);
|
||||
expect((await call("/api/import/covers/queued/..%2F..%2Fx.jpg", { as: "owner" })).status).toBe(404);
|
||||
const set = await call("/api/import/covers/queued", { as: "owner", method: "POST", json: { name: "frame-2.jpg" } });
|
||||
expect(set.status).toBe(200);
|
||||
expect((await call("/api/import/covers/queued", { as: "owner", method: "POST", json: { name: "../evil.jpg" } })).status).toBe(400);
|
||||
expect(await Bun.file(join(dataDir, "courses/queued/media/cover.jpg")).exists()).toBe(true);
|
||||
} finally { await queue.stop(); }
|
||||
});
|
||||
Reference in New Issue
Block a user