From 596099a13f4c98822f2bc637567a61952ff16d88 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 03:17:31 +0100 Subject: [PATCH] feat(import): owner-only API routes and importFeature Co-Authored-By: Claude Opus 5.5 --- packages/feature-import/src/index.ts | 40 +++++++ packages/feature-import/src/routes.ts | 109 ++++++++++++++++++ .../feature-import/src/web/ImportPage.tsx | 1 + packages/feature-import/test/routes.test.ts | 88 ++++++++++++++ 4 files changed, 238 insertions(+) create mode 100644 packages/feature-import/src/index.ts create mode 100644 packages/feature-import/src/routes.ts create mode 100644 packages/feature-import/src/web/ImportPage.tsx create mode 100644 packages/feature-import/test/routes.test.ts diff --git a/packages/feature-import/src/index.ts b/packages/feature-import/src/index.ts new file mode 100644 index 0000000..bc9d734 --- /dev/null +++ b/packages/feature-import/src/index.ts @@ -0,0 +1,40 @@ +import type { Feature, JobHandler, ServerContext, WebFeature } from "@stc/contracts"; +import type { Deps, ImportOptions } from "./context"; +import { importCodeRepo } from "./jobs/code-repo"; +import { importPacktVideo } from "./jobs/packt-video"; +import { importPdfDeck } from "./jobs/pdf-deck"; +import { transcribeCourse } from "./jobs/transcribe"; +import { importVideoFolder } from "./jobs/video-folder"; +import { importRoutes } from "./routes"; + +export type { ImportOptions } from "./context"; +export { deckCacheDir, coverCacheDir, transcriptCacheDir } from "./context"; + +/** + * `ctx` is only read when a job runs or a request arrives, never while this function runs, so the server can pass + * its context object before the job queue (which needs these handlers) exists. + * Environment: STC_PYTHON (Python with PyMuPDF, default python3), STC_FFMPEG / STC_FFPROBE, STC_TRANSCRIBE_URL (+ _KEY, _MODEL). + */ +export function importFeature(ctx: ServerContext, opts: ImportOptions = {}): Feature { + const deps = (): Deps => { + const env = opts.env ?? process.env; + return { ctx, env, tools: { ffmpeg: env.STC_FFMPEG || "ffmpeg", ffprobe: env.STC_FFPROBE || "ffprobe" }, python: env.STC_PYTHON || "python3" }; + }; + const job = (fn: (d: Deps, p: unknown, jc: Parameters[1]) => Promise): JobHandler => (p, jc) => fn(deps(), p, jc); + return { + name: "import", + routes: importRoutes(deps), + jobs: { + "import.packt-video": job(importPacktVideo), + "import.code-repo": job(importCodeRepo), + "import.pdf-deck": job(importPdfDeck), + "import.video-folder": job(importVideoFolder), + "import.transcribe": job(transcribeCourse), + }, + }; +} + +export const importWeb: WebFeature = { + name: "import", + routes: [{ path: "/admin/import", element: () => import("./web/ImportPage"), nav: { label: "Import", area: "admin" } }], +}; diff --git a/packages/feature-import/src/routes.ts b/packages/feature-import/src/routes.ts new file mode 100644 index 0000000..f1cd2b0 --- /dev/null +++ b/packages/feature-import/src/routes.ts @@ -0,0 +1,109 @@ +import { copyFile, mkdir, readdir } from "node:fs/promises"; +import { existsSync } from "node:fs"; +import { join, relative, resolve } from "node:path"; +import type { Feature, RouteHandler } from "@stc/contracts"; +import { serveMedia } from "@stc/core"; +import { coverCacheDir, coursesDir, exists, managedCourse, saveManifest, type Deps } from "./context"; + +export const KINDS = ["packt-video", "code-repo", "pdf-deck", "video-folder"] as const; +const OPTION_KEYS = ["title", "courseId", "attachTo", "media", "transcribe"] as const; +const COVER_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*\.jpg$/; +const GIT_URL = /^(https?:\/\/|ssh:\/\/|git@)\S+$/; + +const json = (body: unknown, status = 200) => Response.json(body, { status }); + +export function importRoutes(deps: () => Deps): Feature["routes"] { + /** Imports read arbitrary server paths, so every route needs the owner role. */ + const owner = (h: RouteHandler): RouteHandler => (req, rc) => { + if (!rc.user) return json({ error: "Sign in required" }, 401); + if (!rc.user.roles.includes("owner")) return json({ error: "Only the owner can import" }, 403); + return h(req, rc); + }; + + return { + "/api/import/info": { + GET: owner(() => { + const d = deps(); + const dir = coursesDir(d); + const served = d.ctx.config.courseDirs.some((c) => resolve(c) === dir); + return json({ transcribe: Boolean(d.env.STC_TRANSCRIBE_URL), coursesDir: dir, coursesDirServed: served }); + }), + }, + "/api/import/jobs": { + GET: owner(async () => { + const jobs = (await deps().ctx.jobs.list()).filter((j) => j.kind.startsWith("import.")); + return json(jobs); + }), + POST: owner(async (req) => { + const d = deps(); + let body: any; + try { body = await req.json(); } catch { return json({ error: "Body must be JSON" }, 400); } + const kind = String(body?.kind ?? ""); + const options: Record = {}; + for (const k of OPTION_KEYS) if (body?.options?.[k] !== undefined && body.options[k] !== "") options[k] = body.options[k]; + if (kind === "transcribe") { + if (!d.env.STC_TRANSCRIBE_URL) return json({ error: "STC_TRANSCRIBE_URL is not set" }, 409); + if (typeof options.courseId !== "string") return json({ error: "courseId is required" }, 400); + return json({ id: await d.ctx.jobs.enqueue("import.transcribe", { courseId: options.courseId }) }, 202); + } + if (!(KINDS as readonly string[]).includes(kind)) return json({ error: `kind must be one of ${KINDS.join(", ")}` }, 400); + const path = typeof body?.path === "string" ? body.path.trim() : ""; + if (!path) return json({ error: "path is required" }, 400); + const isGit = kind === "code-repo" && GIT_URL.test(path); + if (!isGit && !existsSync(path)) return json({ error: `Path not found on the server: ${path}` }, 400); + if (!isGit && !path.startsWith("/")) return json({ error: "Use an absolute path" }, 400); + if ((kind === "code-repo" || kind === "pdf-deck") && typeof options.courseId !== "string") return json({ error: "courseId is required" }, 400); + if (kind === "pdf-deck" && typeof options.attachTo !== "string") return json({ error: "attachTo is required" }, 400); + return json({ id: await d.ctx.jobs.enqueue(`import.${kind}`, { kind, path, options }) }, 202); + }), + }, + "/api/import/courses": { + GET: owner(() => { + const d = deps(); + const base = coursesDir(d); + const out = d.ctx.courses.list() + .filter((c) => c.manifest && !relative(base, c.root).startsWith("..")) + .map((c) => ({ + id: c.id, title: c.manifest!.title, + modules: c.manifest!.modules.map((m) => ({ id: m.id, title: m.title, lessons: m.lessons.map((l) => ({ id: l.id, title: l.title })) })), + })); + return json(out); + }), + }, + "/api/import/covers/:courseId": { + GET: owner(async (_req, rc) => { + const d = deps(); + const id = rc.params.courseId!; + const c = d.ctx.courses.get(id); + const names = (await readdir(coverCacheDir(d, id)).catch(() => [] as string[])).filter((n) => COVER_NAME.test(n)).sort(); + return json({ current: c?.manifest?.cover ?? null, candidates: names }); + }), + POST: owner(async (req, rc) => { + const d = deps(); + let body: any; + try { body = await req.json(); } catch { return json({ error: "Body must be JSON" }, 400); } + const name = String(body?.name ?? ""); + if (!COVER_NAME.test(name)) return json({ error: "Invalid candidate name" }, 400); + const src = join(coverCacheDir(d, rc.params.courseId!), name); + if (!(await exists(src))) return json({ error: "No such candidate" }, 404); + let course; + try { course = await managedCourse(d, rc.params.courseId); } catch (e) { return json({ error: (e as Error).message }, 404); } + await mkdir(join(course.dir, "media"), { recursive: true }); + await copyFile(src, join(course.dir, "media", "cover.jpg")); + course.manifest.cover = "media/cover.jpg"; + await saveManifest(course.dir, course.manifest); + await d.ctx.courses.reload(course.id); + return json({ cover: "media/cover.jpg" }); + }), + }, + "/api/import/covers/:courseId/:name": { + GET: owner(async (req, rc) => { + const name = rc.params.name!; + if (!COVER_NAME.test(name)) return new Response("Not found", { status: 404 }); + const file = join(coverCacheDir(deps(), rc.params.courseId!), name); + if (!(await exists(file))) return new Response("Not found", { status: 404 }); + return serveMedia(req, file); + }), + }, + }; +} diff --git a/packages/feature-import/src/web/ImportPage.tsx b/packages/feature-import/src/web/ImportPage.tsx new file mode 100644 index 0000000..1d8a2c6 --- /dev/null +++ b/packages/feature-import/src/web/ImportPage.tsx @@ -0,0 +1 @@ +export default function ImportPage() { return

Import

; } diff --git a/packages/feature-import/test/routes.test.ts b/packages/feature-import/test/routes.test.ts new file mode 100644 index 0000000..a80b91e --- /dev/null +++ b/packages/feature-import/test/routes.test.ts @@ -0,0 +1,88 @@ +import { expect } from "bun:test"; +import { join } from "node:path"; +import type { AuthUser, ServerContext } from "@stc/contracts"; +import { createApp, createJobQueue } from "@stc/core"; +import { createCourseRepository } from "@stc/course-format"; +import { forEachDb } from "../../db/test/helpers"; +import { importFeature } from "../src"; +import { makeVideo, tmp } from "./fixtures"; + +const owner: AuthUser = { id: "o", email: "o@x", name: "O", roles: ["owner"], groups: [] }; +const learner: AuthUser = { id: "l", email: "l@x", name: "L", roles: ["learner"], groups: [] }; + +async function setup(db: any, env: Record = {}) { + const dataDir = await tmp("stc-data-"); + const courses = await createCourseRepository([join(dataDir, "courses")]); + const ctx = { + config: { port: 0, databaseUrl: "", dataDir, courseDirs: [join(dataDir, "courses")], secret: "t", publicUrl: "" }, + db, courses, + auth: { userFromRequest: async (req: Request) => ({ owner, learner }[req.headers.get("x-user") ?? ""] ?? null) }, + } as unknown as ServerContext; + const feature = importFeature(ctx, { env: { STC_PYTHON: process.env.STC_PYTHON, ...env } }); + const queue = createJobQueue(db, feature.jobs!, { pollMs: 10 }); + ctx.jobs = queue; + const app = createApp(ctx, [feature]); + const call = (path: string, init: RequestInit & { as?: string; json?: unknown } = {}) => + app.fetch(new Request(`http://x${path}`, { + method: init.method, headers: { ...(init.as ? { "x-user": init.as } : {}), ...(init.json !== undefined ? { "content-type": "application/json" } : {}) }, + body: init.json !== undefined ? JSON.stringify(init.json) : undefined, + })); + return { ctx, queue, call, dataDir }; +} +const until = async (f: () => Promise) => { for (let i = 0; i < 600; i++) { if (await f()) return; await Bun.sleep(20); } throw new Error("timeout"); }; + +forEachDb("only the owner reaches the import API", async (db) => { + const { call } = await setup(db); + expect((await call("/api/import/info")).status).toBe(401); + expect((await call("/api/import/info", { as: "learner" })).status).toBe(403); + expect((await call("/api/import/jobs", { as: "learner", method: "POST", json: {} })).status).toBe(403); + const r = await call("/api/import/info", { as: "owner" }); + expect(await r.json()).toMatchObject({ transcribe: false, coursesDirServed: true }); +}); + +forEachDb("job creation validates input", async (db) => { + const { call } = await setup(db); + const post = async (json: unknown) => call("/api/import/jobs", { as: "owner", method: "POST", json }); + expect((await post({ kind: "nope", path: "/tmp" })).status).toBe(400); + expect((await post({ kind: "video-folder" })).status).toBe(400); + expect((await post({ kind: "video-folder", path: "/definitely/not/here" })).status).toBe(400); + expect((await post({ kind: "pdf-deck", path: "/tmp", options: { courseId: "x" } })).status).toBe(400); + expect((await post({ kind: "code-repo", path: "/tmp" })).status).toBe(400); + expect((await post({ kind: "transcribe", options: { courseId: "x" } })).status).toBe(409); +}); + +forEachDb("a folder import runs through the real queue, then cover candidates can be chosen", async (db) => { + const { call, queue, dataDir } = await setup(db); + queue.start(); + try { + const src = join(await tmp(), "Queue Videos"); + await makeVideo(join(src, "1 - a.mp4"), 1); + await makeVideo(join(src, "2 - b.mp4"), 1); + const r = await call("/api/import/jobs", { as: "owner", method: "POST", json: { kind: "video-folder", path: src, options: { title: "Queued", bogus: 1 } } }); + expect(r.status).toBe(202); + const { id } = (await r.json()) as { id: string }; + await until(async () => ["done", "failed"].includes((await queue.get(id))!.status)); + const job = (await queue.get(id))!; + expect(job.error).toBeNull(); + expect(job.status).toBe("done"); + expect(job.result).toMatchObject({ courseId: "queued", warnings: [] }); + expect(job.payload).toEqual({ kind: "video-folder", path: src, options: { title: "Queued" } }); + + const list = (await (await call("/api/import/jobs", { as: "owner" })).json()) as any[]; + expect(list.map((j) => j.id)).toContain(id); + const courses = (await (await call("/api/import/courses", { as: "owner" })).json()) as any[]; + expect(courses).toEqual([{ id: "queued", title: "Queued", modules: [{ id: "m1", title: "Videos", lessons: [{ id: "l1", title: "a" }, { id: "l2", title: "b" }] }] }]); + + const covers = (await (await call("/api/import/covers/queued", { as: "owner" })).json()) as any; + expect(covers.current).toBe("media/cover.jpg"); + expect(covers.candidates).toEqual(["frame-1.jpg", "frame-2.jpg"]); + const img = await call("/api/import/covers/queued/frame-2.jpg", { as: "owner" }); + expect(img.headers.get("content-type")).toBe("image/jpeg"); + expect((await call("/api/import/covers/queued/frame-2.jpg")).status).toBe(401); + expect((await call("/api/import/covers/queued/..%2F..%2Fx.jpg", { as: "owner" })).status).toBe(404); + const set = await call("/api/import/covers/queued", { as: "owner", method: "POST", json: { name: "frame-2.jpg" } }); + expect(set.status).toBe(200); + expect((await call("/api/import/covers/queued", { as: "owner", method: "POST", json: { name: "../evil.jpg" } })).status).toBe(400); + expect(await Bun.file(join(dataDir, "courses/queued/media/cover.jpg")).exists()).toBe(true); + } finally { await queue.stop(); } +});